From 3991a1d73c36449cd7e7f02a6301a377b12f5c05 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 08:10:38 +0800 Subject: [PATCH 01/52] test(ecstore): stabilize late snapshot lease cleanup (#5456) test(ecstore): wait for late snapshot lease cleanup --- crates/ecstore/src/set_disk/mod.rs | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 99accf80e..873837acd 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -5849,23 +5849,27 @@ mod tests { crate::disk::DataDirDeleteStatus::Deleted ); release_slow_candidate.notify_one(); - for _ in 0..10 { - tokio::task::yield_now().await; - } - assert_eq!( - disk2 - .delete_data_dir( - bucket, - data_dir, - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .expect("a token acquired after the deadline must be released"), - crate::disk::DataDirDeleteStatus::Deleted - ); + timeout(Duration::from_secs(1), async { + loop { + match disk2 + .delete_data_dir( + bucket, + data_dir, + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await + .expect("a token acquired after the deadline must be released") + { + crate::disk::DataDirDeleteStatus::Deleted => return, + crate::disk::DataDirDeleteStatus::Deferred => tokio::time::sleep(Duration::from_millis(1)).await, + } + } + }) + .await + .expect("late snapshot lease cleanup must finish within the bounded wait"); } #[tokio::test(start_paused = true)] From b097c94c59a7b91dbe516f868e09917ba7628cab Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 08:52:37 +0800 Subject: [PATCH 02/52] ci: model server as a composition root (#5459) * ci: model server as a composition root * test(ci): cover server to storage layer flow --- .agents/skills/arch-checks/SKILL.md | 10 ++- scripts/check_layer_dependencies.sh | 96 +++++++++++++++++++++++++-- scripts/layer-dependency-baseline.txt | 49 ++++++++------ 3 files changed, 127 insertions(+), 28 deletions(-) diff --git a/.agents/skills/arch-checks/SKILL.md b/.agents/skills/arch-checks/SKILL.md index d355a4fbd..a95347830 100644 --- a/.agents/skills/arch-checks/SKILL.md +++ b/.agents/skills/arch-checks/SKILL.md @@ -10,10 +10,16 @@ never weaken a check to get green. ## `check_layer_dependencies.sh` — layer DAG in `rustfs/src` -Enforces `interface (admin, storage/ecfs, storage/s3_api) → app → infra`; no -upward imports. Known legacy violations live in +Enforces `composition (server, startup/init) → interface (admin, +storage/ecfs, storage/s3_api) → app → infra`; no upward imports. Server source +files are composition roots, while imports of their exported HTTP contracts +are classified as interface dependencies. Known legacy violations live in `scripts/layer-dependency-baseline.txt`. +Dedicated `*_test.rs` and `tests/` modules are outside this production guard. +Inline `#[cfg(test)]` imports remain checked under their source file's layer; +move architecture-crossing test scaffolding into a dedicated test module. + - **New violation**: restructure your change so the dependency points downward (move the shared type/function to the lower layer). - **You legitimately removed a baseline entry**: run diff --git a/scripts/check_layer_dependencies.sh b/scripts/check_layer_dependencies.sh index 32f41fc07..bbb445feb 100755 --- a/scripts/check_layer_dependencies.sh +++ b/scripts/check_layer_dependencies.sh @@ -13,7 +13,14 @@ fi classify_source_layer() { local file="$1" - if [[ "$file" == rustfs/src/app/* ]]; then + if [[ "$file" == rustfs/src/server/* ]] || + [[ "$file" == rustfs/src/startup_*.rs ]] || + [[ "$file" == rustfs/src/init.rs ]] || + [[ "$file" == rustfs/src/main.rs ]] || + [[ "$file" == rustfs/src/lib.rs ]] || + [[ "$file" == rustfs/src/embedded.rs ]]; then + printf 'composition' + elif [[ "$file" == rustfs/src/app/* ]]; then printf 'app' elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]]; then printf 'interface' @@ -30,6 +37,14 @@ classify_target_layer() { local storage_path case "$root" in + init | main | lib | embedded | startup_*) + printf 'composition' + ;; + server) + # Server files are composition roots when they import lower layers, but + # their exported HTTP contracts belong to the interface boundary. + printf 'interface' + ;; app) printf 'app' ;; @@ -53,6 +68,9 @@ classify_target_layer() { layer_rank() { case "$1" in + composition) + printf '4' + ;; interface) printf '3' ;; @@ -68,6 +86,50 @@ layer_rank() { esac } +is_reverse_dependency() { + local source_rank target_rank + + source_rank="$(layer_rank "$1")" + target_rank="$(layer_rank "$2")" + (( source_rank < target_rank )) +} + +assert_dependency_direction() { + local expected="$1" + local source="$2" + local target="$3" + local actual='allowed' + + if is_reverse_dependency "$source" "$target"; then + actual='reverse' + fi + if [[ "$actual" != "$expected" ]]; then + printf 'Layer dependency guard self-test failed: %s -> %s (expected %s, got %s)\n' \ + "$source" "$target" "$expected" "$actual" >&2 + exit 1 + fi +} + +run_layer_model_self_tests() { + local server_source app_source storage_source server_target admin_target app_target storage_target + + server_source="$(classify_source_layer rustfs/src/server/http.rs)" + app_source="$(classify_source_layer rustfs/src/app/bucket_usecase.rs)" + storage_source="$(classify_source_layer rustfs/src/storage/rpc/node_service.rs)" + server_target="$(classify_target_layer server::http)" + admin_target="$(classify_target_layer admin::router)" + app_target="$(classify_target_layer app::bucket_usecase)" + storage_target="$(classify_target_layer storage::rpc)" + + assert_dependency_direction 'allowed' "$server_source" "$admin_target" + assert_dependency_direction 'allowed' "$server_source" "$app_target" + assert_dependency_direction 'allowed' "$server_source" "$storage_target" + assert_dependency_direction 'reverse' "$app_source" "$server_target" + assert_dependency_direction 'reverse' "$storage_source" "$server_target" + assert_dependency_direction 'reverse' "$app_source" "$admin_target" + assert_dependency_direction 'reverse' "$storage_source" "$admin_target" +} + normalize_import_group_item() { local prefix="$1" local item="$2" @@ -182,6 +244,11 @@ normalize_import_path() { emit_crate_use_statements() { (cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do + # The guard is file-scoped: dedicated test modules are excluded, while + # inline #[cfg(test)] imports remain subject to the source file's layer. + if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then + continue + fi perl -0777 -ne ' while (/\buse\s+crate::.*?;/sg) { my $statement = $&; @@ -194,6 +261,26 @@ emit_crate_use_statements() { done) } +write_baseline_file() { + local entries="$1" + + cat >"$BASELINE_FILE" <<'EOF' +# Layer dependency baseline for the rustfs binary crate. +# +# The guard models production imports as: +# composition -> interface -> app -> infra +# +# Canonical dependency entry: +# dep|source_file|source_layer->target_layer|crate::imported_symbol +# +# Canonical conceptual cycle entry: +# cycle|left_layer<->right_layer +EOF + cat "$entries" >>"$BASELINE_FILE" +} + +run_layer_model_self_tests + normalize_baseline_file() { local input="$1" local output="$2" @@ -265,10 +352,7 @@ while IFS= read -r line; do printf '%s->%s\n' "$source_layer" "$target_layer" >>"$EDGES_RAW" fi - source_rank="$(layer_rank "$source_layer")" - target_rank="$(layer_rank "$target_layer")" - - if (( source_rank < target_rank )); then + if is_reverse_dependency "$source_layer" "$target_layer"; then printf 'dep|%s|%s->%s|crate::%s\n' "$file" "$source_layer" "$target_layer" "$import_path" >>"$VIOLATIONS_RAW" fi done < <(normalize_import_path "$text") @@ -292,7 +376,7 @@ done <"${TMP_DIR}/edges_sorted.txt" | sort -u >"${TMP_DIR}/cycles_sorted.txt" cat "${TMP_DIR}/violations_sorted.txt" "${TMP_DIR}/cycles_sorted.txt" | sort -u >"$CURRENT_BASELINE" if [[ "$MODE" == "update" ]]; then - cp "$CURRENT_BASELINE" "$BASELINE_FILE" + write_baseline_file "$CURRENT_BASELINE" echo "Updated baseline: $BASELINE_FILE" exit 0 fi diff --git a/scripts/layer-dependency-baseline.txt b/scripts/layer-dependency-baseline.txt index 9b41b7b12..33354b8eb 100644 --- a/scripts/layer-dependency-baseline.txt +++ b/scripts/layer-dependency-baseline.txt @@ -1,35 +1,44 @@ # Layer dependency baseline for the rustfs binary crate. # -# These are intra-crate module references within rustfs/ that cross the -# conceptual layer boundaries (app, infra/server, interface/admin). -# Since they live inside one Cargo crate, Rust doesn't enforce separation. -# The list is maintained for architectural awareness during code review. +# The guard models production imports as: +# composition -> interface -> app -> infra # -# Format for dependency entries: -# status|source_file|direction|imported_symbol|reason +# Canonical dependency entry: +# dep|source_file|source_layer->target_layer|crate::imported_symbol # -# Format for conceptual cycles: -# status|cycle|direction_pair|reason -# -# Status: -# accepted - reviewed and intentionally allowed -# todo - should be resolved in a future refactor - +# Canonical conceptual cycle entry: +# cycle|left_layer<->right_layer cycle|app<->infra cycle|app<->interface +cycle|composition<->infra +cycle|composition<->interface cycle|infra<->interface +dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::ENV_SCANNER_ENABLED +dep|rustfs/src/admin/handlers/scanner.rs|interface->composition|crate::startup_background::scanner_enabled_from_env +dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::DependencyReadiness +dep|rustfs/src/app/admin_usecase.rs|app->interface|crate::server::collect_dependency_readiness_report dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook -dep|rustfs/src/init.rs|infra->interface|crate::admin +dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::server::RemoteAddr +dep|rustfs/src/app/object_usecase.rs|app->interface|crate::server::convert_ecstore_object_info +dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::DependencyReadiness +dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::DependencyReadinessReport +dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::ReadinessDegradedReason +dep|rustfs/src/cluster_snapshot.rs|infra->interface|crate::server::snapshot_dependency_readiness_report dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context -dep|rustfs/src/server/http.rs|infra->interface|crate::admin -dep|rustfs/src/server/layer.rs|infra->interface|crate::admin::console::is_console_path +dep|rustfs/src/storage/access.rs|infra->interface|crate::server::RemoteAddr +dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::server::cors dep|rustfs/src/storage/ecfs_extend.rs|infra->interface|crate::storage::ecfs::ListObjectUnorderedQuery -dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::ecfs::FS -dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::ecfs::validate_object_lock_configuration_input -dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::s3_api::common::rustfs_initiator -dep|rustfs/src/storage/ecfs_test.rs|infra->interface|crate::storage::s3_api::common::rustfs_owner +dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::convert_ecstore_object_info +dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_audit_module_enabled +dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::is_notify_module_enabled +dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_audit_module_enabled +dep|rustfs/src/storage/helper.rs|infra->interface|crate::server::refresh_notify_module_enabled +dep|rustfs/src/storage/rpc/http_service.rs|infra->interface|crate::server::RPC_PREFIX dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_dynamic_config_runtime_state dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::config::reload_runtime_config_snapshot dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::admin::service::site_replication::reload_site_replication_runtime_state +dep|rustfs/src/storage/rpc/node_service.rs|infra->interface|crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM +dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::heal_enabled_from_env +dep|rustfs/src/storage/rpc/node_service/heal.rs|infra->composition|crate::startup_background::scanner_enabled_from_env From 920705417c33315492f43d6e248d56b610623226 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 08:53:16 +0800 Subject: [PATCH 03/52] refactor(ecstore): correct bucket config static names (#5458) refactor(ecstore): fix bucket config static names --- crates/ecstore/src/store/bucket.rs | 7 ++++--- crates/ecstore/src/store/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 19408114a..46c9e7aa0 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -310,12 +310,13 @@ impl ECStore { meta.set_created(opts.created_at); if opts.lock_enabled { - meta.object_lock_config_xml = crate::bucket::utils::serialize::(&enableObjcetLockConfig)?; - meta.versioning_config_xml = crate::bucket::utils::serialize::(&enableVersioningConfig)?; + meta.object_lock_config_xml = + crate::bucket::utils::serialize::(&ENABLED_OBJECT_LOCK_CONFIG)?; + meta.versioning_config_xml = crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG)?; } if opts.versioning_enabled { - meta.versioning_config_xml = crate::bucket::utils::serialize::(&enableVersioningConfig)?; + meta.versioning_config_xml = crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG)?; } await_bucket_namespace_operation( diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 4c9ea3039..6ddb6d5bb 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -428,11 +428,11 @@ impl ECStore { } lazy_static! { - static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration { + static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration { object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), ..Default::default() }; - static ref enableVersioningConfig: VersioningConfiguration = VersioningConfiguration { + static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration { status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)), ..Default::default() }; From 2d4f77fd3b2457f64f362ae6578c9e4608318ad7 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 08:55:23 +0800 Subject: [PATCH 04/52] fix(iam): add correctly named policy APIs (#5457) --- crates/iam/src/manager.rs | 30 ++++++++++++++++++++++++--- crates/iam/src/store/object.rs | 4 ++-- crates/iam/src/sys.rs | 25 +++++++++++++++++----- rustfs/src/admin/handlers/policies.rs | 2 +- rustfs/src/admin/handlers/user.rs | 2 +- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/crates/iam/src/manager.rs b/crates/iam/src/manager.rs index feb53fbe3..3acd8bdab 100644 --- a/crates/iam/src/manager.rs +++ b/crates/iam/src/manager.rs @@ -556,7 +556,7 @@ where Ok(now) } - pub async fn list_polices(&self, bucket_name: &str) -> Result> { + pub async fn list_policies(&self, bucket_name: &str) -> Result> { let mut m = HashMap::new(); self.api.load_policy_docs(&mut m).await?; @@ -588,6 +588,15 @@ where Ok(filtered) } + /// Backward-compatible misspelling retained until the next breaking release. + #[deprecated( + since = "1.0.0", + note = "use list_policies instead; this alias will be removed in the next breaking release" + )] + pub async fn list_polices(&self, bucket_name: &str) -> Result> { + self.list_policies(bucket_name).await + } + pub async fn merge_policies(&self, name: &str) -> (String, Policy) { let mut policies = Vec::new(); let mut to_merge = Vec::new(); @@ -2184,7 +2193,7 @@ where } } -pub fn get_default_policyes() -> HashMap { +pub fn get_default_policies() -> HashMap { let default_policies = &DEFAULT_POLICIES; default_policies .iter() @@ -2201,6 +2210,15 @@ pub fn get_default_policyes() -> HashMap { .collect() } +/// Backward-compatible misspelling retained until the next breaking release. +#[deprecated( + since = "1.0.0", + note = "use get_default_policies instead; this alias will be removed in the next breaking release" +)] +pub fn get_default_policyes() -> HashMap { + get_default_policies() +} + fn set_default_canned_policies(policies: &mut HashMap) { let default_policies = &DEFAULT_POLICIES; for (k, v) in default_policies.iter() { @@ -2900,7 +2918,7 @@ mod tests { #[test] fn test_get_default_policies() { - let policies = get_default_policyes(); + let policies = get_default_policies(); // Should contain some default policies assert!(!policies.is_empty()); @@ -2913,6 +2931,12 @@ mod tests { } } + #[test] + #[allow(deprecated)] + fn deprecated_get_default_policyes_matches_current_api() { + assert_eq!(get_default_policyes().len(), get_default_policies().len()); + } + #[test] fn test_get_token_signing_key() { // This function returns the global action credential's secret key diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index 500cc3510..f7e652118 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -22,7 +22,7 @@ use crate::{ cache::{Cache, CacheEntity}, error::{is_err_no_such_policy, is_err_no_such_user}, keyring, - manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policyes}, + manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policies}, root_credentials, }; use futures::future::join_all; @@ -1127,7 +1127,7 @@ impl Store for ObjectStore { let cache_snapshot = cache.snapshot(); let listed_config_items = self.list_all_iamconfig_items().await?; - let mut policy_docs_cache = CacheEntity::new(get_default_policyes()); + let mut policy_docs_cache = CacheEntity::new(get_default_policies()); if let Some(policies_list) = listed_config_items.get(POLICIES_LIST_KEY) { // Load in fixed-size chunks so each policy is fetched exactly once. diff --git a/crates/iam/src/sys.rs b/crates/iam/src/sys.rs index 608632b20..7a4822a1a 100644 --- a/crates/iam/src/sys.rs +++ b/crates/iam/src/sys.rs @@ -18,7 +18,7 @@ use crate::error::is_err_no_such_temp_account; use crate::error::{Error, Result}; use crate::federation::OIDC_VIRTUAL_PARENT_CLAIM; use crate::manager::extract_jwt_claims; -use crate::manager::get_default_policyes; +use crate::manager::get_default_policies; use crate::manager::{IamCache, IamSyncMetricsSnapshot}; use crate::store::GroupInfo; use crate::store::MappedPolicy; @@ -249,7 +249,7 @@ impl IamSys { } pub async fn delete_policy(&self, name: &str, notify: bool) -> Result<()> { - for k in get_default_policyes().keys() { + for k in get_default_policies().keys() { if k == name { return Err(Error::other("system policy can not be deleted")); } @@ -291,8 +291,17 @@ impl IamSys { self.store.api.load_mapped_policies(user_type, is_group, m).await } + pub async fn list_policies(&self, bucket_name: &str) -> Result> { + self.store.list_policies(bucket_name).await + } + + /// Backward-compatible misspelling retained until the next breaking release. + #[deprecated( + since = "1.0.0", + note = "use list_policies instead; this alias will be removed in the next breaking release" + )] pub async fn list_polices(&self, bucket_name: &str) -> Result> { - self.store.list_polices(bucket_name).await + self.list_policies(bucket_name).await } pub async fn list_policy_docs(&self, bucket_name: &str) -> Result> { @@ -1683,11 +1692,17 @@ mod tests { use super::*; use crate::cache::{Cache, CacheEntity}; use crate::error::Error; - use crate::manager::get_default_policyes; + use crate::manager::get_default_policies; use crate::store::{GroupInfo, MappedPolicy, Store, UserType}; use rustfs_credentials::{Credentials, init_global_action_credentials}; use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata}; use rustfs_policy::policy::Args; + + #[test] + #[allow(deprecated)] + fn deprecated_list_polices_api_is_available() { + let _ = IamSys::::list_polices; + } use rustfs_policy::policy::action::{Action, AdminAction, S3Action}; use rustfs_policy::policy::policy_uses_existing_object_tag_conditions; use serde_json::Value; @@ -1925,7 +1940,7 @@ mod tests { } async fn load_all(&self, cache: &Cache) -> Result<()> { - let mut policy_docs = get_default_policyes(); + let mut policy_docs = get_default_policies(); let custom_claim_policy = Policy::parse_config(CUSTOM_STS_CLAIM_POLICY_JSON.as_bytes()).expect("custom STS claim policy should parse"); policy_docs.insert(CUSTOM_STS_CLAIM_POLICY.to_string(), PolicyDoc::new(custom_claim_policy)); diff --git a/rustfs/src/admin/handlers/policies.rs b/rustfs/src/admin/handlers/policies.rs index ad7cddc8f..354920acb 100644 --- a/rustfs/src/admin/handlers/policies.rs +++ b/rustfs/src/admin/handlers/policies.rs @@ -148,7 +148,7 @@ impl Operation for ListCannedPolicies { return Err(s3_error!(InternalError, "iam is not initialized")); }; - let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| { + let policies = iam_store.list_policies(&query.bucket).await.map_err(|e| { warn!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_POLICY, diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index 463cd02b8..f1e1ef099 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -724,7 +724,7 @@ impl Operation for ExportIam { match file { ALL_POLICIES_FILE => { let policies: HashMap = iam_store - .list_polices("") + .list_policies("") .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; let json_str = serde_json::to_vec(&policies) From 2e5cef513fb31e25940f1c77e208f28b302561bb Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 09:31:28 +0800 Subject: [PATCH 05/52] chore(release): prepare 1.0.0-beta.12 (#5461) * chore(release): prepare 1.0.0-beta.12 * chore(release): align release assets for 1.0.0-beta.12 * chore(deps): refresh release dependencies Co-Authored-By: heihutu --------- Co-authored-by: houseme Co-authored-by: heihutu --- Cargo.lock | 246 ++++++++++++++++++++--------------------- Cargo.toml | 102 ++++++++--------- README.md | 2 +- README_ZH.md | 2 +- flake.nix | 2 +- helm/rustfs/Chart.yaml | 4 +- rustfs.spec | 7 +- 7 files changed, 184 insertions(+), 181 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cf81a4e3..6b927192e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -642,9 +642,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -870,7 +870,7 @@ dependencies = [ "bytes", "fastrand", "hex", - "http 1.4.2", + "http 1.5.0", "sha1 0.10.7", "time", "tokio", @@ -934,7 +934,7 @@ dependencies = [ "bytes-utils", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.1.0", "percent-encoding", @@ -970,7 +970,7 @@ dependencies = [ "hex", "hmac 0.13.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "lru 0.16.4", "percent-encoding", @@ -1001,7 +1001,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1027,7 +1027,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1054,7 +1054,7 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1076,7 +1076,7 @@ dependencies = [ "hex", "hmac 0.13.0", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "p256 0.13.2", "percent-encoding", "sha2 0.11.0", @@ -1108,7 +1108,7 @@ dependencies = [ "bytes", "crc-fast", "hex", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "md-5 0.11.0", @@ -1142,7 +1142,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "percent-encoding", @@ -1161,7 +1161,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.4.2", + "http 1.5.0", "hyper", "hyper-rustls", "hyper-util", @@ -1224,7 +1224,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.1.0", "http-body-util", @@ -1245,7 +1245,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", @@ -1271,7 +1271,7 @@ checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -1285,7 +1285,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.1.0", "http-body-util", @@ -1337,7 +1337,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -1368,7 +1368,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "mime", @@ -3229,7 +3229,7 @@ dependencies = [ "futures-util", "headers", "htmlescape", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "libc", @@ -3654,7 +3654,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "e2e_test" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "anyhow", "astral-tokio-tar", @@ -3672,7 +3672,7 @@ dependencies = [ "flate2", "futures", "hex", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-util", @@ -4384,7 +4384,7 @@ dependencies = [ "google-cloud-gax", "hex", "hmac 0.13.0", - "http 1.4.2", + "http 1.5.0", "jsonwebtoken 10.4.0", "reqwest", "rustc_version", @@ -4409,7 +4409,7 @@ dependencies = [ "futures", "google-cloud-rpc", "google-cloud-wkt", - "http 1.4.2", + "http 1.5.0", "pin-project", "rand 0.10.2", "serde", @@ -4431,7 +4431,7 @@ dependencies = [ "google-cloud-rpc", "google-cloud-wkt", "h2", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -4544,7 +4544,7 @@ dependencies = [ "google-cloud-type", "google-cloud-wkt", "hex", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "md5", "percent-encoding", @@ -4624,7 +4624,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "indexmap 2.14.0", "slab", "tokio", @@ -4727,7 +4727,7 @@ dependencies = [ "base64 0.22.1", "bytes", "headers-core", - "http 1.4.2", + "http 1.5.0", "httpdate", "mime", "sha1 0.10.7", @@ -4739,7 +4739,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -4961,9 +4961,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -4987,7 +4987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -4998,7 +4998,7 @@ checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "pin-project-lite", ] @@ -5044,7 +5044,7 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "httparse", "httpdate", @@ -5061,7 +5061,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.2", + "http 1.5.0", "hyper", "hyper-util", "log", @@ -5095,7 +5095,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "hyper", "ipnet", @@ -6704,10 +6704,10 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", - "http 1.4.2", + "http 1.5.0", "rand 0.8.7", "serde", "serde_json", @@ -6815,7 +6815,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "humantime", "itertools 0.14.0", "parking_lot", @@ -6871,7 +6871,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek 2.2.0", "hmac 0.12.1", - "http 1.4.2", + "http 1.5.0", "itertools 0.10.5", "log", "oauth2", @@ -6932,7 +6932,7 @@ checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", - "http 1.4.2", + "http 1.5.0", "opentelemetry", "reqwest", ] @@ -6944,7 +6944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "flate2", - "http 1.4.2", + "http 1.5.0", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -7820,7 +7820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -7840,7 +7840,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph 0.8.3", @@ -7861,7 +7861,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", @@ -7874,7 +7874,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.119", @@ -8329,9 +8329,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b9503711b03773e43b31668c7b5bd279ee7cd9b7d18cff7c23a42cc1d08e5a" +checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84" dependencies = [ "arc-swap", "arcstr", @@ -8481,7 +8481,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -8627,7 +8627,7 @@ dependencies = [ "async-tungstenite", "futures-io", "futures-util", - "http 1.4.2", + "http 1.5.0", "rustls-native-certs", "rustls-pki-types", "rustls-webpki", @@ -8657,7 +8657,7 @@ dependencies = [ "flume", "futures-io", "futures-util", - "http 1.4.2", + "http 1.5.0", "log", "mqttbytes-core-next", "rumqttc-core-next", @@ -8845,7 +8845,7 @@ dependencies = [ [[package]] name = "rustfs" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "aes-gcm", "anyhow", @@ -8874,7 +8874,7 @@ dependencies = [ "hex-simd", "hmac 0.13.0", "hotpath", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -8981,7 +8981,7 @@ dependencies = [ [[package]] name = "rustfs-audit" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "chrono", @@ -9003,12 +9003,12 @@ dependencies = [ [[package]] name = "rustfs-checksums" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "base64-simd", "bytes", "crc-fast", - "http 1.4.2", + "http 1.5.0", "md-5 0.11.0", "pretty_assertions", "sha1 0.11.0", @@ -9018,7 +9018,7 @@ dependencies = [ [[package]] name = "rustfs-common" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "chrono", "metrics", @@ -9033,7 +9033,7 @@ dependencies = [ [[package]] name = "rustfs-concurrency" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "insta", "rustfs-io-core", @@ -9045,7 +9045,7 @@ dependencies = [ [[package]] name = "rustfs-config" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "const-str", "serde", @@ -9054,7 +9054,7 @@ dependencies = [ [[package]] name = "rustfs-credentials" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "base64-simd", "hmac 0.13.0", @@ -9067,7 +9067,7 @@ dependencies = [ [[package]] name = "rustfs-crypto" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "aes-gcm", "argon2", @@ -9087,7 +9087,7 @@ dependencies = [ [[package]] name = "rustfs-data-usage" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "rmp-serde", @@ -9097,7 +9097,7 @@ dependencies = [ [[package]] name = "rustfs-ecstore" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "aes-gcm", "arc-swap", @@ -9129,7 +9129,7 @@ dependencies = [ "hex-simd", "hmac 0.13.0", "hotpath", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -9235,7 +9235,7 @@ dependencies = [ [[package]] name = "rustfs-extension-schema" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "serde", "serde_json", @@ -9244,7 +9244,7 @@ dependencies = [ [[package]] name = "rustfs-filemeta" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "arc-swap", "byteorder", @@ -9270,12 +9270,12 @@ dependencies = [ [[package]] name = "rustfs-heal" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "base64 0.23.0", "futures", - "http 1.4.2", + "http 1.5.0", "metrics", "rustfs-common", "rustfs-concurrency", @@ -9300,13 +9300,13 @@ dependencies = [ [[package]] name = "rustfs-iam" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "arc-swap", "async-trait", "base64-simd", "futures", - "http 1.4.2", + "http 1.5.0", "jsonwebtoken 11.0.0", "moka", "openidconnect", @@ -9337,7 +9337,7 @@ dependencies = [ [[package]] name = "rustfs-io-core" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "bytes", "memmap2", @@ -9349,7 +9349,7 @@ dependencies = [ [[package]] name = "rustfs-io-metrics" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "criterion", "metrics", @@ -9412,11 +9412,11 @@ dependencies = [ [[package]] name = "rustfs-keystone" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "bytes", "futures", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -9438,7 +9438,7 @@ dependencies = [ [[package]] name = "rustfs-kms" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "aes-gcm", "arc-swap", @@ -9472,7 +9472,7 @@ dependencies = [ [[package]] name = "rustfs-lifecycle" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "metrics", @@ -9494,7 +9494,7 @@ dependencies = [ [[package]] name = "rustfs-lock" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "crossbeam-queue", @@ -9516,7 +9516,7 @@ dependencies = [ [[package]] name = "rustfs-log-analyzer" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "chrono", "flate2", @@ -9534,7 +9534,7 @@ dependencies = [ [[package]] name = "rustfs-madmin" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "chrono", "humantime", @@ -9548,7 +9548,7 @@ dependencies = [ [[package]] name = "rustfs-notify" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "arc-swap", "async-trait", @@ -9582,7 +9582,7 @@ dependencies = [ [[package]] name = "rustfs-object-capacity" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "criterion", "futures", @@ -9600,7 +9600,7 @@ dependencies = [ [[package]] name = "rustfs-object-data-cache" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "bytes", "criterion", @@ -9616,7 +9616,7 @@ dependencies = [ [[package]] name = "rustfs-obs" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "chrono", "crossbeam-channel", @@ -9668,7 +9668,7 @@ dependencies = [ [[package]] name = "rustfs-policy" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "base64-simd", @@ -9697,7 +9697,7 @@ dependencies = [ [[package]] name = "rustfs-protocols" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "astral-tokio-tar", "async-compression", @@ -9710,7 +9710,7 @@ dependencies = [ "futures-util", "hex", "hmac 0.13.0", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-util", @@ -9759,7 +9759,7 @@ dependencies = [ [[package]] name = "rustfs-protos" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "flatbuffers", "prost 0.14.4", @@ -9782,7 +9782,7 @@ dependencies = [ [[package]] name = "rustfs-replication" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "byteorder", "bytes", @@ -9799,7 +9799,7 @@ dependencies = [ [[package]] name = "rustfs-rio" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "aes-gcm", "arc-swap", @@ -9811,7 +9811,7 @@ dependencies = [ "futures", "hex-simd", "hotpath", - "http 1.4.2", + "http 1.5.0", "http-body-util", "md-5 0.11.0", "pin-project-lite", @@ -9837,7 +9837,7 @@ dependencies = [ [[package]] name = "rustfs-rio-v2" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "aes-gcm", "bytes", @@ -9859,14 +9859,14 @@ dependencies = [ [[package]] name = "rustfs-s3-ops" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "rustfs-s3-types", ] [[package]] name = "rustfs-s3-types" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "serde", "serde_json", @@ -9874,7 +9874,7 @@ dependencies = [ [[package]] name = "rustfs-s3select-api" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "bytes", @@ -9882,7 +9882,7 @@ dependencies = [ "datafusion", "futures", "futures-core", - "http 1.4.2", + "http 1.5.0", "metrics", "parking_lot", "rustfs-common", @@ -9903,7 +9903,7 @@ dependencies = [ [[package]] name = "rustfs-s3select-query" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-recursion", "async-trait", @@ -9919,7 +9919,7 @@ dependencies = [ [[package]] name = "rustfs-scanner" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "bytes", @@ -9927,7 +9927,7 @@ dependencies = [ "futures", "hex-simd", "hmac 0.13.0", - "http 1.4.2", + "http 1.5.0", "metrics", "rand 0.10.2", "rmp-serde", @@ -9957,18 +9957,18 @@ dependencies = [ [[package]] name = "rustfs-security-governance" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "thiserror 2.0.19", ] [[package]] name = "rustfs-signer" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "base64-simd", "bytes", - "http 1.4.2", + "http 1.5.0", "hyper", "rustfs-utils", "s3s", @@ -9981,7 +9981,7 @@ dependencies = [ [[package]] name = "rustfs-storage-api" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "insta", @@ -9995,7 +9995,7 @@ dependencies = [ [[package]] name = "rustfs-targets" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "arc-swap", "async-nats", @@ -10048,7 +10048,7 @@ dependencies = [ [[package]] name = "rustfs-test-utils" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "rustfs-data-usage", "rustfs-ecstore", @@ -10063,7 +10063,7 @@ dependencies = [ [[package]] name = "rustfs-tls-runtime" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "arc-swap", "metrics", @@ -10083,11 +10083,11 @@ dependencies = [ [[package]] name = "rustfs-trusted-proxies" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "async-trait", "axum", - "http 1.4.2", + "http 1.5.0", "ipnetwork", "metrics", "moka", @@ -10119,7 +10119,7 @@ dependencies = [ [[package]] name = "rustfs-utils" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "base64-simd", "blake2", @@ -10133,7 +10133,7 @@ dependencies = [ "hex-simd", "highway", "hmac 0.13.0", - "http 1.4.2", + "http 1.5.0", "hyper", "local-ip-address", "lz4", @@ -10160,7 +10160,7 @@ dependencies = [ [[package]] name = "rustfs-zip" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" dependencies = [ "astral-tokio-tar", "async-compression", @@ -10190,7 +10190,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "http 1.4.2", + "http 1.5.0", "reqwest", "rustify_derive", "serde", @@ -10230,9 +10230,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", @@ -10370,7 +10370,7 @@ dependencies = [ "futures", "hex-simd", "hmac 0.13.0", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "httparse", @@ -11498,7 +11498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -11734,13 +11734,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -11844,7 +11844,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "httparse", "rand 0.8.7", "rustls-pki-types", @@ -11896,7 +11896,7 @@ dependencies = [ "bytes", "flate2", "h2", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "hyper", @@ -11983,7 +11983,7 @@ dependencies = [ "bitflags 2.13.1", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "pin-project-lite", "tower", @@ -12003,7 +12003,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.1.0", "http-body-util", "percent-encoding", @@ -12176,7 +12176,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http 1.4.2", + "http 1.5.0", "httparse", "log", "rand 0.9.5", @@ -12363,7 +12363,7 @@ checksum = "30ffcc0e81025065dda612ec1e26a3d81bb16ef3062354873d17a35965d68522" dependencies = [ "async-trait", "derive_builder", - "http 1.4.2", + "http 1.5.0", "reqwest", "rustify", "rustify_derive", diff --git a/Cargo.toml b/Cargo.toml index 3898c1ff9..1be11daf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ edition = "2024" license = "Apache-2.0" repository = "https://github.com/rustfs/rustfs" rust-version = "1.97.1" -version = "1.0.0-beta.11" +version = "1.0.0-beta.12" homepage = "https://rustfs.com" description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. " keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"] @@ -86,58 +86,58 @@ redundant_clone = "warn" [workspace.dependencies] # RustFS Internal Crates -rustfs = { path = "./rustfs", version = "1.0.0-beta.11" } -rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.11" } -rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.11" } -rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.11" } -rustfs-common = { path = "crates/common", version = "1.0.0-beta.11" } -rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.11" } -rustfs-config = { path = "./crates/config", version = "1.0.0-beta.11" } -rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.11" } -rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.11" } -rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.11" } -rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.11" } -rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.11" } -rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.11" } -rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.11" } -rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.11" } -rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.11" } -rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.11" } -rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.11" } -rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.11" } -rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.11" } -rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.11" } -rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.11" } -rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.11" } -rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.11" } -rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.11" } -rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.11" } -rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.11" } -rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.11" } -rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.11" } -rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.11" } -rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.11" } -rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.11" } -rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.11" } -rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.11" } -rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.11" } -rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.11" } -rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.11" } -rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.11" } -rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.11" } -rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.11" } -rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.11" } -rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.11" } -rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.11" } -rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.11" } -rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.11" } -rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.11" } +rustfs = { path = "./rustfs", version = "1.0.0-beta.12" } +rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" } +rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" } +rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" } +rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" } +rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" } +rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" } +rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" } +rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" } +rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" } +rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" } +rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" } +rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" } +rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" } +rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" } +rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" } +rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" } +rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" } +rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" } +rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" } +rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" } +rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" } +rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" } +rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" } +rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" } +rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" } +rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" } +rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" } +rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" } +rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" } +rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" } +rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" } +rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" } +rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" } +rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" } +rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" } +rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" } +rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" } +rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" } +rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" } +rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" } +rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" } +rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" } +rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" } +rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" } +rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" } # Async Runtime and Networking async-channel = "2.5.0" async_zip = { default-features = false, version = "0.0.18" } mysql_async = { default-features = false, version = "0.37" } -async-compression = { version = "0.4.42" } +async-compression = { version = "0.4.43" } async-recursion = "1.1.1" async-trait = "0.1.91" async-nats = { version = "0.50.0", default-features = false } @@ -152,7 +152,7 @@ lapin = { default-features = false, version = "4.10.0" } hyper = { version = "1.11.0" } hyper-rustls = { default-features = false, version = "0.27.9" } hyper-util = { version = "0.1.20" } -http = "1.4.2" +http = "1.5.0" http-body = "1.1.0" http-body-util = "0.1.4" minlz = "1.2.3" @@ -200,7 +200,7 @@ jsonwebtoken = { version = "11.0.0" } openidconnect = { default-features = false, version = "4.0" } pbkdf2 = "0.13.0" rsa = { version = "=0.10.0-rc.18" } -rustls = { default-features = false, version = "0.23.42" } +rustls = { default-features = false, version = "0.23.43" } rustls-native-certs = "0.8" rustls-pki-types = "1.15.1" sha1 = "0.11.0" @@ -283,7 +283,7 @@ reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" } reed-solomon-simd = "3.1.0" regex = { version = "1.13.1" } rumqttc = { package = "rumqttc-next", version = "0.33.3" } -redis = { version = "1.4.1" } +redis = { version = "1.5.0" } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } diff --git a/README.md b/README.md index 59c1a8ca7..718889f9c 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ chown -R 10001:10001 data logs docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest # Using specific version -docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11 +docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12 ``` If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command diff --git a/README_ZH.md b/README_ZH.md index 162eba04a..789af1f51 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -113,7 +113,7 @@ chown -R 10001:10001 data logs docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest # 使用指定版本运行 -docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11 +docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12 ``` 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录： diff --git a/flake.nix b/flake.nix index 226c2f15e..7b153ab37 100644 --- a/flake.nix +++ b/flake.nix @@ -59,7 +59,7 @@ { default = rustPlatform.buildRustPackage { pname = "rustfs"; - version = "1.0.0-beta.11"; + version = "1.0.0-beta.12"; src = ./.; diff --git a/helm/rustfs/Chart.yaml b/helm/rustfs/Chart.yaml index 63b137b12..4685edcb3 100644 --- a/helm/rustfs/Chart.yaml +++ b/helm/rustfs/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rustfs description: RustFS helm chart to deploy RustFS on kubernetes cluster. type: application -version: "0.11.0" -appVersion: "1.0.0-beta.11" +version: "0.12.0" +appVersion: "1.0.0-beta.12" home: https://rustfs.com icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg maintainers: diff --git a/rustfs.spec b/rustfs.spec index fb5028f05..d9fdae840 100644 --- a/rustfs.spec +++ b/rustfs.spec @@ -1,9 +1,9 @@ %global _enable_debug_packages 0 %global _empty_manifest_terminate_build 0 -%global prerelease beta.11 +%global prerelease beta.12 Name: rustfs Version: 1.0.0 -Release: beta.11 +Release: beta.12 Summary: High-performance distributed object storage for MinIO alternative License: Apache-2.0 @@ -58,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown %_bindir/rustfs %changelog +* Thu Jul 30 2026 overtrue +- Update RPM package to RustFS 1.0.0-beta.12 + * Thu Jul 23 2026 overtrue - Update RPM package to RustFS 1.0.0-beta.11 From 67904a6c18ea4ea4e24c3bd3e070a62ddc6ce966 Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 30 Jul 2026 11:14:38 +0800 Subject: [PATCH 06/52] fix(ecstore): start with unresolved Kubernetes peers (#5460) * fix(ecstore): start with unresolved Kubernetes peers * fix(ecstore): infer Kubernetes endpoint identity safely * fix(ecstore): fail closed on unsafe format migration * fix(ecstore): reject poisoned format heal candidates * fix(ecstore): reject unsafe legacy migration outliers * fix(ecstore): resume interrupted format migrations * fix(ecstore): preserve Kubernetes startup compatibility --- Cargo.lock | 1 + Cargo.toml | 1 + crates/config/README.md | 4 + crates/config/src/constants/app.rs | 4 + crates/ecstore/Cargo.toml | 1 + .../src/cluster/rpc/peer_rest_client.rs | 206 ++- crates/ecstore/src/cluster/rpc/remote_disk.rs | 6 +- crates/ecstore/src/core/sets.rs | 219 ++- crates/ecstore/src/layout/disks_layout.rs | 13 +- crates/ecstore/src/layout/endpoint.rs | 21 +- crates/ecstore/src/layout/endpoints.rs | 1221 +++++++++++++++-- crates/ecstore/src/layout/format.rs | 99 +- crates/ecstore/src/runtime/sources.rs | 25 +- .../ecstore/src/services/notification_sys.rs | 13 +- crates/ecstore/src/services/tier/tier.rs | 74 +- crates/ecstore/src/set_disk/mod.rs | 5 +- crates/ecstore/src/set_disk/ops/heal.rs | 67 +- crates/ecstore/src/set_disk/ops/locking.rs | 123 +- crates/ecstore/src/store/heal.rs | 139 +- crates/ecstore/src/store/init.rs | 18 +- crates/ecstore/src/store/init_format.rs | 1209 ++++++++++++++-- crates/ecstore/src/store/mod.rs | 4 +- crates/utils/src/net.rs | 24 +- crates/utils/src/string.rs | 13 +- docs/architecture/compat-cleanup-register.md | 4 + helm/README.md | 82 +- helm/rustfs/templates/_helpers.tpl | 18 +- helm/rustfs/templates/statefulset.yaml | 96 +- helm/rustfs/values.yaml | 30 +- scripts/test_helm_templates.sh | 339 ++++- 30 files changed, 3618 insertions(+), 461 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b927192e..dd420db90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9128,6 +9128,7 @@ dependencies = [ "google-cloud-storage", "hex-simd", "hmac 0.13.0", + "hostname", "hotpath", "http 1.5.0", "http-body 1.1.0", diff --git a/Cargo.toml b/Cargo.toml index 1be11daf9..d8512078f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -256,6 +256,7 @@ hashbrown = { version = "0.17.1" } hex = "0.4.3" hex-simd = "0.8.0" highway = { version = "1.3.0" } +hostname = "0.4.2" ipnetwork = { version = "0.21.1" } lazy_static = "1.5.0" libc = "0.2.189" diff --git a/crates/config/README.md b/crates/config/README.md index 573153824..92141d4e8 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -66,6 +66,10 @@ Current guidance: - `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node. +## Distributed endpoint locality + +- `RUSTFS_LOCAL_ENDPOINT_HOST` identifies this server's host in a distributed `RUSTFS_VOLUMES` topology without resolving every peer during startup. Set it to exactly one host, without a scheme, port, or path. It is accepted only for orchestrated URL topologies and must match at least one endpoint on the RustFS server port; invalid or unmatched values fail startup. Leave it unset to retain DNS-based locality discovery. + ## Scanner environment aliases - `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`) diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 405bb7345..9a6a06471 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -131,6 +131,10 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS"; /// Environment variable for server volumes. pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES"; +/// Environment variable identifying this server's host in distributed endpoint +/// lists without relying on DNS locality discovery. +pub const ENV_LOCAL_ENDPOINT_HOST: &str = "RUSTFS_LOCAL_ENDPOINT_HOST"; + /// Environment variable to explicitly bypass local physical disk independence checks. pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK"; diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index dbc1d769f..31a5446d1 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -105,6 +105,7 @@ tempfile.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] } hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] } +hostname.workspace = true rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } rustls-pki-types.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] } diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 10d2b4a3e..dc4f0bef6 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -58,7 +58,7 @@ use std::{ collections::HashMap, io::Cursor, sync::{ - Arc, + Arc, Weak, atomic::{AtomicBool, Ordering}, }, time::SystemTime, @@ -350,6 +350,44 @@ impl PeerRestClient { } } + fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result { + let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?; + if !matches!(url.scheme(), "http" | "https") + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || url.path() != "/" + { + return Err(Error::other("peer grid host has an invalid URL shape")); + } + let url_host = url.host().ok_or_else(|| Error::other("peer grid host is missing a host"))?; + let topology_host = match url.port() { + Some(port) => format!("{url_host}:{port}"), + None => url_host.to_string(), + }; + let explicit_port = url.port(); + let name = match url_host { + url::Host::Domain(domain) => domain.to_string(), + url::Host::Ipv4(address) => address.to_string(), + url::Host::Ipv6(address) if explicit_port.is_none() => format!("[{address}]"), + url::Host::Ipv6(address) => address.to_string(), + }; + let port = url + .port_or_known_default() + .filter(|port| *port > 0) + .ok_or_else(|| Error::other("peer grid host is missing a valid port"))?; + let host = XHost { + name, + port, + is_port_set: explicit_port.is_some(), + }; + if topology_host != peer_host_port { + return Err(Error::other("peer topology host does not match its grid URL")); + } + Ok(host) + } + fn build_clients_from_slots( slots: Vec<(String, Option, bool)>, ) -> (Vec>, Vec>, Vec) { @@ -363,14 +401,14 @@ impl PeerRestClient { } let client = match grid_host { - Some(grid_host) => match XHost::try_from(peer_host_port.clone()) { + Some(grid_host) => match Self::parse_topology_host(&peer_host_port, &grid_host) { Ok(host) => { let mut client = PeerRestClient::new(host, grid_host); client.topology_member = peer_host_port.clone(); Some(client) } Err(err) => { - warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}"); + warn!(peer = %peer_host_port, "peer topology host parse failed while constructing peer client: {err:?}"); None } }, @@ -519,9 +557,8 @@ impl PeerRestClient { } let grid_host = self.grid_host.clone(); - let offline = Arc::clone(&self.offline); - let recovery_running = Arc::clone(&self.recovery_running); - let span = Self::recovery_monitor_span(&grid_host); + let offline = Arc::downgrade(&self.offline); + let recovery_running = Arc::downgrade(&self.recovery_running); // The offline flag and its recovery are the silent half of // rustfs/backlog#888: log the monitor's start and its success so an // "offline then back" episode leaves a trace on the observing node. @@ -530,13 +567,34 @@ impl PeerRestClient { grid_host = %self.grid_host, "peer RPC connection marked offline after a network-like failure; starting background recovery monitor" ); + drop(Self::spawn_recovery_monitor(grid_host, offline, recovery_running)); + } + + fn spawn_recovery_monitor( + grid_host: String, + offline: Weak, + recovery_running: Weak, + ) -> tokio::task::JoinHandle<()> { + let span = Self::recovery_monitor_span(&grid_host); super::spawn_background_monitor(span, async move { let mut delay = get_drive_active_check_interval(); let connect_timeout = get_drive_active_check_timeout(); for attempt in 1..=PEER_REST_RECOVERY_MAX_ATTEMPTS { + if offline.strong_count() == 0 || recovery_running.strong_count() == 0 { + return; + } tokio::time::sleep(delay).await; + if offline.strong_count() == 0 || recovery_running.strong_count() == 0 { + return; + } if Self::perform_connectivity_check(&grid_host, connect_timeout).await.is_ok() { + let Some(offline) = offline.upgrade() else { + return; + }; + let Some(recovery_running) = recovery_running.upgrade() else { + return; + }; offline.store(false, Ordering::Release); recovery_running.store(false, Ordering::Release); info!( @@ -556,8 +614,10 @@ impl PeerRestClient { attempts = PEER_REST_RECOVERY_MAX_ATTEMPTS, "peer recovery monitor reached max attempts; will retry on next request" ); - recovery_running.store(false, Ordering::Release); - }); + if let Some(recovery_running) = recovery_running.upgrade() { + recovery_running.store(false, Ordering::Release); + } + }) } #[cfg(test)] @@ -1807,9 +1867,13 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO mod tests { use super::*; use crate::config::com::STORAGE_CLASS_SUB_SYS; + use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType}; + use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE}; use serde_json::Value; + use serial_test::serial; use std::io::{self, Write}; use std::sync::{Arc, Mutex}; + use temp_env::async_with_vars; use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt}; #[test] @@ -1927,30 +1991,115 @@ mod tests { fn build_clients_from_slots_preserves_missing_remote_topology_slots() { let slots = vec![ ("127.0.0.1:9000".to_string(), None, true), - ("127.0.0.1:9001".to_string(), Some("http://127.0.0.1:9001".to_string()), false), + ( + "rustfs-1.invalid:9001".to_string(), + Some("http://rustfs-1.invalid:9001".to_string()), + false, + ), + ("rustfs-2.invalid".to_string(), Some("http://rustfs-2.invalid".to_string()), false), ("127.0.0.1:notaport".to_string(), Some("http://127.0.0.1:notaport".to_string()), false), ("127.0.0.1:9003".to_string(), None, false), ]; let (remote, all, remote_topology_hosts) = PeerRestClient::build_clients_from_slots(slots); - assert_eq!(remote.len(), 3, "local node is excluded but remote slots are not compacted away"); - assert_eq!(all.len(), 4, "all slots preserve the sorted cluster topology shape"); + assert_eq!(remote.len(), 4, "local node is excluded but remote slots are not compacted away"); + assert_eq!(all.len(), 5, "all slots preserve the sorted cluster topology shape"); assert_eq!( remote_topology_hosts, vec![ - "127.0.0.1:9001".to_string(), + "rustfs-1.invalid:9001".to_string(), + "rustfs-2.invalid".to_string(), "127.0.0.1:notaport".to_string(), "127.0.0.1:9003".to_string() ] ); - assert!(remote[0].is_some(), "valid remote peer should get a client"); - assert!(remote[1].is_none(), "unparseable remote peer should remain observable as a missing slot"); - assert!(remote[2].is_none(), "missing grid host should remain observable as a missing slot"); + let unresolved = remote[0] + .as_ref() + .expect("temporarily unresolved remote peer should retain a client"); + assert_eq!(unresolved.host.to_string(), "rustfs-1.invalid:9001"); + let default_port = remote[1] + .as_ref() + .expect("temporarily unresolved scheme-default remote peer should retain a client"); + assert_eq!(default_port.host.to_string(), "rustfs-2.invalid"); + assert_eq!(default_port.host.port, 80); + assert!(!default_port.host.is_port_set); + assert!(remote[2].is_none(), "unparseable remote peer should remain observable as a missing slot"); + assert!(remote[3].is_none(), "missing grid host should remain observable as a missing slot"); assert!(all[0].is_none(), "local node is represented by the local server_info row"); assert!(all[1].is_some()); - assert!(all[2].is_none()); + assert!(all[2].is_some()); assert!(all[3].is_none()); + assert!(all[4].is_none()); + } + + #[test] + fn topology_host_parser_preserves_names_and_bracketed_ipv6() { + let domain = PeerRestClient::parse_topology_host("rustfs-1.invalid", "https://rustfs-1.invalid") + .expect("unresolved HTTPS topology host should parse without DNS"); + assert_eq!(domain.to_string(), "rustfs-1.invalid"); + assert_eq!(domain.port, 443); + assert!(!domain.is_port_set); + + let ipv6 = PeerRestClient::parse_topology_host("[2001:db8::1]:9000", "http://[2001:db8::1]:9000") + .expect("bracketed IPv6 topology host should parse without changing its identity"); + assert_eq!(ipv6.to_string(), "[2001:db8::1]:9000"); + + let default_port_ipv6 = PeerRestClient::parse_topology_host("[2001:db8::2]", "http://[2001:db8::2]") + .expect("scheme-default IPv6 topology host should parse without DNS"); + assert_eq!(default_port_ipv6.to_string(), "[2001:db8::2]"); + assert_eq!(default_port_ipv6.port, 80); + assert!(!default_port_ipv6.is_port_set); + + assert!(PeerRestClient::parse_topology_host("peer.invalid:0", "http://peer.invalid:0").is_err()); + assert!(PeerRestClient::parse_topology_host("peer-a.invalid:9000", "http://peer-b.invalid:9000").is_err()); + assert!(PeerRestClient::parse_topology_host("peer.invalid:9000", "http://peer.invalid:9000/unexpected").is_err()); + } + + #[tokio::test] + #[serial] + async fn unresolved_default_port_endpoint_topology_retains_all_peer_clients() { + let volumes = (0..4) + .map(|index| format!("http://rustfs-{index}.invalid:80/data{index}")) + .collect::>(); + let layout = DisksLayout::from_volumes(&volumes).expect("distributed default-port topology should parse"); + + async_with_vars( + [ + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("orchestrated")), + (ENV_LOCAL_ENDPOINT_HOST, Some("rustfs-0.invalid")), + (ENV_KUBERNETES_SERVICE_HOST, None), + ], + async { + let (server_pools, setup_type) = EndpointServerPools::create_server_endpoints("0.0.0.0:80", &layout) + .await + .expect("explicit local identity should avoid peer DNS during endpoint construction"); + assert_eq!(setup_type, SetupType::DistErasure); + + let (remote, all, remote_topology_hosts) = + PeerRestClient::build_clients_from_slots(server_pools.peer_grid_host_slots_sorted()); + assert_eq!(remote.len(), 3); + assert!( + remote.iter().all(Option::is_some), + "unresolved remote peers must retain reconnectable clients" + ); + assert_eq!(all.len(), 4); + assert_eq!(all.iter().filter(|client| client.is_none()).count(), 1); + assert_eq!(remote_topology_hosts.len(), 3); + assert!( + remote_topology_hosts.iter().all(|host| !host.contains(':')), + "scheme-default ports must preserve the legacy topology identity" + ); + assert!( + remote + .iter() + .flatten() + .all(|client| client.host.port == 80 && !client.host.is_port_set), + "scheme-default peers must retain the effective dial port" + ); + }, + ) + .await; } #[test] @@ -2740,6 +2889,31 @@ mod tests { assert!(!client.offline.load(Ordering::Acquire)); } + #[tokio::test(start_paused = true)] + async fn dropped_peer_client_releases_and_stops_its_recovery_monitor() { + let client = test_peer_client(); + client.offline.store(true, Ordering::Release); + client.recovery_running.store(true, Ordering::Release); + let offline = Arc::downgrade(&client.offline); + let recovery_running = Arc::downgrade(&client.recovery_running); + let handle = PeerRestClient::spawn_recovery_monitor(client.grid_host.clone(), offline.clone(), recovery_running.clone()); + let started = tokio::time::Instant::now(); + + drop(client); + + assert!(offline.upgrade().is_none(), "detached recovery must not retain offline state"); + assert!( + recovery_running.upgrade().is_none(), + "detached recovery must not retain its running state" + ); + handle.await.expect("recovery monitor should not panic"); + assert_eq!( + tokio::time::Instant::now(), + started, + "recovery monitor should stop before advancing to its first delayed probe" + ); + } + #[tokio::test] async fn peer_rest_client_finalize_result_keeps_online_for_app_errors_mentioning_unavailable() { // Regression: application error text containing "unavailable" (a diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index d5090d54f..0d8d507e4 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -4814,9 +4814,11 @@ mod tests { async fn test_remote_disk_endpoints_with_different_schemes() { let test_cases = vec![ ("http://server:9000", "server:9000"), - ("https://secure-server:443", "secure-server"), // Default HTTPS port is omitted + ("http://plain-server:80", "plain-server"), + ("http://plain-server", "plain-server"), + ("https://secure-server:443", "secure-server"), ("http://192.168.1.100:8080", "192.168.1.100:8080"), - ("https://secure-server", "secure-server"), // No port specified + ("https://secure-server", "secure-server"), ]; for (url_str, expected_hostname) in test_cases { diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index 4d8d9f4e7..7b4ba2a60 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -37,7 +37,9 @@ use crate::{ runtime::instance::{InstanceContext, bootstrap_ctx}, runtime::sources as runtime_sources, set_disk::{PreparedGetObjectMetadata, SetDisks}, - store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, + store::init_format::{ + check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum, + }, }; use futures::{ future::join_all, @@ -947,7 +949,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { #[tracing::instrument(skip(self))] async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { - let (disks, _) = init_storage_disks_with_errors( + let (disks, init_errs) = init_storage_disks_with_errors( &self.endpoints.endpoints, &DiskOption { cleanup: false, @@ -955,15 +957,36 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { }, ) .await; - let (formats, errs) = load_format_erasure_all(&disks, true).await; + let (formats, mut errs) = load_format_erasure_all(&disks, true).await; + for (err, init_err) in errs.iter_mut().zip(init_errs) { + if init_err.is_some() { + *err = init_err; + } + } + if errs.iter().any(|err| { + matches!( + err, + Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend) + ) + }) { + return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))); + } if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) { info!("failed to check formats erasure values: {}", err); return Ok((HealResultItem::default(), Some(err))); } - let ref_format = match get_format_erasure_in_quorum(&formats) { - Ok(format) => format, + let (ref_format, quorum_members) = match select_format_erasure_in_quorum(&formats, 0) { + Ok((format, members)) if format.shared_identity() == self.format.shared_identity() => (format, members), + Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))), Err(err) => return Ok((HealResultItem::default(), Some(err))), }; + if formats + .iter() + .zip(quorum_members) + .any(|(format, member)| format.is_some() && !member) + { + return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))); + } let mut res = HealResultItem { heal_item_type: HealItemType::Metadata.to_string(), detail: "disk-format".to_string(), @@ -985,11 +1008,6 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { return Ok((res, Some(StorageError::NoHealRequired))); } - // if !self.format.eq(&ref_format) { - // info!("format ({:?}) not eq ref_format ({:?})", self.format, ref_format); - // return Ok((res, Some(Error::new(DiskError::CorruptedFormat)))); - // } - let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs); if !dry_run { let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count]; @@ -1298,7 +1316,7 @@ mod tests { assert_eq!(result, (Some(3), Some(1), Some(0))); } - async fn multipart_listing_test_sets() -> (Vec, Arc) { + async fn two_set_test_sets() -> (Vec, Arc) { let format = FormatV3::new(2, 2); let mut temp_dirs = Vec::new(); let mut all_endpoints = Vec::new(); @@ -1339,8 +1357,8 @@ mod tests { Arc::new(RwLock::new(disks)), 2, 1, - 0, set_index, + 0, endpoints, format.clone(), vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())], @@ -1373,11 +1391,114 @@ mod tests { (temp_dirs, sets) } + #[tokio::test] + async fn set_format_heal_accepts_quorum_from_a_nonzero_set() { + let (_temp_dirs, sets) = two_set_test_sets().await; + + let (result, err) = sets.disk_set[1] + .heal_format(false) + .await + .expect("the second erasure set should load its own format quorum"); + + assert!(matches!(err, Some(StorageError::NoHealRequired)), "unexpected heal result: {err:?}"); + assert_eq!(result.disk_count, 2); + assert_eq!(result.set_count, 1); + } + + #[tokio::test] + async fn format_heal_rejects_foreign_majorities_at_set_and_pool_scopes() { + let (_temp_dirs, _canonical_format, sets) = setup_heal_format_sets(2, true).await; + let set_disks = set_level_heal_view(&sets).await; + + let (_, set_err) = set_disks + .heal_format(false) + .await + .expect("set format heal should report a typed mismatch"); + assert!( + matches!(set_err, Some(StorageError::CorruptedFormat)), + "foreign set majority must not replace the cached format: {set_err:?}" + ); + + let (_, pool_err) = sets + .heal_format(false) + .await + .expect("pool format heal should report a typed mismatch"); + assert!( + matches!(pool_err, Some(StorageError::CorruptedFormat)), + "foreign pool majority must not replace the cached format: {pool_err:?}" + ); + } + + #[tokio::test] + async fn pool_format_heal_rejects_a_wrong_slot_minority() { + let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await; + let mut poisoned_format = canonical_format.clone(); + poisoned_format.erasure.this = canonical_format.erasure.sets[0][0]; + replace_heal_test_format(&sets, 2, &poisoned_format).await; + let probe_err = new_disk( + &sets.endpoints.endpoints.as_ref()[2], + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect_err("a wrong-slot local format must fail disk initialization"); + assert_eq!(probe_err, DiskError::InconsistentDisk); + + let (_, pool_err) = sets + .heal_format(false) + .await + .expect("pool format heal should report a typed slot mismatch"); + assert!( + matches!(pool_err, Some(StorageError::CorruptedFormat)), + "a wrong-slot minority must not be reported as no-heal-required: {pool_err:?}" + ); + assert_eq!( + read_heal_test_format(&sets, 2).await, + poisoned_format, + "format heal must not overwrite a wrong-slot disk" + ); + } + + #[tokio::test] + async fn format_heal_rejects_a_foreign_minority_at_set_and_pool_scopes() { + let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await; + let mut poisoned_format = canonical_format.clone(); + poisoned_format.id = Uuid::new_v4(); + poisoned_format.erasure.this = poisoned_format.erasure.sets[0][2]; + replace_heal_test_format(&sets, 2, &poisoned_format).await; + let set_disks = set_level_heal_view(&sets).await; + + let (_, set_err) = set_disks + .heal_format(false) + .await + .expect("set format heal should report a typed identity mismatch"); + assert!( + matches!(set_err, Some(StorageError::CorruptedFormat)), + "a foreign minority must not be reported as no-heal-required: {set_err:?}" + ); + + let (_, pool_err) = sets + .heal_format(false) + .await + .expect("pool format heal should report a typed identity mismatch"); + assert!( + matches!(pool_err, Some(StorageError::CorruptedFormat)), + "a foreign minority must not be reported as no-heal-required: {pool_err:?}" + ); + assert_eq!( + read_heal_test_format(&sets, 2).await, + poisoned_format, + "format heal must not overwrite a foreign disk" + ); + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() { let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await; - let (_temp_dirs, sets) = multipart_listing_test_sets().await; + let (_temp_dirs, sets) = two_set_test_sets().await; let bucket = format!("multipart-list-{}", Uuid::new_v4().simple()); sets.make_bucket(&bucket, &MakeBucketOptions::default()) .await @@ -1616,11 +1737,15 @@ mod tests { // formatting the first `num_formatted` of them against a shared reference // format and leaving the rest unformatted. Returns the live TempDir handles // (must be kept alive), the reference format, and the assembled `Sets`. - // `disk_set` is intentionally empty: these tests only drive `heal_format` - // with `dry_run == true`, which never touches `disk_set`. - async fn setup_heal_format_sets(num_formatted: usize) -> (Vec, FormatV3, Sets) { + // `disk_set` is intentionally empty: these tests only exercise paths that + // return before pool-level healing delegates into a set. + async fn setup_heal_format_sets(num_formatted: usize, foreign_identity: bool) -> (Vec, FormatV3, Sets) { const SET_DRIVE_COUNT: usize = 3; let ref_format = FormatV3::new(1, SET_DRIVE_COUNT); + let mut stored_format = ref_format.clone(); + if foreign_identity { + stored_format.id = Uuid::new_v4(); + } let mut dirs = Vec::with_capacity(SET_DRIVE_COUNT); let mut endpoints = Vec::with_capacity(SET_DRIVE_COUNT); @@ -1645,8 +1770,8 @@ mod tests { ) .await .expect("disk should be created"); - let mut disk_format = ref_format.clone(); - disk_format.erasure.this = ref_format.erasure.sets[0][i]; + let mut disk_format = stored_format.clone(); + disk_format.erasure.this = stored_format.erasure.sets[0][i]; save_format_file(&Some(disk), &Some(disk_format)) .await .expect("format should be saved"); @@ -1677,6 +1802,60 @@ mod tests { (dirs, ref_format, sets) } + async fn set_level_heal_view(sets: &Sets) -> Arc { + let endpoints = sets.endpoints.endpoints.as_ref().clone(); + let mut disks = Vec::with_capacity(endpoints.len()); + for endpoint in &endpoints { + disks.push(Some( + new_disk( + endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("fresh set-level disk handle should open"), + )); + } + + SetDisks::new( + "test-owner".to_string(), + Arc::new(RwLock::new(disks)), + endpoints.len(), + 1, + 0, + 0, + endpoints, + sets.format.clone(), + Vec::new(), + ) + .await + } + + async fn replace_heal_test_format(sets: &Sets, disk_index: usize, format: &FormatV3) { + let disk = new_disk( + &sets.endpoints.endpoints.as_ref()[disk_index], + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("heal test disk should open"); + save_format_file(&Some(disk.clone()), &Some(format.clone())) + .await + .expect("poisoned test format should be written"); + } + + async fn read_heal_test_format(sets: &Sets, disk_index: usize) -> FormatV3 { + let path = std::path::Path::new(&sets.endpoints.endpoints.as_ref()[disk_index].get_file_path()) + .join(crate::disk::RUSTFS_META_BUCKET) + .join(crate::disk::FORMAT_CONFIG_FILE); + let data = tokio::fs::read(path).await.expect("test format should be readable"); + FormatV3::try_from(data.as_slice()).expect("test format should parse") + } + // Regression for #956 (NoHealRequired path): with every disk already // formatted, `heal_format` reports exactly one drive record per disk // (N = set_count * set_drive_count), each carrying a real endpoint. Before @@ -1685,7 +1864,7 @@ mod tests { #[tokio::test] #[serial] async fn heal_format_no_heal_required_reports_one_record_per_disk() { - let (_dirs, _ref_format, sets) = setup_heal_format_sets(3).await; + let (_dirs, _ref_format, sets) = setup_heal_format_sets(3, false).await; let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed"); // All disks formatted -> NoHealRequired early return, still returns `res`. @@ -1715,7 +1894,7 @@ mod tests { #[serial] async fn heal_format_heal_path_reports_one_record_per_disk_aligned() { // Disks 0 and 1 formatted (quorum), disk 2 unformatted. - let (_dirs, _ref_format, sets) = setup_heal_format_sets(2).await; + let (_dirs, _ref_format, sets) = setup_heal_format_sets(2, false).await; let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed"); // Unformatted disk present -> heal path, not NoHealRequired. diff --git a/crates/ecstore/src/layout/disks_layout.rs b/crates/ecstore/src/layout/disks_layout.rs index 06a74494c..f880bfeac 100644 --- a/crates/ecstore/src/layout/disks_layout.rs +++ b/crates/ecstore/src/layout/disks_layout.rs @@ -203,7 +203,7 @@ fn get_all_sets>(set_drive_count: usize, is_ellipses: bool, args: for args in set_args.iter() { for arg in args { if unique_args.contains(arg) { - return Err(Error::other(format!("Input args {arg} has duplicate ellipses"))); + return Err(Error::other("input arguments contain a duplicate endpoint after ellipsis expansion")); } unique_args.insert(arg); } @@ -924,4 +924,15 @@ mod test { } } } + + #[test] + fn layout_errors_do_not_echo_url_credentials() { + for volumes in [ + vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"], + vec!["http://:ellipsis...secret@server/path"], + ] { + let err = DisksLayout::from_volumes(&volumes).unwrap_err(); + assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}"); + } + } } diff --git a/crates/ecstore/src/layout/endpoint.rs b/crates/ecstore/src/layout/endpoint.rs index 6a8e86e3e..5e9ca0c58 100644 --- a/crates/ecstore/src/layout/endpoint.rs +++ b/crates/ecstore/src/layout/endpoint.rs @@ -88,6 +88,7 @@ impl TryFrom<&str> for Endpoint { // - All field should be empty except Host and Path. if !((url.scheme() == "http" || url.scheme() == "https") && url.username().is_empty() + && url.password().is_none() && url.fragment().is_none() && url.query().is_none()) { @@ -366,6 +367,12 @@ mod test { expected_type: None, expected_err: Some(Error::other("invalid URL endpoint format")), }, + TestCase { + arg: "http://:topsecret@server/path", + expected_endpoint: None, + expected_type: None, + expected_err: Some(Error::other("invalid URL endpoint format")), + }, TestCase { arg: "http://:/path", expected_endpoint: None, @@ -505,8 +512,18 @@ mod test { let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap(); assert_eq!(endpoint.host_port(), "example.com:9000"); - let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap(); - assert_eq!(endpoint_no_port.host_port(), "example.com"); + for endpoint in [ + Endpoint::try_from("http://example.com/path").unwrap(), + Endpoint::try_from("http://example.com:80/path").unwrap(), + ] { + assert_eq!(endpoint.host_port(), "example.com"); + } + for endpoint in [ + Endpoint::try_from("https://example.com/path").unwrap(), + Endpoint::try_from("https://example.com:443/path").unwrap(), + ] { + assert_eq!(endpoint.host_port(), "example.com"); + } let file_endpoint = Endpoint::try_from("/tmp/data").unwrap(); assert_eq!(file_endpoint.host_port(), ""); diff --git a/crates/ecstore/src/layout/endpoints.rs b/crates/ecstore/src/layout/endpoints.rs index b8e380270..1962ea77c 100644 --- a/crates/ecstore/src/layout/endpoints.rs +++ b/crates/ecstore/src/layout/endpoints.rs @@ -12,17 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - layout::{ - disks_layout::DisksLayout, - endpoint::{Endpoint, EndpointType}, - }, - runtime::sources as runtime_sources, +use crate::layout::{ + disks_layout::DisksLayout, + endpoint::{Endpoint, EndpointType}, }; use rustfs_config::{ DEFAULT_STARTUP_TOPOLOGY_RETRY_MAX_DELAY_SECS, DEFAULT_STARTUP_TOPOLOGY_WAIT_TIMEOUT_SECS, DEFAULT_UNSAFE_BYPASS_DISK_CHECK, - ENV_KUBERNETES_SERVICE_HOST, ENV_MINIO_CI, ENV_STARTUP_TOPOLOGY_RETRY_MAX_DELAY, ENV_STARTUP_TOPOLOGY_WAIT_MODE, - ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, ENV_UNSAFE_BYPASS_DISK_CHECK, + ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_MINIO_CI, ENV_STARTUP_TOPOLOGY_RETRY_MAX_DELAY, + ENV_STARTUP_TOPOLOGY_WAIT_MODE, ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, ENV_UNSAFE_BYPASS_DISK_CHECK, }; use rustfs_utils::{XHost, check_local_server_addr, get_env_opt_str, get_host_ip, is_local_host}; use std::{ @@ -132,10 +129,10 @@ impl> TryFrom<&[T]> for Endpoints { // Loop through args and adds to endpoint list. for (i, arg) in args.iter().enumerate() { - let endpoint = match Endpoint::try_from(arg.as_ref()) { - Ok(ep) => ep, - Err(e) => return Err(Error::other(format!("'{}': {}", arg.as_ref(), e))), - }; + let endpoint = Endpoint::try_from(arg.as_ref()).map_err(|err| { + let err = std::io::Error::from(err); + Error::new(err.kind(), format!("{err} (endpoint argument #{})", i + 1)) + })?; // All endpoints have to be same type and scheme if applicable. if i == 0 { @@ -213,17 +210,23 @@ impl PoolEndpointList { /// creates a list of endpoints per pool, resolves their relevant /// hostnames and discovers those are local or remote. async fn create_pool_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result { - Self::create_pool_endpoints_with(server_addr, disks_layout, None).await + Self::create_pool_endpoints_with(server_addr, disks_layout, None, None).await } /// Same as [`create_pool_endpoints`] but lets tests inject an explicit - /// startup topology convergence policy instead of resolving it from the - /// environment. + /// startup topology convergence policy and local endpoint host instead of + /// resolving them from the environment. async fn create_pool_endpoints_with( server_addr: &str, disks_layout: &DisksLayout, policy_override: Option, + local_endpoint_host: Option<&str>, ) -> Result { + let unsupported_explicit_host = || { + Error::other(format!( + "{ENV_LOCAL_ENDPOINT_HOST} is only supported for distributed URL endpoints in orchestrated mode" + )) + }; if disks_layout.is_empty_layout() { return Err(Error::other("invalid number of endpoints")); } @@ -232,6 +235,10 @@ impl PoolEndpointList { // For single arg, return single drive EC setup. if disks_layout.is_single_drive_layout() { + if local_endpoint_host.is_some() { + return Err(unsupported_explicit_host()); + } + let mut endpoint = Endpoint::try_from(disks_layout.get_single_drive_layout())?; endpoint.update_is_local(server_addr.port())?; @@ -293,7 +300,18 @@ impl PoolEndpointList { .first() .and_then(|eps| eps.as_ref().first()) .is_some_and(|ep| ep.get_type() == EndpointType::Url); - let policy = policy_override.unwrap_or_else(|| StartupTopologyPolicy::resolve(distributed)); + let all_distributed = pool_endpoint_list + .inner + .iter() + .flat_map(|endpoints| endpoints.as_ref()) + .all(|endpoint| endpoint.get_type() == EndpointType::Url); + let policy = match policy_override { + Some(policy) => policy, + None => StartupTopologyPolicy::resolve(distributed, local_endpoint_host.is_some())?, + }; + if local_endpoint_host.is_some() && (!all_distributed || !policy.is_orchestrated()) { + return Err(unsupported_explicit_host()); + } info!( target: "rustfs::ecstore::endpoints", mode = ?policy.mode, @@ -305,14 +323,18 @@ impl PoolEndpointList { let convergence_started = Instant::now(); let dns_retry_deadline = DnsRetryDeadline::new(policy.wait_timeout, policy.retry_max_delay); - pool_endpoint_list - .update_is_local(server_addr.port(), &dns_retry_deadline) - .await?; + if let Some(local_endpoint_host) = local_endpoint_host { + pool_endpoint_list.update_is_local_from_explicit_host(disks_layout, server_addr.port(), local_endpoint_host)?; + } else { + pool_endpoint_list + .update_is_local(server_addr.port(), &dns_retry_deadline) + .await?; - // Collapse divergent local/remote verdicts for the same host:port that - // orchestrated DNS churn can produce during startup, before any check - // relies on the local flag. - normalize_same_host_local_state(pool_endpoint_list.as_mut()); + // Collapse divergent local/remote verdicts for the same host:port that + // orchestrated DNS churn can produce during startup, before any check + // relies on the local flag. + normalize_same_host_local_state(pool_endpoint_list.as_mut()); + } for endpoints in pool_endpoint_list.inner.iter_mut() { // Check whether same path is not used in endpoints of a host on different port. @@ -424,7 +446,9 @@ impl PoolEndpointList { } match ep.url.port() { None => { - let _ = ep.url.set_port(Some(server_addr.port())); + if local_endpoint_host.is_none() { + let _ = ep.url.set_port(Some(server_addr.port())); + } } Some(port) => { // If endpoint is local, but port is different than serverAddrPort, then make it as remote. @@ -498,6 +522,73 @@ impl PoolEndpointList { Ok(()) } + + fn update_is_local_from_explicit_host( + &mut self, + disks_layout: &DisksLayout, + local_port: u16, + raw_local_host: &str, + ) -> Result<()> { + let local_host = parse_explicit_local_endpoint_host(raw_local_host)?; + let mut local_endpoint_found = false; + let mut host_path_ports = HashMap::new(); + debug_assert_eq!(self.inner.len(), disks_layout.pools.len()); + + for (endpoints, pool_layout) in self.inner.iter_mut().zip(&disks_layout.pools) { + debug_assert_eq!(endpoints.as_ref().len(), pool_layout.iter().map(Vec::len).sum::()); + for (endpoint, raw_endpoint) in endpoints.as_mut().iter_mut().zip(pool_layout.iter().flatten()) { + let endpoint_host = match endpoint.url.host().map(|host| host.to_owned()) { + Some(Host::Domain(domain)) => { + let canonical_domain = domain_without_optional_trailing_dot(&domain) + .ok_or_else(|| { + Error::new( + ErrorKind::InvalidInput, + "explicit local endpoint identity requires canonical endpoint hostnames", + ) + })? + .to_string(); + Host::Domain(canonical_domain) + } + Some(host) => host, + None => { + return Err(Error::other( + "explicit local endpoint identity requires every distributed endpoint to have a host", + )); + } + }; + let explicit_port = explicit_url_port(raw_endpoint)?; + let endpoint_port = explicit_port.unwrap_or(local_port); + if explicit_port.is_none() { + let _ = endpoint.url.set_port(Some(local_port)); + } + let path = endpoint.get_file_path(); + endpoint.is_local = endpoint_host == local_host && endpoint_port == local_port; + match host_path_ports.entry((endpoint_host, path)) { + Entry::Occupied(entry) if *entry.get() != endpoint_port => { + return Err(Error::other( + "same path can not be served by different port on the same explicit endpoint host", + )); + } + Entry::Occupied(_) => { + return Err(Error::other("duplicate distributed endpoint after explicit host normalization")); + } + Entry::Vacant(entry) => { + entry.insert(endpoint_port); + } + } + + local_endpoint_found |= endpoint.is_local; + } + } + + if !local_endpoint_found { + return Err(Error::other(format!( + "{ENV_LOCAL_ENDPOINT_HOST} does not match any distributed endpoint on the local server port {local_port}" + ))); + } + + Ok(()) + } } const DNS_RETRY_BASE_DELAY: Duration = Duration::from_millis(500); @@ -672,18 +763,170 @@ fn is_retryable_dns_error(err: &Error) -> bool { return true; } - if matches!(err.raw_os_error(), Some(-3) | Some(-2)) { + if err.raw_os_error().is_some_and(is_retryable_dns_raw_os_error) { return true; } - let message = err.to_string().to_ascii_lowercase(); + let message = err.to_string().trim().to_ascii_lowercase(); // Kubernetes and Docker DNS records can be observed as negative lookups // while headless service records are still propagating during startup. - message.contains("temporary failure in name resolution") - || message.contains("try again") - || message.contains("name or service not known") - || message.contains("no such host") - || message.contains("nodename nor servname provided") + matches!( + message.as_str(), + "temporary failure in name resolution" + | "failed to lookup address information: temporary failure in name resolution" + | "name or service not known" + | "failed to lookup address information: name or service not known" + | "name does not resolve" + | "failed to lookup address information: name does not resolve" + | "no such host" + | "no such host is known." + | "nodename nor servname provided, or not known" + | "failed to lookup address information: nodename nor servname provided, or not known" + ) +} + +fn is_retryable_dns_raw_os_error(code: i32) -> bool { + if matches!(code, -3 | -2 | 11001 | 11002 | 11004) { + return true; + } + + cfg!(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd", + target_os = "dragonfly" + )) && matches!(code, 2 | 8) +} + +fn explicit_url_port(raw: &str) -> Result> { + let invalid = || { + Error::new( + ErrorKind::InvalidInput, + "explicit local endpoint identity requires canonical HTTP(S) endpoint URLs", + ) + }; + if raw.trim() != raw || raw.contains('\\') || raw.bytes().any(|byte| byte.is_ascii_control()) { + return Err(invalid()); + } + + let (scheme, remainder) = raw.split_once("://").ok_or_else(&invalid)?; + if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") { + return Err(invalid()); + } + let authority = remainder + .split('/') + .next() + .filter(|authority| !authority.is_empty()) + .ok_or_else(&invalid)?; + if authority.contains('@') { + return Err(invalid()); + } + + let port = if let Some(bracketed) = authority.strip_prefix('[') { + let (_, suffix) = bracketed.split_once(']').ok_or_else(&invalid)?; + if suffix.is_empty() { + return Ok(None); + } + suffix.strip_prefix(':').ok_or_else(&invalid)? + } else if let Some((_, port)) = authority.rsplit_once(':') { + port + } else { + return Ok(None); + }; + + port.parse().map(Some).map_err(|_| invalid()) +} + +fn infer_kubernetes_local_endpoint_host( + disks_layout: &DisksLayout, + local_port: u16, + raw_kernel_hostname: &str, +) -> Result> { + let raw_kernel_hostname = raw_kernel_hostname.trim(); + let Host::Domain(kernel_hostname) = Host::parse(raw_kernel_hostname) + .map_err(|_| Error::new(ErrorKind::InvalidData, "kernel hostname is not a valid DNS name"))? + else { + return Err(Error::new( + ErrorKind::InvalidData, + "kernel hostname must be a DNS name for Kubernetes endpoint inference", + )); + }; + let kernel_hostname = domain_without_optional_trailing_dot(&kernel_hostname) + .filter(|hostname| !hostname.contains('*')) + .ok_or_else(|| Error::new(ErrorKind::InvalidData, "kernel hostname is not a canonical DNS name"))?; + let kernel_hostname_is_fqdn = kernel_hostname.contains('.'); + let mut candidates = HashSet::new(); + + for raw_endpoint in disks_layout.pools.iter().flat_map(|pool| pool.iter().flatten()) { + let endpoint = Endpoint::try_from(raw_endpoint.as_str())?; + let Some(Host::Domain(endpoint_host)) = endpoint.url.host() else { + continue; + }; + let endpoint_host = domain_without_optional_trailing_dot(endpoint_host) + .ok_or_else(|| Error::new(ErrorKind::InvalidInput, "distributed endpoint host is not canonical"))?; + let hostname_matches = if kernel_hostname_is_fqdn { + endpoint_host == kernel_hostname + } else { + endpoint_host.split('.').next() == Some(kernel_hostname) + }; + if !hostname_matches { + continue; + } + + let endpoint_port = explicit_url_port(raw_endpoint)?.unwrap_or(local_port); + if endpoint_port == local_port { + candidates.insert(endpoint_host.to_string()); + } + } + + match candidates.len() { + 0 => Ok(None), + 1 => Ok(candidates.into_iter().next()), + _ => Err(Error::new( + ErrorKind::InvalidInput, + "kernel hostname matches multiple distributed endpoint hosts; set RUSTFS_LOCAL_ENDPOINT_HOST explicitly", + )), + } +} + +fn parse_explicit_local_endpoint_host(raw: &str) -> Result> { + let invalid = || { + Error::other(format!( + "{ENV_LOCAL_ENDPOINT_HOST} must contain exactly one host without a scheme, port, or path" + )) + }; + let raw = raw.trim(); + if raw.is_empty() { + return Err(invalid()); + } + + let host = Host::parse(raw).map_err(|_| invalid())?; + let host = match host { + Host::Domain(domain) => Host::Domain( + domain_without_optional_trailing_dot(&domain) + .ok_or_else(&invalid)? + .to_string(), + ), + host => host, + }; + if matches!(&host, Host::Domain(domain) if domain.contains('*')) + || matches!(&host, Host::Ipv4(ip) if ip.is_unspecified()) + || matches!(&host, Host::Ipv6(ip) if ip.is_unspecified() + || ip.to_ipv4_mapped().is_some_and(|mapped| mapped.is_unspecified())) + { + return Err(Error::other(format!( + "{ENV_LOCAL_ENDPOINT_HOST} must not be a wildcard or unspecified address" + ))); + } + + Ok(host) +} + +fn domain_without_optional_trailing_dot(domain: &str) -> Option<&str> { + let canonical = domain.strip_suffix('.').unwrap_or(domain); + (!canonical.is_empty() && !canonical.ends_with('.')).then_some(canonical) } fn dns_retry_delay(attempt: u32, max_delay: Duration) -> Duration { @@ -743,7 +986,7 @@ pub(crate) struct StartupTopologyPolicy { impl StartupTopologyPolicy { /// Resolves the policy from the environment. `distributed` is true when the /// endpoints are URL style (the only ones that resolve hostnames). - fn resolve(distributed: bool) -> Self { + fn resolve(distributed: bool, explicit_local_host: bool) -> Result { // `KUBERNETES_SERVICE_HOST` is platform-injected, so probe it directly // rather than through the RUSTFS/MINIO config alias helpers. Self::resolve_from( @@ -752,6 +995,7 @@ impl StartupTopologyPolicy { distributed, get_env_opt_str(ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT).as_deref(), get_env_opt_str(ENV_STARTUP_TOPOLOGY_RETRY_MAX_DELAY).as_deref(), + explicit_local_host, ) } @@ -762,28 +1006,48 @@ impl StartupTopologyPolicy { distributed: bool, timeout_env: Option<&str>, max_delay_env: Option<&str>, - ) -> Self { - let mode = match mode_env.map(|value| value.trim().to_ascii_lowercase()).as_deref() { + explicit_local_host: bool, + ) -> Result { + let mode_env = mode_env.map(|value| value.trim().to_ascii_lowercase()); + let auto_mode = || { + if !distributed { + StartupTopologyWaitMode::FailFast + } else if kubernetes { + StartupTopologyWaitMode::Orchestrated + } else { + StartupTopologyWaitMode::Bounded + } + }; + let mode = match mode_env.as_deref() { Some("orchestrated") => StartupTopologyWaitMode::Orchestrated, Some("bounded") => StartupTopologyWaitMode::Bounded, Some("fail-fast") | Some("failfast") | Some("strict") => StartupTopologyWaitMode::FailFast, - // "auto", unset, or unrecognized: derive from the environment. - // Only URL-style (distributed) endpoints resolve hostnames and need - // to wait for DNS/topology convergence; local path endpoints never - // do, so they fail fast regardless of the platform. - _ => { - if !distributed { - StartupTopologyWaitMode::FailFast - } else if kubernetes { - StartupTopologyWaitMode::Orchestrated - } else { - StartupTopologyWaitMode::Bounded - } + None | Some("") | Some("auto") => auto_mode(), + // RUSTFS_COMPAT_TODO(rustfs-5416-wait-mode): Preserve unknown-mode auto fallback without an anchor. Remove after every supported direct-upgrade chart validates this setting. + Some(_) if !explicit_local_host => auto_mode(), + Some(_) => { + return Err(Error::new( + ErrorKind::InvalidInput, + format!( + "invalid {ENV_STARTUP_TOPOLOGY_WAIT_MODE}; expected auto, orchestrated, bounded, fail-fast, failfast, or strict" + ), + )); } }; - let retry_max_delay = max_delay_env - .and_then(parse_wait_duration) + let parsed_retry_max_delay = max_delay_env.and_then(parse_wait_duration); + if parsed_retry_max_delay.is_some_and(|duration| duration.is_zero()) { + // RUSTFS_COMPAT_TODO(rustfs-5416-zero-retry-delay): Keep zero on the safe default during direct upgrades. Remove after supported configurations no longer rely on the historical parser. + warn!( + event = "startup_topology_zero_retry_delay_defaulted", + component = "ecstore", + subsystem = "endpoint_topology", + state = "compat_default", + "zero startup topology retry delay was replaced with the safe default" + ); + } + let retry_max_delay = parsed_retry_max_delay + .filter(|duration| !duration.is_zero()) .unwrap_or(Duration::from_secs(DEFAULT_STARTUP_TOPOLOGY_RETRY_MAX_DELAY_SECS)); let wait_timeout = match mode { @@ -796,11 +1060,11 @@ impl StartupTopologyPolicy { StartupTopologyWaitMode::FailFast => Duration::ZERO, }; - Self { + Ok(Self { mode, wait_timeout, retry_max_delay, - } + }) } fn is_orchestrated(&self) -> bool { @@ -921,23 +1185,84 @@ impl EndpointServerPools { server_addr: &str, disks_layout: &DisksLayout, ) -> Result<(EndpointServerPools, SetupType)> { - Self::create_server_endpoints_with(server_addr, disks_layout, None).await + let mut local_endpoint_host = get_env_opt_str(ENV_LOCAL_ENDPOINT_HOST); + let mut policy_override = None; + let first_endpoint = disks_layout + .pools + .iter() + .flat_map(|pool| pool.iter().flatten()) + .next() + .map(|endpoint| Endpoint::try_from(endpoint.as_str())) + .transpose()?; + let distributed = first_endpoint + .as_ref() + .is_some_and(|endpoint| endpoint.get_type() == EndpointType::Url); + let mut all_distributed_hosts_are_ip_literals = distributed; + if distributed { + for raw_endpoint in disks_layout.pools.iter().flat_map(|pool| pool.iter().flatten()) { + let endpoint = Endpoint::try_from(raw_endpoint.as_str())?; + if !matches!(endpoint.url.host(), Some(Host::Ipv4(_) | Host::Ipv6(_))) { + all_distributed_hosts_are_ip_literals = false; + break; + } + } + } + let wait_mode = get_env_opt_str(ENV_STARTUP_TOPOLOGY_WAIT_MODE).map(|value| value.trim().to_ascii_lowercase()); + let infer_kubernetes_host = local_endpoint_host.is_none() + && distributed + && !all_distributed_hosts_are_ip_literals + && std::env::var_os(ENV_KUBERNETES_SERVICE_HOST).is_some() + && matches!(wait_mode.as_deref(), None | Some("") | Some("auto") | Some("orchestrated")); + if infer_kubernetes_host { + let kernel_hostname = hostname::get() + .map_err(|err| { + Error::other(format!("failed to read the kernel hostname for Kubernetes endpoint identity: {err}")) + })? + .into_string() + .map_err(|_| Error::new(ErrorKind::InvalidData, "kernel hostname is not valid UTF-8"))?; + let local_port = check_local_server_addr(server_addr)?.port(); + match infer_kubernetes_local_endpoint_host(disks_layout, local_port, &kernel_hostname)? { + Some(inferred_host) => local_endpoint_host = Some(inferred_host), + None => { + // RUSTFS_COMPAT_TODO(rustfs-5416-kubernetes-alias-dns): Keep implicit DNS for pre-anchor Kubernetes aliases. Remove after supported charts always provide an explicit local endpoint host. + if matches!(wait_mode.as_deref(), None | Some("") | Some("auto")) { + policy_override = Some(StartupTopologyPolicy::resolve_from( + Some("bounded"), + true, + distributed, + get_env_opt_str(ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT).as_deref(), + get_env_opt_str(ENV_STARTUP_TOPOLOGY_RETRY_MAX_DELAY).as_deref(), + false, + )?); + } + warn!( + event = "kubernetes_endpoint_identity_dns_fallback", + component = "ecstore", + subsystem = "endpoint_topology", + state = "legacy_dns", + "kernel hostname did not match a configured endpoint; using compatibility DNS locality" + ); + } + } + } + Self::create_server_endpoints_with(server_addr, disks_layout, policy_override, local_endpoint_host.as_deref()).await } /// Same as [`create_server_endpoints`] but lets tests inject an explicit - /// startup topology convergence policy instead of resolving it from the - /// environment (which would otherwise vary with the CI runner's ambient - /// `KUBERNETES_SERVICE_HOST`). + /// startup topology convergence policy and local endpoint host instead of + /// resolving them from the environment. async fn create_server_endpoints_with( server_addr: &str, disks_layout: &DisksLayout, policy_override: Option, + local_endpoint_host: Option<&str>, ) -> Result<(EndpointServerPools, SetupType)> { if disks_layout.pools.is_empty() { return Err(Error::other("Invalid arguments specified")); } - let pool_eps = PoolEndpointList::create_pool_endpoints_with(server_addr, disks_layout, policy_override).await?; + let pool_eps = + PoolEndpointList::create_pool_endpoints_with(server_addr, disks_layout, policy_override, local_endpoint_host).await?; let mut ret: EndpointServerPools = Vec::with_capacity(pool_eps.as_ref().len()).into(); for (i, eps) in pool_eps.inner.into_iter().enumerate() { @@ -1104,7 +1429,7 @@ impl EndpointServerPools { continue; } let host = endpoint.host_port(); - if endpoint.is_local && endpoint.url.port() == Some(runtime_sources::rustfs_port()) && local.is_none() { + if endpoint.is_local && local.is_none() { local = Some(host.clone()); } @@ -1333,28 +1658,65 @@ mod test { use super::*; - #[cfg(target_os = "linux")] use serial_test::serial; use std::path::Path; - #[cfg(target_os = "linux")] use temp_env::async_with_vars; #[cfg(target_os = "linux")] use tempfile::tempdir; + fn local_flags(endpoints: &Endpoints) -> Vec { + endpoints.as_ref().iter().map(|endpoint| endpoint.is_local).collect() + } + #[test] fn retryable_dns_error_accepts_startup_dns_transients() { assert!(is_retryable_dns_error(&Error::new(ErrorKind::TimedOut, "resolver timeout"))); - assert!(is_retryable_dns_error(&Error::other( - "failed to lookup address information: Name or service not known" - ))); - assert!(is_retryable_dns_error(&Error::other("no such host"))); - assert!(is_retryable_dns_error(&Error::other("nodename nor servname provided, or not known"))); + for message in [ + "temporary failure in name resolution", + "failed to lookup address information: temporary failure in name resolution", + "name or service not known", + "failed to lookup address information: name or service not known", + "name does not resolve", + " Failed to lookup address information: Name does not resolve ", + "no such host", + "no such host is known.", + "nodename nor servname provided, or not known", + "failed to lookup address information: nodename nor servname provided, or not known", + ] { + assert!(is_retryable_dns_error(&Error::other(message)), "{message:?} must be retryable"); + } + assert!(is_retryable_dns_error(&Error::new(ErrorKind::NotFound, "no such host"))); + } + + #[tokio::test] + async fn system_resolver_negative_result_reaches_the_dns_allowlist() { + let err = get_host_ip(Host::Domain("rustfs-startup-negative.invalid")) + .await + .expect_err("the reserved .invalid domain must not resolve"); + assert!( + is_retryable_dns_error(&err), + "system resolver error kind {:?} and message {err:?} must retain retry provenance", + err.kind() + ); } #[test] fn retryable_dns_error_accepts_resolver_raw_os_codes() { - assert!(is_retryable_dns_error(&Error::from_raw_os_error(-3))); - assert!(is_retryable_dns_error(&Error::from_raw_os_error(-2))); + for code in [-3, -2, 11001, 11002, 11004] { + assert!(is_retryable_dns_error(&Error::from_raw_os_error(code))); + } + + let positive_eai = cfg!(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd", + target_os = "dragonfly" + )); + for code in [2, 8] { + assert_eq!(is_retryable_dns_error(&Error::from_raw_os_error(code)), positive_eai); + } } #[test] @@ -1364,6 +1726,8 @@ mod test { "invalid URL endpoint format" ))); assert!(!is_retryable_dns_error(&Error::other("mixed scheme is not supported"))); + assert!(!is_retryable_dns_error(&Error::other("request failed: no such host"))); + assert!(!is_retryable_dns_error(&Error::other("try again"))); } #[test] @@ -1409,25 +1773,27 @@ mod test { #[test] fn startup_policy_auto_maps_environment_to_mode() { // Kubernetes -> orchestrated (unbounded by default). - let k8s = StartupTopologyPolicy::resolve_from(None, true, true, None, None); + let k8s = StartupTopologyPolicy::resolve_from(None, true, true, None, None, false).expect("auto mode should resolve"); assert_eq!(k8s.mode, StartupTopologyWaitMode::Orchestrated); assert_eq!(k8s.wait_timeout, Duration::MAX); assert!(k8s.is_orchestrated()); // Distributed URL endpoints, non-Kubernetes -> bounded (default window). - let bounded = StartupTopologyPolicy::resolve_from(None, false, true, None, None); + let bounded = + StartupTopologyPolicy::resolve_from(None, false, true, None, None, false).expect("auto mode should resolve"); assert_eq!(bounded.mode, StartupTopologyWaitMode::Bounded); assert_eq!(bounded.wait_timeout, Duration::from_secs(DEFAULT_STARTUP_TOPOLOGY_WAIT_TIMEOUT_SECS)); assert!(!bounded.is_orchestrated()); // Local path endpoints -> fail-fast (zero wait window). - let local = StartupTopologyPolicy::resolve_from(None, false, false, None, None); + let local = StartupTopologyPolicy::resolve_from(None, false, false, None, None, false).expect("auto mode should resolve"); assert_eq!(local.mode, StartupTopologyWaitMode::FailFast); assert_eq!(local.wait_timeout, Duration::ZERO); // Local path endpoints stay fail-fast even under Kubernetes: they have // no hostnames to resolve, so there is nothing to wait for. - let k8s_local = StartupTopologyPolicy::resolve_from(None, true, false, None, None); + let k8s_local = + StartupTopologyPolicy::resolve_from(None, true, false, None, None, false).expect("auto mode should resolve"); assert_eq!(k8s_local.mode, StartupTopologyWaitMode::FailFast); assert_eq!(k8s_local.wait_timeout, Duration::ZERO); } @@ -1435,30 +1801,64 @@ mod test { #[test] fn startup_policy_explicit_mode_overrides_auto_and_parses_durations() { // Explicit mode wins even under Kubernetes auto-detection. - let forced = StartupTopologyPolicy::resolve_from(Some("bounded"), true, true, Some("2m"), Some("4s")); + let forced = StartupTopologyPolicy::resolve_from(Some("bounded"), true, true, Some("2m"), Some("4s"), false) + .expect("bounded mode should resolve"); assert_eq!(forced.mode, StartupTopologyWaitMode::Bounded); assert_eq!(forced.wait_timeout, Duration::from_secs(120)); assert_eq!(forced.retry_max_delay, Duration::from_secs(4)); // Explicit orchestrated can still be capped by an explicit timeout. - let capped = StartupTopologyPolicy::resolve_from(Some("orchestrated"), false, false, Some("10m"), None); + let capped = StartupTopologyPolicy::resolve_from(Some("orchestrated"), false, false, Some("10m"), None, false) + .expect("orchestrated mode should resolve"); assert_eq!(capped.mode, StartupTopologyWaitMode::Orchestrated); assert_eq!(capped.wait_timeout, Duration::from_secs(600)); - // Unrecognized mode falls back to auto (here: Kubernetes -> orchestrated). - let auto = StartupTopologyPolicy::resolve_from(Some("bogus"), true, true, None, None); - assert_eq!(auto.mode, StartupTopologyWaitMode::Orchestrated); + for auto_value in ["", "auto", " AUTO "] { + let auto = StartupTopologyPolicy::resolve_from(Some(auto_value), true, true, None, None, false) + .expect("explicit auto mode should resolve"); + assert_eq!(auto.mode, StartupTopologyWaitMode::Orchestrated); + } + + let legacy_unknown = StartupTopologyPolicy::resolve_from(Some("invalid-mode"), true, true, None, None, false) + .expect("unknown mode without an explicit host should retain auto compatibility"); + assert_eq!(legacy_unknown.mode, StartupTopologyWaitMode::Orchestrated); + + let invalid = StartupTopologyPolicy::resolve_from(Some("invalid-mode"), true, true, None, None, true).unwrap_err(); + assert_eq!(invalid.kind(), ErrorKind::InvalidInput); + assert!(invalid.to_string().contains(ENV_STARTUP_TOPOLOGY_WAIT_MODE)); // fail-fast accepts a few spellings, trimmed and case-insensitive. for alias in ["fail-fast", "failfast", "strict", " Strict "] { assert_eq!( - StartupTopologyPolicy::resolve_from(Some(alias), false, true, None, None).mode, + StartupTopologyPolicy::resolve_from(Some(alias), false, true, None, None, false) + .expect("fail-fast alias should resolve") + .mode, StartupTopologyWaitMode::FailFast, "alias {alias:?} should resolve to fail-fast" ); } } + #[test] + fn startup_policy_defaults_zero_and_malformed_retry_delays() { + for zero in ["0", "0ms"] { + for explicit_local_host in [false, true] { + let policy = + StartupTopologyPolicy::resolve_from(Some("orchestrated"), true, true, None, Some(zero), explicit_local_host) + .expect("historical zero retry delays should use the safe default"); + assert_eq!(policy.retry_max_delay, Duration::from_secs(DEFAULT_STARTUP_TOPOLOGY_RETRY_MAX_DELAY_SECS)); + } + } + + let malformed = + StartupTopologyPolicy::resolve_from(Some("orchestrated"), true, true, None, Some("invalid-duration"), false) + .expect("malformed historical values should retain default fallback"); + assert_eq!( + malformed.retry_max_delay, + Duration::from_secs(DEFAULT_STARTUP_TOPOLOGY_RETRY_MAX_DELAY_SECS) + ); + } + #[test] fn normalize_same_host_local_state_unifies_divergent_verdicts() { let mut d1 = Endpoint::try_from("http://node1:9000/d1").unwrap(); @@ -1528,12 +1928,8 @@ mod test { ]; let layout = DisksLayout::from_volumes(args.as_slice()).unwrap(); - let bounded = StartupTopologyPolicy { - mode: StartupTopologyWaitMode::Bounded, - wait_timeout: Duration::from_secs(1), - retry_max_delay: DNS_RETRY_MAX_DELAY, - }; - let bounded_err = PoolEndpointList::create_pool_endpoints_with("0.0.0.0:9000", &layout, Some(bounded)) + let bounded = bounded_test_policy(); + let bounded_err = PoolEndpointList::create_pool_endpoints_with("0.0.0.0:9000", &layout, Some(bounded), None) .await .unwrap_err(); assert!( @@ -1543,17 +1939,644 @@ mod test { "bounded mode should run the DNS-IP cross-port check: {bounded_err}" ); - let orchestrated = StartupTopologyPolicy { - mode: StartupTopologyWaitMode::Orchestrated, - wait_timeout: Duration::MAX, - retry_max_delay: DNS_RETRY_MAX_DELAY, - }; - let resolved = PoolEndpointList::create_pool_endpoints_with("0.0.0.0:9000", &layout, Some(orchestrated)) + let orchestrated = orchestrated_test_policy(); + let resolved = PoolEndpointList::create_pool_endpoints_with("0.0.0.0:9000", &layout, Some(orchestrated), None) .await .expect("orchestrated mode should defer the DNS-IP cross-port check"); assert_eq!(resolved.setup_type, SetupType::DistErasure); } + #[test] + fn explicit_local_endpoint_host_accepts_only_a_canonical_host() { + assert_eq!( + parse_explicit_local_endpoint_host(" BÜCHER.example. ").expect("trimmed IDNA host should parse"), + Host::Domain("xn--bcher-kva.example".to_string()) + ); + assert_eq!( + parse_explicit_local_endpoint_host("[2001:db8::1]").expect("bracketed IPv6 host should parse"), + Host::::Ipv6("2001:db8::1".parse().expect("test IPv6 literal should parse")) + ); + assert_eq!( + parse_explicit_local_endpoint_host("192.0.2.10").expect("IPv4 host should parse"), + Host::::Ipv4("192.0.2.10".parse().expect("test IPv4 literal should parse")) + ); + + for invalid in [ + "", + ".", + "example.com..", + "example.com%2e%2e", + "http://example.com", + "example.com:9000", + "example.com/path", + "*", + "*.example.com", + "%2A.example.com", + "0.0.0.0", + "[::]", + "[::ffff:0.0.0.0]", + ] { + assert!( + parse_explicit_local_endpoint_host(invalid).is_err(), + "{invalid:?} must not be accepted as a host-only anchor" + ); + } + } + + #[test] + fn kubernetes_kernel_hostname_inference_requires_one_canonical_host() { + let layout = + DisksLayout::from_volumes(["http://rustfs-{0...2}.rustfs-headless.ns.svc.cluster.local:9000/data"].as_slice()) + .expect("StatefulSet topology should parse"); + for ordinal in 0..3 { + let pod_name = format!("rustfs-{ordinal}"); + let expected_host = format!("{pod_name}.rustfs-headless.ns.svc.cluster.local"); + assert_eq!( + infer_kubernetes_local_endpoint_host(&layout, 9000, &pod_name) + .expect("short kernel hostname should infer safely") + .as_deref(), + Some(expected_host.as_str()) + ); + } + assert_eq!( + infer_kubernetes_local_endpoint_host(&layout, 9000, "rustfs-1.rustfs-headless.ns.svc.cluster.local",) + .expect("FQDN kernel hostname should require an exact match") + .as_deref(), + Some("rustfs-1.rustfs-headless.ns.svc.cluster.local") + ); + assert!( + infer_kubernetes_local_endpoint_host(&layout, 9000, "other-pod") + .expect("no matching endpoint is not ambiguous") + .is_none() + ); + + let wrong_port = DisksLayout::from_volumes(["http://rustfs-1.rustfs-headless.ns.svc.cluster.local:9001/data"].as_slice()) + .expect("wrong-port topology should parse"); + assert!( + infer_kubernetes_local_endpoint_host(&wrong_port, 9000, "rustfs-1") + .expect("a host at another port is not local") + .is_none() + ); + + let ambiguous = DisksLayout::from_volumes( + [ + "http://rustfs-0.first-headless.ns.svc.cluster.local:9000/data0", + "http://rustfs-0.second-headless.ns.svc.cluster.local:9000/data1", + ] + .as_slice(), + ) + .expect("ambiguous topology should parse"); + let err = infer_kubernetes_local_endpoint_host(&ambiguous, 9000, "rustfs-0").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + assert!(err.to_string().contains(ENV_LOCAL_ENDPOINT_HOST)); + } + + #[test] + fn explicit_url_port_distinguishes_omitted_and_scheme_default_ports() { + assert_eq!(explicit_url_port("http://rustfs-0.example/data").unwrap(), None); + assert_eq!(explicit_url_port("http://rustfs-0.example:80/data").unwrap(), Some(80)); + assert_eq!(explicit_url_port("https://[2001:db8::1]:443/data").unwrap(), Some(443)); + + for invalid in [ + r"http://rustfs-0.example:80\data", + r"http:\\rustfs-0.example:80\data", + r"http:/\rustfs-0.example:80\data", + "http://rustfs-0.example:\t80/data", + "https://[2001:db8::1]:\n443/data", + "http://rustfs-0.example:80 ", + ] { + assert!(explicit_url_port(invalid).is_err(), "{invalid:?} must not bypass raw-port validation"); + } + } + + #[test] + fn endpoint_parse_errors_do_not_echo_url_credentials() { + let err = Endpoints::try_from(["http://:topsecret@server/path"].as_slice()).unwrap_err(); + + assert_eq!(err.to_string(), "invalid URL endpoint format (endpoint argument #1)"); + assert!(!err.to_string().contains("topsecret")); + } + + #[tokio::test] + async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port() { + let args = vec![ + "http://rustfs-0.rustfs-headless.ns.svc.cluster.local./data0", + "http://rustfs-0.rustfs-headless.ns.svc.cluster.local.:9443/data1", + "http://rustfs-0.rustfs-headless.ns.svc.cluster.local:9001/data2", + "http://localhost:9443/data3", + ]; + let layout = DisksLayout::from_volumes(args.as_slice()).expect("distributed test topology should parse"); + let orchestrated = orchestrated_test_policy(); + + let (server_pools, setup_type) = EndpointServerPools::create_server_endpoints_with( + "0.0.0.0:9443", + &layout, + Some(orchestrated), + Some("RUSTFS-0.RUSTFS-HEADLESS.NS.SVC.CLUSTER.LOCAL"), + ) + .await + .expect("equivalent FQDN spellings should form one local peer identity"); + let local_flags = local_flags(&server_pools.0[0].endpoints); + + assert_eq!(local_flags, vec![true, true, false, false]); + assert_eq!(setup_type, SetupType::DistErasure); + assert_eq!( + server_pools.0[0].endpoints.as_ref()[1].url.host_str(), + Some("rustfs-0.rustfs-headless.ns.svc.cluster.local.") + ); + + let slots = server_pools.peer_grid_host_slots_sorted(); + assert_eq!(slots.len(), 3); + assert_eq!(slots.iter().filter(|(_, _, is_local)| *is_local).count(), 1); + assert!( + slots.iter().all(|(_, grid_host, is_local)| *is_local || grid_host.is_some()), + "canonical host aliases must not create a remote slot without a grid URL" + ); + } + + #[tokio::test] + async fn explicit_ip_endpoint_host_selects_only_the_matching_address() { + let orchestrated = orchestrated_test_policy(); + + for (endpoints, anchor) in [ + ( + [ + "http://192.0.2.10:9443/data0", + "http://192.0.2.11:9443/data1", + "http://192.0.2.12:9443/data2", + "http://192.0.2.13:9443/data3", + ], + "192.0.2.10", + ), + ( + [ + "http://[2001:db8::1]:9443/data0", + "http://[2001:db8::2]:9443/data1", + "http://[2001:db8::3]:9443/data2", + "http://[2001:db8::4]:9443/data3", + ], + "[2001:db8::1]", + ), + ] { + let layout = DisksLayout::from_volumes(endpoints.as_slice()).expect("distributed IP topology should parse"); + let resolved = + PoolEndpointList::create_pool_endpoints_with("0.0.0.0:9443", &layout, Some(orchestrated), Some(anchor)) + .await + .expect("explicit IP anchor should select its exact endpoint"); + let local_flags = local_flags(&resolved.inner[0]); + + assert_eq!(local_flags, vec![true, false, false, false]); + } + } + + #[serial] + #[tokio::test] + async fn create_server_endpoints_reads_explicit_local_host_from_environment() { + assert_eq!(ENV_LOCAL_ENDPOINT_HOST, "RUSTFS_LOCAL_ENDPOINT_HOST"); + async_with_vars( + [ + ("RUSTFS_LOCAL_ENDPOINT_HOST", Some("rustfs-0.rustfs-headless.ns.svc.cluster.local")), + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("orchestrated")), + ], + async { + let endpoints = vec![ + "http://rustfs-0.rustfs-headless.ns.svc.cluster.local:9000/data0".to_string(), + "http://localhost:9000/data1".to_string(), + "http://missing-1.invalid:9000/data2".to_string(), + "http://missing-2.invalid:9000/data3".to_string(), + ]; + + let (pools, setup_type) = EndpointServerPools::from_volumes("0.0.0.0:9000", endpoints) + .await + .expect("production endpoint entry should consume the explicit host environment"); + let local_flags = local_flags(&pools.0[0].endpoints); + + assert_eq!(local_flags, vec![true, false, false, false]); + assert_eq!(setup_type, SetupType::DistErasure); + }, + ) + .await; + } + + #[serial] + #[tokio::test] + async fn create_server_endpoints_infers_kubernetes_pod_host_without_peer_dns() { + let raw_hostname = hostname::get() + .expect("kernel hostname should be available") + .into_string() + .expect("kernel hostname should be UTF-8"); + let Host::Domain(kernel_hostname) = Host::parse(raw_hostname.trim()).expect("kernel hostname should be a DNS name") + else { + panic!("kernel hostname should be a DNS name"); + }; + let kernel_hostname = + domain_without_optional_trailing_dot(&kernel_hostname).expect("kernel hostname should be canonical"); + let local_host = if kernel_hostname.contains('.') { + kernel_hostname.to_string() + } else { + format!("{kernel_hostname}.rustfs-headless.ns.svc.cluster.local") + }; + + async_with_vars( + [ + (ENV_LOCAL_ENDPOINT_HOST, None), + (ENV_KUBERNETES_SERVICE_HOST, Some("10.0.0.1")), + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("auto")), + ], + async { + let (pools, setup_type) = EndpointServerPools::from_volumes( + "0.0.0.0:9000", + vec![ + format!("http://{local_host}:9000/data0"), + "http://permanently-missing.invalid:9000/data1".to_string(), + ], + ) + .await + .expect("kernel hostname inference should avoid resolving the missing peer"); + + assert_eq!(local_flags(&pools.0[0].endpoints), vec![true, false]); + assert_eq!(setup_type, SetupType::DistErasure); + }, + ) + .await; + } + + #[serial] + #[tokio::test] + async fn create_server_endpoints_keeps_ip_literal_kubernetes_topologies() { + async_with_vars( + [ + (ENV_LOCAL_ENDPOINT_HOST, None), + (ENV_KUBERNETES_SERVICE_HOST, Some("10.0.0.1")), + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("auto")), + ], + async { + let (pools, setup_type) = EndpointServerPools::from_volumes( + "127.0.0.1:9000", + vec![ + "http://127.0.0.1:9000/data0".to_string(), + "http://192.0.2.10:9000/data1".to_string(), + ], + ) + .await + .expect("literal IP endpoints should retain DNS-free locality detection"); + + assert_eq!(local_flags(&pools.0[0].endpoints), vec![true, false]); + assert_eq!(setup_type, SetupType::DistErasure); + }, + ) + .await; + } + + #[serial] + #[tokio::test] + async fn create_server_endpoints_bounds_kubernetes_alias_dns_fallback() { + async_with_vars( + [ + (ENV_LOCAL_ENDPOINT_HOST, None), + (ENV_KUBERNETES_SERVICE_HOST, Some("10.0.0.1")), + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("auto")), + (ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, Some("0ms")), + ], + async { + let err = EndpointServerPools::from_volumes( + "0.0.0.0:9000", + vec![ + "http://unrelated-0.example.invalid:9000/data0".to_string(), + "http://unrelated-1.example.invalid:9000/data1".to_string(), + ], + ) + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::Other); + assert!(err.to_string().contains(TOPOLOGY_TIMEOUT_HINT)); + assert!(!err.to_string().contains(ENV_LOCAL_ENDPOINT_HOST)); + }, + ) + .await; + } + + #[serial] + #[tokio::test] + async fn create_server_endpoints_preserves_resolvable_kubernetes_aliases() { + async_with_vars( + [ + (ENV_LOCAL_ENDPOINT_HOST, None), + (ENV_KUBERNETES_SERVICE_HOST, Some("10.0.0.1")), + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("auto")), + (ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, Some("0ms")), + ], + async { + let (pools, setup_type) = EndpointServerPools::from_volumes( + "0.0.0.0:9000", + vec![ + "http://localhost:9000/data0".to_string(), + "http://localhost:9001/data1".to_string(), + ], + ) + .await + .expect("legacy DNS locality should preserve resolvable non-Pod aliases"); + + assert_eq!(local_flags(&pools.0[0].endpoints), vec![true, false]); + assert_eq!(setup_type, SetupType::DistErasure); + }, + ) + .await; + } + + #[serial] + #[tokio::test] + async fn create_server_endpoints_rejects_an_unknown_wait_mode() { + async_with_vars( + [ + ("RUSTFS_LOCAL_ENDPOINT_HOST", Some("rustfs-0.example")), + (ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("invalid-mode")), + ], + async { + let err = EndpointServerPools::from_volumes( + "0.0.0.0:9000", + vec![ + "http://rustfs-0.example:9000/data0".to_string(), + "http://rustfs-1.example:9000/data1".to_string(), + ], + ) + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::InvalidInput); + assert!(err.to_string().contains(ENV_STARTUP_TOPOLOGY_WAIT_MODE)); + }, + ) + .await; + } + + #[tokio::test] + async fn explicit_local_endpoint_host_matches_the_local_pool_in_multi_pool_topology() { + let layout = DisksLayout::from_volumes( + [ + "http://rustfs-{0...1}.rustfs-headless.ns.svc.cluster.local:9000/data", + "http://rustfs-pool1-{0...1}.rustfs-headless.ns.svc.cluster.local:9000/data", + ] + .as_slice(), + ) + .expect("multi-pool distributed topology should parse"); + let orchestrated = orchestrated_test_policy(); + + let resolved = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &layout, + Some(orchestrated), + Some("rustfs-pool1-0.rustfs-headless.ns.svc.cluster.local"), + ) + .await + .expect("multi-pool explicit host should select its local pool"); + let local_endpoints = resolved + .inner + .iter() + .flat_map(|endpoints| endpoints.as_ref()) + .filter(|endpoint| endpoint.is_local) + .collect::>(); + + assert_eq!(local_endpoints.len(), 1); + assert_eq!( + local_endpoints[0].url.host_str(), + Some("rustfs-pool1-0.rustfs-headless.ns.svc.cluster.local") + ); + assert_eq!(local_endpoints[0].pool_idx, 1); + } + + #[tokio::test] + async fn explicit_local_endpoint_host_fails_closed_for_invalid_context_or_zero_match() { + let args = vec![ + "http://192.0.2.10:9000/data0", + "http://192.0.2.11:9000/data1", + "http://192.0.2.12:9000/data2", + "http://192.0.2.13:9000/data3", + ]; + let layout = DisksLayout::from_volumes(args.as_slice()).expect("distributed test topology should parse"); + let orchestrated = orchestrated_test_policy(); + let bounded = bounded_test_policy(); + + let single_drive_layout = + DisksLayout::from_volumes(["/data"].as_slice()).expect("single-drive test topology should parse"); + let single_drive = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &single_drive_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert!( + single_drive + .to_string() + .contains("only supported for distributed URL endpoints in orchestrated mode") + ); + + let zero_match = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &layout, + Some(orchestrated), + Some("rustfs-0.rustfs-headless.ns.svc.cluster.local"), + ) + .await + .unwrap_err(); + assert!(zero_match.to_string().contains("does not match any distributed endpoint")); + + for invalid_host in ["rustfs-0.example..", "rustfs-0.example%2e%2e"] { + let invalid_layout = DisksLayout::from_volumes( + [ + format!("http://{invalid_host}:9000/data0"), + "http://rustfs-1.example:9000/data1".to_string(), + ] + .as_slice(), + ) + .expect("noncanonical hostname reaches explicit identity validation"); + let err = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &invalid_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + } + + for aliases in [ + ["http://remote.example:9000/data", "http://remote.example.:9000/data"], + ["http://remote.example/data", "http://remote.example:9000/data"], + ] { + let alias_layout = + DisksLayout::from_volumes([aliases[0], aliases[1], "http://rustfs-0.example:9000/local-data"].as_slice()) + .expect("raw endpoint aliases are distinct before explicit identity normalization"); + let err = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &alias_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("duplicate distributed endpoint"), + "canonical endpoint aliases must not occupy multiple erasure slots: {err}" + ); + } + + for (scheme, scheme_port, mismatched_local_port) in [("http", 80, 9000), ("https", 443, 9443)] { + let default_port_endpoints = (0..4) + .map(|index| format!("{scheme}://127.0.0.{}:{scheme_port}/data{index}", index + 1)) + .collect::>(); + let default_port_layout = DisksLayout::from_volumes(default_port_endpoints.as_slice()) + .expect("default-port distributed topology should parse"); + + let mismatch = PoolEndpointList::create_pool_endpoints_with( + &format!("0.0.0.0:{mismatched_local_port}"), + &default_port_layout, + Some(orchestrated), + Some("127.0.0.1"), + ) + .await + .unwrap_err(); + assert!( + mismatch.to_string().contains("does not match any distributed endpoint"), + "{scheme} default port must not be replaced by the local server port: {mismatch}" + ); + + let (server_pools, _) = EndpointServerPools::create_server_endpoints_with( + &format!("0.0.0.0:{scheme_port}"), + &default_port_layout, + Some(orchestrated), + Some("127.0.0.1"), + ) + .await + .expect("matching scheme-default topology should build server pools"); + assert!(server_pools.0[0].endpoints.as_ref()[0].is_local); + let slots = server_pools.peer_grid_host_slots_sorted(); + let mut local_slots = 0; + for (peer, grid_host, is_local) in slots { + assert!( + !peer.contains(':'), + "{scheme} peer identity must preserve the legacy default-port spelling: {peer}" + ); + if is_local { + local_slots += 1; + assert!(grid_host.is_none()); + } else { + assert!(grid_host.is_some()); + } + } + assert_eq!(local_slots, 1, "{scheme} default port must retain exactly one local peer slot"); + } + + let noncanonical_layout = + DisksLayout::from_volumes([r"http://rustfs-0.example:80\data0", "http://rustfs-1.example:9000/data1"].as_slice()) + .expect("noncanonical URL reaches endpoint validation"); + let err = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &noncanonical_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::InvalidInput); + + let distinct_host_ports_layout = DisksLayout::from_volumes( + [ + "http://rustfs-0.example:9000/data", + "http://rustfs-1.example:9001/data", + "http://rustfs-2.example:9000/data2", + "http://rustfs-3.example:9000/data3", + ] + .as_slice(), + ) + .expect("distinct-host port topology should parse"); + let distinct_host_ports = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &distinct_host_ports_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .expect("the same path on different hosts and ports must remain valid"); + assert_eq!(distinct_host_ports.setup_type, SetupType::DistErasure); + + let duplicate_path_layout = DisksLayout::from_volumes( + [ + "http://rustfs-0.example:9000/data0", + "http://rustfs-1.example:9000/data1", + "http://rustfs-1.example:9001/data1", + "http://rustfs-2.example:9000/data2", + ] + .as_slice(), + ) + .expect("distributed duplicate-path test topology should parse"); + let duplicate_path = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &duplicate_path_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert!( + duplicate_path + .to_string() + .contains("same path can not be served by different port on the same explicit endpoint host") + ); + + let cross_pool_duplicate_layout = DisksLayout::from_volumes( + [ + "http://rustfs-{0...1}.example:9000/data", + "http://rustfs-{0...1}.example:9001/data", + ] + .as_slice(), + ) + .expect("cross-pool duplicate-path topology should parse"); + let cross_pool_duplicate = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &cross_pool_duplicate_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert!( + cross_pool_duplicate + .to_string() + .contains("same path can not be served by different port on the same explicit endpoint host") + ); + + let non_orchestrated = + PoolEndpointList::create_pool_endpoints_with("0.0.0.0:9000", &layout, Some(bounded), Some("192.0.2.10")) + .await + .unwrap_err(); + assert!( + non_orchestrated + .to_string() + .contains("only supported for distributed URL endpoints in orchestrated mode") + ); + + let mixed_layout = DisksLayout::from_volumes(["http://rustfs-{0...1}.example/data", "/data{0...1}"].as_slice()) + .expect("mixed multi-pool test topology should parse"); + let mixed = PoolEndpointList::create_pool_endpoints_with( + "0.0.0.0:9000", + &mixed_layout, + Some(orchestrated), + Some("rustfs-0.example"), + ) + .await + .unwrap_err(); + assert!( + mixed + .to_string() + .contains("only supported for distributed URL endpoints in orchestrated mode") + ); + } + #[tokio::test] async fn retry_dns_operation_retries_with_backoff_without_real_sleep() { let deadline = DnsRetryDeadline::new(Duration::from_secs(1), DNS_RETRY_MAX_DELAY); @@ -1720,7 +2743,7 @@ mod test { ), ( vec!["ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"], - Some(Error::other("'ftp://server/d1': io error invalid URL endpoint format")), + Some(Error::other("invalid URL endpoint format")), 10, ), ( @@ -1745,7 +2768,7 @@ mod test { "192.168.1.210:9000/tmp/dir2", "192.168.110:9000/tmp/dir3", ], - Some(Error::other("'192.168.1.210:9000/tmp/dir0': io error")), + Some(Error::other("invalid URL endpoint format: missing scheme http or https")), 13, ), ]; @@ -2297,8 +3320,13 @@ mod test { match ( test_case.expected_err, - PoolEndpointList::create_pool_endpoints_with(test_case.server_addr, &disks_layout, Some(bounded_test_policy())) - .await, + PoolEndpointList::create_pool_endpoints_with( + test_case.server_addr, + &disks_layout, + Some(bounded_test_policy()), + None, + ) + .await, ) { (None, Err(err)) => panic!("Test {}: error: expected = , got = {}", test_case.num, err), (Some(err), Ok(_)) => panic!("Test {}: error: expected = {}, got = ", test_case.num, err), @@ -2367,6 +3395,14 @@ mod test { } } + fn orchestrated_test_policy() -> StartupTopologyPolicy { + StartupTopologyPolicy { + mode: StartupTopologyWaitMode::Orchestrated, + wait_timeout: Duration::MAX, + retry_max_delay: DNS_RETRY_MAX_DELAY, + } + } + fn get_expected_endpoints(args: Vec, prefix: String) -> (Vec, Vec) { let mut urls = vec![]; let mut local_flags = vec![]; @@ -2415,7 +3451,8 @@ mod test { }; let ret = - EndpointServerPools::create_server_endpoints_with(test_case.0, &disks_layout, Some(bounded_test_policy())).await; + EndpointServerPools::create_server_endpoints_with(test_case.0, &disks_layout, Some(bounded_test_policy()), None) + .await; if let Err(err) = ret { if test_case.2 { diff --git a/crates/ecstore/src/layout/format.rs b/crates/ecstore/src/layout/format.rs index 592f6f93b..f0a75abf1 100644 --- a/crates/ecstore/src/layout/format.rs +++ b/crates/ecstore/src/layout/format.rs @@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Error as JsonError; use uuid::Uuid; -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] pub enum FormatMetaVersion { #[serde(rename = "1")] V1, @@ -27,7 +27,7 @@ pub enum FormatMetaVersion { Unknown, } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] pub enum FormatBackend { #[serde(rename = "xl")] Erasure, @@ -64,7 +64,7 @@ pub struct FormatErasureV3 { pub distribution_algo: DistributionAlgoVersion, } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] pub enum FormatErasureVersion { #[serde(rename = "1")] V1, @@ -77,7 +77,7 @@ pub enum FormatErasureVersion { Unknown, } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)] pub enum DistributionAlgoVersion { #[serde(rename = "CRCMOD")] V1, @@ -121,6 +121,15 @@ pub struct FormatV3 { pub disk_info: Option, } +pub(crate) type SharedFormatIdentity<'a> = ( + &'a FormatMetaVersion, + &'a FormatBackend, + &'a Uuid, + &'a FormatErasureVersion, + &'a [Vec], + &'a DistributionAlgoVersion, +); + impl TryFrom<&[u8]> for FormatV3 { type Error = JsonError; @@ -198,52 +207,24 @@ impl FormatV3 { } pub fn check_other(&self, other: &FormatV3) -> Result<()> { - let mut tmp = other.clone(); - let this = tmp.erasure.this; - tmp.erasure.this = Uuid::nil(); - - if self.erasure.sets.len() != other.erasure.sets.len() { - return Err(Error::other(format!( - "Expected number of sets {}, got {}", - self.erasure.sets.len(), - other.erasure.sets.len() - ))); + if self.shared_identity() != other.shared_identity() { + return Err(Error::other("storage formats do not match")); } - for i in 0..self.erasure.sets.len() { - if self.erasure.sets[i].len() != other.erasure.sets[i].len() { - return Err(Error::other(format!( - "Each set should be of same size, expected {}, got {}", - self.erasure.sets[i].len(), - other.erasure.sets[i].len() - ))); - } + self.find_disk_index_by_disk_id(other.erasure.this).map(|_| ()) + } - for j in 0..self.erasure.sets[i].len() { - if self.erasure.sets[i][j] != other.erasure.sets[i][j] { - return Err(Error::other(format!( - "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)", - i, - j, - self.erasure.sets[i][j].to_string(), - other.erasure.sets[i][j].to_string(), - ))); - } - } - } - - for i in 0..tmp.erasure.sets.len() { - for j in 0..tmp.erasure.sets[i].len() { - if this == tmp.erasure.sets[i][j] { - return Ok(()); - } - } - } - - Err(Error::other(format!( - "DriveID {:?} not found in any drive sets {:?}", - this, other.erasure.sets - ))) + /// Fields that must agree across every disk in one erasure format, + /// excluding the disk-specific `this` UUID and runtime-only `disk_info`. + pub(crate) fn shared_identity(&self) -> SharedFormatIdentity<'_> { + ( + &self.version, + &self.format, + &self.id, + &self.erasure.version, + &self.erasure.sets, + &self.erasure.distribution_algo, + ) } } @@ -437,6 +418,30 @@ mod test { assert!(result.is_ok()); } + #[test] + fn test_check_other_rejects_shared_identity_mismatches() { + type FormatMutation = (&'static str, fn(&mut FormatV3)); + + let format = FormatV3::new(1, 2); + let mutations: [FormatMutation; 5] = [ + ("meta version", |other| other.version = FormatMetaVersion::Unknown), + ("backend", |other| other.format = FormatBackend::ErasureSingle), + ("deployment id", |other| other.id = Uuid::new_v4()), + ("erasure version", |other| other.erasure.version = FormatErasureVersion::V2), + ("distribution algorithm", |other| { + other.erasure.distribution_algo = DistributionAlgoVersion::V2 + }), + ]; + + for (field, mutate) in mutations { + let mut other = format.clone(); + other.erasure.this = format.erasure.sets[0][0]; + mutate(&mut other); + + assert!(format.check_other(&other).is_err(), "{field} mismatch must be rejected"); + } + } + #[test] fn test_check_other_different_set_count() { let format1 = FormatV3::new(2, 4); diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index c3f4f4f68..f2df75d2a 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -424,7 +424,11 @@ pub(crate) async fn record_local_disk_id(instance_ctx: &Arc, di pub(crate) async fn replace_local_disk_id(previous: Option, current: Option, endpoint: String) { let id_map = local_disk_id_map_handle(); let mut disk_id_map = id_map.write().await; - if let Some(previous_id) = previous { + if let Some(previous_id) = previous + && disk_id_map + .get(&previous_id) + .is_some_and(|registered_endpoint| registered_endpoint == &endpoint) + { disk_id_map.remove(&previous_id); } if let Some(current_id) = current { @@ -558,10 +562,14 @@ pub(crate) async fn init_tier_config_mgr(store: Arc) -> Result<()> { #[cfg(test)] mod tests { - use super::{LockRegistry, local_node_name, set_local_node_name}; + use super::{ + LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, replace_local_disk_id, + set_local_node_name, + }; use crate::disk::endpoint::Endpoint; use rustfs_lock::{LocalClient, LockClient}; use std::{collections::HashMap, sync::Arc}; + use uuid::Uuid; fn url_endpoint(raw: &str) -> Endpoint { Endpoint { @@ -607,4 +615,17 @@ mod tests { assert_eq!(observed, next); } + + #[tokio::test] + #[serial_test::serial] + async fn clearing_a_stale_disk_id_does_not_remove_another_endpoint() { + clear_local_disk_id_map_for_test().await; + let disk_id = Uuid::new_v4(); + replace_local_disk_id(None, Some(disk_id), "endpoint-a".to_string()).await; + + replace_local_disk_id(Some(disk_id), None, "endpoint-b".to_string()).await; + + assert_eq!(local_disk_path_by_id(&disk_id).await, Some("endpoint-a".to_string())); + clear_local_disk_id_map_for_test().await; + } } diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 564120055..c690707ad 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -2044,15 +2044,10 @@ fn synthesized_disks(host: &str, endpoints: &EndpointServerPools, state: ItemSta /// Whether `peer_host` refers to the same node as an endpoint whose /// `host_port()` is `ep_host_port`. /// -/// `PeerRestClient::host` is an `XHost`, which resolves names to an address on -/// construction (`hosts_sorted` -> `XHost::try_from` -> `to_socket_addrs`), so -/// `peer_host` is the resolved `IP:port`. An endpoint's `host_port()`, however, -/// is `url.host():port` — still the raw `hostname:port` on hostname-based -/// deployments. A plain string compare therefore misses on hostname clusters, -/// leaving the synthesized/degraded drive list empty and `unknownDisks` at 0 -/// (rustfs/rustfs#4607 follow-up). Compare directly first (fast path / IP -/// deployments), then canonicalize the endpoint side through the same `XHost` -/// resolution and compare again. +/// Current topology clients preserve the endpoint `hostname:port`, so the +/// direct comparison is the normal path. The resolution fallback keeps +/// compatibility with older or manually constructed clients whose `XHost` +/// contains a resolved `IP:port` (rustfs/rustfs#4607 follow-up). fn endpoint_host_matches(peer_host: &str, ep_host_port: &str) -> bool { if peer_host == ep_host_port { return true; diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index ce0bc97e0..f60999573 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -72,6 +72,7 @@ use crate::{ cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState}, config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata}, disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, + layout::endpoints::EndpointServerPools, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, runtime::sources as runtime_sources, set_disk::get_lock_acquire_timeout, @@ -904,14 +905,17 @@ async fn remote_tier_mutation_peers() -> io::Result io::Result>> { + let (peers, _, remote_topology_hosts) = PeerRestClient::new_clients_with_topology(endpoints).await; let peers = peers .into_iter() .flatten() .map(|peer| Arc::new(peer) as Arc) .collect::>(); - ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_host_count)?; + ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_topology_hosts.len())?; Ok(peers) } @@ -4307,6 +4311,34 @@ fn tier_config_not_initialized_error(operation: &str) -> std::io::Error { #[cfg(test)] mod tests { use super::*; + use crate::layout::{ + endpoint::Endpoint, + endpoints::{Endpoints, PoolEndpoints, SetupType}, + }; + + struct SetupTypeGuard { + previous: SetupType, + } + + impl SetupTypeGuard { + async fn switch_to(next: SetupType) -> Self { + let previous = runtime_sources::current_setup_type().await; + runtime_sources::set_setup_type(next).await; + Self { previous } + } + } + + impl Drop for SetupTypeGuard { + fn drop(&mut self) { + let previous = self.previous.clone(); + let handle = tokio::runtime::Handle::current(); + tokio::task::block_in_place(|| { + handle.block_on(async move { + runtime_sources::set_setup_type(previous).await; + }); + }); + } + } fn build_s3_tier(name: &str) -> TierConfig { TierConfig { @@ -6354,6 +6386,42 @@ mod tests { assert!(err.to_string().contains("without peer commit clients"), "{err}"); } + #[tokio::test(flavor = "multi_thread")] + #[serial_test::serial] + async fn tier_mutation_peer_composition_preserves_unresolved_topology_slots() { + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let mut endpoint = Endpoint::try_from(format!("http://rustfs-{disk_index}.invalid:9000/data{disk_index}").as_str()) + .expect("unresolved topology endpoint should parse without DNS"); + endpoint.is_local = disk_index == 0; + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + let topology = EndpointServerPools::from(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: "unresolved-tier-mutation-topology".to_string(), + platform: "test".to_string(), + }]); + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + + let peers = remote_tier_mutation_peers_from_topology(topology) + .await + .expect("every unresolved remote topology slot should retain a tier mutation client"); + assert_eq!( + peers.iter().map(|peer| peer.peer_label()).collect::>(), + vec![ + "http://rustfs-1.invalid:9000".to_string(), + "http://rustfs-2.invalid:9000".to_string(), + "http://rustfs-3.invalid:9000".to_string(), + ] + ); + } + #[tokio::test] async fn coordinator_fanout_prepare_failure_aborts_prepared_peers_without_cas() { let manager = TierConfigMgr::new(); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 873837acd..d356108d3 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -110,7 +110,10 @@ use crate::{ object_api::{GetObjectReader, ObjectInfo, PutObjReader}, // event::name::EventName, services::event_notification::{EventArgs, send_event}, - store::init_format::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file}, + store::init_format::{ + formats_match_reference_slots, get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, + save_format_file, + }, }; use bytes::Bytes; use bytesize::ByteSize; diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index db9f27c7e..91e8570d0 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -1373,11 +1373,24 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { let disks = self.disks.read().await.clone(); let (formats, errs) = load_format_erasure_all(&disks, true).await; - let ref_format = match get_format_erasure_in_quorum(&formats) { - Ok(format) => format, + if errs.iter().any(|err| { + matches!( + err, + Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend) + ) + }) { + return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))); + } + let slot_offset = self + .set_index + .checked_mul(self.set_drive_count) + .ok_or_else(|| Error::other("erasure set slot offset overflow"))?; + let ref_format = match get_format_erasure_in_quorum(&formats, slot_offset) { + Ok(format) if format.shared_identity() == self.format.shared_identity() => format, + Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))), Err(err) => { let can_use_cached_layout = count_errs(&errs, &DiskError::UnformattedDisk) > 0 - && formats.iter().flatten().all(|format| self.format.check_other(format).is_ok()) + && formats_match_reference_slots(&formats, &self.format, slot_offset) && errs .iter() .all(|err| err.is_none() || matches!(err, Some(DiskError::UnformattedDisk))); @@ -1388,6 +1401,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { } } }; + if !formats_match_reference_slots(&formats, &ref_format, slot_offset) { + return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))); + } let endpoints = crate::layout::endpoints::Endpoints::from(self.set_endpoints.clone()); let before_drives = crate::layout::set_heal::formats_to_drives_info(&endpoints, &formats, &errs); @@ -1543,11 +1559,16 @@ mod heal_result_report_tests { use crate::disk::error::DiskError; use crate::disk::format::FormatV3; use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk}; + use crate::error::Error; use crate::object_api::{ObjectOptions, PutObjReader}; use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; + use crate::storage_api_contracts::heal::HealOperations as _; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; - use crate::{config::storageclass, store::init_format::save_format_file}; + use crate::{ + config::storageclass, + store::init_format::{load_format_erasure, save_format_file}, + }; use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode}; use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE}; use std::sync::Arc; @@ -1863,6 +1884,44 @@ mod heal_result_report_tests { } } + #[tokio::test] + async fn format_heal_cached_layout_rejects_a_disk_from_another_slot() { + let mut _temp_dirs = Vec::new(); + let mut endpoints = Vec::new(); + let mut disks = Vec::new(); + for disk_index in 0..3 { + let (temp_dir, mut endpoint, disk) = real_disk().await; + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + _temp_dirs.push(temp_dir); + endpoints.push(endpoint); + disks.push(Some(disk)); + } + let set = set_disks_with(disks.clone(), endpoints, 1).await; + let mut wrong_slot = set.format.clone(); + wrong_slot.erasure.this = set.format.erasure.sets[0][1]; + save_format_file(&disks[0], &Some(wrong_slot)) + .await + .expect("wrong-slot format fixture should be saved"); + let mut correct_slot = set.format.clone(); + correct_slot.erasure.this = set.format.erasure.sets[0][2]; + save_format_file(&disks[2], &Some(correct_slot)) + .await + .expect("correct format fixture should be saved"); + + let (_, heal_err) = set + .heal_format(false) + .await + .expect("format heal should report the quorum failure in its result"); + + assert!(matches!(heal_err, Some(Error::CorruptedFormat))); + let unformatted = load_format_erasure(disks[1].as_ref().expect("second disk should be online"), true) + .await + .expect_err("a rejected fallback must not format the missing slot"); + assert_eq!(unformatted, DiskError::UnformattedDisk); + } + // Regression for #955: an offline disk must contribute exactly one drive // record. Before the fix the offline branch fell through and pushed a second // (Corrupt) record for the same disk, so `before/after.drives` grew to diff --git a/crates/ecstore/src/set_disk/ops/locking.rs b/crates/ecstore/src/set_disk/ops/locking.rs index a9787ba13..6d842633b 100644 --- a/crates/ecstore/src/set_disk/ops/locking.rs +++ b/crates/ecstore/src/set_disk/ops/locking.rs @@ -343,20 +343,23 @@ impl SetDisks { } }; - // The drive's format may place it in a different erasure set than this - // one. Claiming a misplaced drive into `self.disks` would let two sets - // manage the same drive and degrade together, so reject it here - // (backlog#799 B19). - if set_idx != self.set_index { + // Claiming a misplaced drive into `self.disks` would let two slots or + // sets manage the same drive and degrade together (backlog#799 B19). + if set_idx != self.set_index || self.set_endpoints.get(disk_idx) != Some(ep) { warn!( - "renew_disk: drive {:?} belongs to set {} but is being renewed on set {}; skipping", - ep, set_idx, self.set_index + endpoint = %ep, + format_set_index = set_idx, + format_disk_index = disk_idx, + endpoint_pool_index = ep.pool_idx, + endpoint_set_index = ep.set_idx, + endpoint_disk_index = ep.disk_idx, + expected_pool_index = self.pool_index, + expected_set_index = self.set_index, + "renew_disk rejected a drive whose endpoint and format do not identify the same topology slot" ); return; } - // Check that the endpoint matches - let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await; new_disk.enable_health_check(); @@ -715,6 +718,108 @@ mod tests { drop(temp_dirs); } + #[tokio::test] + async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() { + let disk_count = 3; + let format = FormatV3::new(1, disk_count); + let mut temp_dirs = Vec::with_capacity(disk_count); + let mut endpoints = Vec::with_capacity(disk_count); + let mut fixture_disks = Vec::with_capacity(disk_count); + + for disk_idx in 0..disk_count { + let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await; + temp_dirs.push(temp_dir); + endpoints.push(endpoint); + fixture_disks.push(disk); + } + + let set_disks = SetDisks::new( + "test-owner".to_string(), + Arc::new(RwLock::new(vec![Some(fixture_disks[0].clone()), None, None])), + disk_count, + disk_count / 2, + 0, + 0, + endpoints.clone(), + format.clone(), + Vec::new(), + ) + .await; + + let mut other_cluster_format = format.clone(); + other_cluster_format.id = Uuid::new_v4(); + other_cluster_format.erasure.this = format.erasure.sets[0][2]; + save_format_file(&Some(fixture_disks[2].clone()), &Some(other_cluster_format)) + .await + .expect("other-cluster format should be written for the rejection test"); + + set_disks.renew_disk(&endpoints[2]).await; + + let disks = set_disks.get_disks_internal().await; + assert_eq!( + disks[0] + .as_ref() + .expect("the canonical first slot must remain attached") + .endpoint(), + endpoints[0] + ); + assert!( + disks[2].is_none(), + "a disk from another deployment must remain detached even when its slot UUID matches" + ); + + let mut correct_format = format.clone(); + correct_format.erasure.this = format.erasure.sets[0][2]; + let replacement_disk = new_disk( + &endpoints[2], + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("third endpoint should reopen after other-cluster rejection"); + save_format_file(&Some(replacement_disk), &Some(correct_format)) + .await + .expect("correct slot format should be restored"); + + set_disks.renew_disk(&endpoints[2]).await; + + let disks = set_disks.get_disks_internal().await; + assert_eq!( + disks[0] + .as_ref() + .expect("the canonical first slot must remain attached") + .endpoint(), + endpoints[0] + ); + assert_eq!(disks[2].as_ref().expect("the restored third slot should attach").endpoint(), endpoints[2]); + + let third_disk = disks[2].clone(); + let mut wrong_slot_format = format.clone(); + wrong_slot_format.erasure.this = format.erasure.sets[0][0]; + save_format_file(&third_disk, &Some(wrong_slot_format)) + .await + .expect("wrong-slot format should be written for the rejection test"); + set_disks.disks.write().await[2] = None; + + let mut misplaced_endpoint = endpoints[2].clone(); + misplaced_endpoint.set_disk_index(0); + set_disks.renew_disk(&misplaced_endpoint).await; + + let disks = set_disks.get_disks_internal().await; + assert_eq!( + disks[0] + .as_ref() + .expect("the canonical first slot must remain attached") + .endpoint(), + endpoints[0] + ); + assert!(disks[2].is_none(), "a disk claiming another endpoint's slot must remain detached"); + + drop(temp_dirs); + } + // SetDisks split P0 (#816): the borrow handle must mirror the core state and // the List operation family must run identically through it. #[tokio::test] diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 26190dd85..fe99c1291 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -30,6 +30,7 @@ impl ECStore { }; let mut count_no_heal = 0; + let mut first_error = None; for pool in self.pools.iter() { let (mut result, err) = pool.heal_format(dry_run).await?; if let Some(err) = err { @@ -37,8 +38,8 @@ impl ECStore { StorageError::NoHealRequired => { count_no_heal += 1; } - _ => { - continue; + err => { + first_error.get_or_insert(err); } } } @@ -47,6 +48,9 @@ impl ECStore { r.before.drives.append(&mut result.before.drives); r.after.drives.append(&mut result.after.drives); } + if let Some(err) = first_error { + return Ok((r, Some(err))); + } if count_no_heal == self.pools.len() { info!( event = EVENT_HEAL_FORMAT_COMPLETED, @@ -165,3 +169,134 @@ impl ECStore { Err(StorageError::NotImplemented) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::disk::{DiskOption, format::FormatV3, new_disk}; + use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + use crate::store::init_format::{load_format_erasure, save_format_file}; + + #[tokio::test] + async fn handle_heal_format_continues_after_a_pool_error() { + let canonical_format = FormatV3::new(1, 3); + let mut foreign_format = canonical_format.clone(); + foreign_format.id = Uuid::new_v4(); + let mut temp_dirs = Vec::new(); + let mut endpoints = Vec::new(); + let mut disks = Vec::new(); + + for disk_index in 0..3 { + let temp_dir = tempfile::tempdir().expect("temporary disk root should be created"); + let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8")) + .expect("temporary endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("temporary disk should open"); + let mut disk_format = foreign_format.clone(); + disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index]; + save_format_file(&Some(disk.clone()), &Some(disk_format)) + .await + .expect("foreign format should be written"); + temp_dirs.push(temp_dir); + endpoints.push(endpoint); + disks.push(Some(disk)); + } + + let pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 3, + endpoints: Endpoints::from(endpoints), + cmd_line: "foreign-format-majority-test".to_string(), + platform: "test".to_string(), + }; + let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1) + .await + .expect("test pool should build around the cached canonical format"); + + let mut recoverable_format = FormatV3::new(1, 3); + recoverable_format.id = canonical_format.id; + let mut recoverable_temp_dirs = Vec::new(); + let mut recoverable_endpoints = Vec::new(); + let mut recoverable_disks = Vec::new(); + let mut unformatted_disk = None; + for disk_index in 0..3 { + let temp_dir = tempfile::tempdir().expect("temporary disk root should be created"); + let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8")) + .expect("temporary endpoint should parse"); + endpoint.set_pool_index(1); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("temporary disk should open"); + if disk_index < 2 { + let mut disk_format = recoverable_format.clone(); + disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index]; + save_format_file(&Some(disk.clone()), &Some(disk_format)) + .await + .expect("recoverable format should be written"); + } else { + unformatted_disk = Some(disk.clone()); + } + recoverable_temp_dirs.push(temp_dir); + recoverable_endpoints.push(endpoint); + recoverable_disks.push(Some(disk)); + } + let recoverable_pool_endpoints = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 3, + endpoints: Endpoints::from(recoverable_endpoints), + cmd_line: "recoverable-format-test".to_string(), + platform: "test".to_string(), + }; + let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1) + .await + .expect("recoverable test pool should build"); + + let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]); + let store = ECStore { + id: canonical_format.id, + disk_map: HashMap::new(), + pools: vec![pool, recoverable_pool], + peer_sys: S3PeerSys::new(&endpoint_pools), + pool_meta: RwLock::new(PoolMeta::default()), + rebalance_meta: RwLock::new(None), + decommission_cancelers: RwLock::new(Vec::new()), + start_gate: Mutex::new(()), + pool_meta_save_gate: Mutex::new(()), + ctx: crate::runtime::instance::bootstrap_ctx(), + }; + + let (result, err) = store + .handle_heal_format(false) + .await + .expect("format heal should return the typed pool error"); + assert!( + matches!(err, Some(StorageError::CorruptedFormat)), + "foreign format majority must not be downgraded to a successful heal: {err:?}" + ); + assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected"); + let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true) + .await + .expect("the later pool should be healed despite the first pool error"); + assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]); + } +} diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 4de6f3f45..4e7128188 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -101,6 +101,10 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool { matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES } +fn should_retry_format_load(err: &Error) -> bool { + !matches!(err, Error::CorruptedFormat) +} + fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool { rebalance_meta_loaded && !decommission_running } @@ -294,7 +298,7 @@ impl ECStore { // periodic monitoring until format loading succeeds. Startup RPC // failures can still spawn recovery probes for peers that come up // after this node. - let (disks, errs) = init_format::init_disks( + let (mut disks, errs) = init_format::init_disks( &pool_eps.endpoints, &DiskOption { cleanup: true, @@ -311,7 +315,7 @@ impl ECStore { loop { match init_format::connect_load_init_formats( pool_first_is_local, - &disks, + &mut disks, pool_eps.set_count, pool_eps.drives_per_set, deployment_id, @@ -319,6 +323,7 @@ impl ECStore { .await { Ok(fm) => break Ok(fm), + Err(e) if !should_retry_format_load(&e) => break Err(e), // Wrap the final error if we are giving up Err(e) if times >= 10 => { break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}"))); @@ -551,7 +556,7 @@ mod tests { LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, load_pool_meta_for_startup, pool_first_endpoint_is_local, pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, - should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, + should_retry_format_load, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, }; #[cfg(feature = "test-util")] use crate::{ @@ -773,6 +778,13 @@ mod tests { assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0)); } + #[test] + fn test_should_retry_format_load_rejects_permanent_corruption() { + assert!(!should_retry_format_load(&StorageError::CorruptedFormat)); + assert!(should_retry_format_load(&StorageError::ErasureReadQuorum)); + assert!(should_retry_format_load(&StorageError::FirstDiskWait)); + } + #[test] fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() { assert!(should_auto_start_rebalance_after_init(false, true)); diff --git a/crates/ecstore/src/store/init_format.rs b/crates/ecstore/src/store/init_format.rs index 01ecb84b6..bd4ccb301 100644 --- a/crates/ecstore/src/store/init_format.rs +++ b/crates/ecstore/src/store/init_format.rs @@ -20,16 +20,21 @@ use crate::{ disk::{ DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET, error::DiskError, - format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, + format::{FormatBackend, FormatErasureVersion, FormatMetaVersion, FormatV3}, new_disk, }, layout::endpoints::Endpoints, }; -use futures::future::join_all; -use std::collections::{HashMap, hash_map::Entry}; +use futures::{future::join_all, stream, stream::StreamExt}; +use std::collections::{HashMap, HashSet}; +use tokio::io::AsyncReadExt; use tracing::{debug, error, info, warn}; use uuid::Uuid; +const LEGACY_FORMAT_READ_CONCURRENCY: usize = 16; +const LEGACY_FORMAT_BASE_MAX_BYTES: usize = 16 * 1024; +const LEGACY_FORMAT_MAX_BYTES_PER_DISK: usize = 64; + pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(eps.as_ref().len()); @@ -59,7 +64,7 @@ pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec], + disks: &mut [Option], set_count: usize, set_drive_count: usize, deployment_id: Option, @@ -68,35 +73,51 @@ pub async fn connect_load_init_formats( check_disk_fatal_errs(&errs)?; - check_format_erasure_values(&formats, set_drive_count)?; + let all_unformatted = should_init_erasure_disks(&errs); + let formats_present = formats.iter().flatten().count(); + let mut format_quorum = (formats_present > 0).then(|| select_format_erasure_in_quorum(&formats, 0)); + if format_quorum.as_ref().is_none_or(Result::is_err) + && errs.iter().any(|error| { + matches!( + error, + Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend) + ) + }) + { + return Err(Error::CorruptedFormat); + } + let resumable_partial_migration = formats_present > 0 + && errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk))) + && format_quorum.as_ref().is_some_and(Result::is_err) + && formats.iter().zip(&errs).all(|(format, error)| { + format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound)) + }); - if first_disk && should_init_erasure_disks(&errs) { - // UnformattedDisk, try migrate from MinIO format first, else create new format - info!("first_disk && should_init_erasure_disks"); - match try_migrate_format(disks, set_count, set_drive_count).await { - Ok(LegacyFormatOutcome::Migrated(fm)) => { + if first_disk && (all_unformatted || resumable_partial_migration) { + info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format"); + match try_migrate_format(disks, &formats, set_count, set_drive_count).await { + Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => { info!("Migrated format from MinIO config"); - return Ok(*fm); + retain_format_quorum_members(disks, &format, &quorum_members, set_drive_count).await?; + return Ok(*format); } Ok(LegacyFormatOutcome::Incompatible) => { - // A MinIO format.json was found on disk but could not be migrated - // (topology/version mismatch or parse failure). Falling through to - // create a FRESH RustFS format changes the object placement layout, - // so the pre-existing MinIO objects will not be readable. Surface - // this loudly instead of silently discarding the legacy data. error!( - "Detected MinIO format.json on disk but could NOT migrate it; initializing a fresh RustFS format instead. \ - Existing MinIO objects will not be readable under the new format. Ensure the RustFS pool / erasure-set \ - topology exactly matches the original MinIO deployment, and that no stale .rustfs.sys/format.json remains." + event = "legacy_format_migration_rejected", + component = "ecstore", + subsystem = "store_init", + state = "incompatible", + "detected MinIO format.json but could not migrate it safely" ); + return Err(Error::CorruptedFormat); } Ok(LegacyFormatOutcome::None) => {} - Err(e) => { - warn!("MinIO format migration attempt failed, will initialize a fresh format: {e}"); - } + Err(e) => return Err(e), + } + if all_unformatted { + let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?; + return Ok(fm); } - let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?; - return Ok(fm); } info!( @@ -114,11 +135,53 @@ pub async fn connect_load_init_formats( return Err(Error::FirstDiskWait); } - let fm = get_format_erasure_in_quorum(&formats)?; + let (fm, quorum_members) = match format_quorum.take() { + Some(result) => result?, + None => select_format_erasure_in_quorum(&formats, 0)?, + }; + check_format_erasure_value_for_topology(&fm, formats.len(), set_drive_count)?; + retain_format_quorum_members(disks, &fm, &quorum_members, set_drive_count).await?; Ok(fm) } +async fn retain_format_quorum_members( + disks: &mut [Option], + format: &FormatV3, + quorum_members: &[bool], + set_drive_count: usize, +) -> Result<()> { + if set_drive_count == 0 || quorum_members.len() != disks.len() { + return Err(Error::CorruptedFormat); + } + for (disk, belongs_to_quorum) in disks.iter().zip(quorum_members) { + if !belongs_to_quorum && let Some(disk) = disk { + disk.set_disk_id(None).await?; + } + } + for (index, (disk, belongs_to_quorum)) in disks.iter().zip(quorum_members).enumerate() { + if !belongs_to_quorum { + continue; + } + let disk_id = format + .erasure + .sets + .get(index / set_drive_count) + .and_then(|set| set.get(index % set_drive_count)) + .ok_or(Error::CorruptedFormat)?; + disk.as_ref() + .ok_or(Error::CorruptedFormat)? + .set_disk_id(Some(*disk_id)) + .await?; + } + for (disk, belongs_to_quorum) in disks.iter_mut().zip(quorum_members) { + if !belongs_to_quorum { + *disk = None; + } + } + Ok(()) +} + pub fn quorum_unformatted_disks(errs: &[Option]) -> bool { count_errs(errs, &DiskError::UnformattedDisk) > (errs.len() / 2) } @@ -166,14 +229,17 @@ async fn init_format_erasure( save_format_file_all(disks, &fms).await?; - get_format_erasure_in_quorum(&fms) + get_format_erasure_in_quorum(&fms, 0) } /// Outcome of attempting to migrate an on-disk MinIO `format.json`. enum LegacyFormatOutcome { /// A compatible MinIO format was found and migrated into RustFS format files. /// Boxed to keep the enum small (`FormatV3` is large; the others are unit variants). - Migrated(Box), + Migrated { + format: Box, + quorum_members: Vec, + }, /// A MinIO `format.json` was present but could not be migrated (topology / /// version mismatch, or a parse failure). The caller must decide how to /// proceed; creating a fresh format would leave the legacy objects unreadable. @@ -185,112 +251,261 @@ enum LegacyFormatOutcome { /// Tries to migrate an on-disk MinIO `format.json` into RustFS format files. /// /// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present -/// and, if so, whether it was compatible. `Err` is only returned for genuine IO -/// failures while persisting the migrated format. +/// and, if so, whether it was compatible. `Err` is returned for read or persist +/// failures that prevent a conclusive migration decision. async fn try_migrate_format( disks: &[Option], + rustfs_formats: &[Option], set_count: usize, set_drive_count: usize, ) -> Result { + let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?; let mut legacy_seen = false; + let mut invalid_legacy_format = false; + let mut read_error: Option = None; + let mut legacy_formats = vec![None; disks.len()]; - for disk in disks.iter().flatten() { - let data = match disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await { - Ok(d) if !d.is_empty() => d, - _ => continue, + let reads = stream::iter(disks.iter().enumerate().map(|(index, disk)| async move { + let Some(disk) = disk else { + return (index, None); + }; + let reader = match disk.read_file(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await { + Ok(reader) => reader, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return (index, None), + Err(err) => return (index, Some(Err(err))), + }; + let data = match read_legacy_format_bytes(reader, legacy_format_max_bytes).await { + Ok(data) => data, + Err(err) => return (index, Some(Err(err))), + }; + if data.is_empty() { + return (index, Some(Ok(None))); + } + + match FormatV3::try_from(data.as_slice()) { + Ok(format) => (index, Some(Ok(Some(format)))), + Err(err) => { + warn!( + event = "legacy_format_decode_failed", + component = "ecstore", + subsystem = "store_init", + disk_index = index, + error = %err, + "failed to decode legacy storage format" + ); + (index, Some(Ok(None))) + } + } + })) + .buffer_unordered(LEGACY_FORMAT_READ_CONCURRENCY); + tokio::pin!(reads); + while let Some((index, read)) = reads.next().await { + let Some(read) = read else { + continue; }; - // A non-empty MinIO format.json exists on at least one disk. legacy_seen = true; - - let fm = match FormatV3::try_from(data.as_ref()) { - Ok(fm) => fm, - Err(e) => { - warn!("failed to parse MinIO format.json, skipping this disk: {e}"); - continue; + match read { + Ok(Some(format)) => legacy_formats[index] = Some(format), + Ok(None) => invalid_legacy_format = true, + Err(err) => { + read_error.get_or_insert_with(|| err.into()); } - }; - - let Some(first_set) = fm.erasure.sets.first() else { - warn!("MinIO format.json has empty erasure.sets, skipping this disk"); - continue; - }; - if fm.erasure.sets.len() != set_count || first_set.len() != set_drive_count { - warn!( - "MinIO format topology mismatch: got {}x{}, expected {}x{}; skipping migration for this disk", - fm.erasure.sets.len(), - first_set.len(), - set_count, - set_drive_count - ); - continue; } - - if fm.erasure.version != FormatErasureVersion::V3 { - warn!( - "MinIO format erasure version is not V3 ({:?}); skipping migration for this disk", - fm.erasure.version - ); - continue; - } - - let mut fms = vec![None; disks.len()]; - for (idx, disk_opt) in disks.iter().enumerate() { - if disk_opt.is_none() { - continue; - } - let set_idx = idx / set_drive_count; - let disk_idx = idx % set_drive_count; - if set_idx >= fm.erasure.sets.len() || disk_idx >= fm.erasure.sets[set_idx].len() { - continue; - } - let mut newfm = fm.clone(); - newfm.erasure.this = fm.erasure.sets[set_idx][disk_idx]; - fms[idx] = Some(newfm); - } - - save_format_file_all(disks, &fms).await?; - return Ok(LegacyFormatOutcome::Migrated(Box::new(get_format_erasure_in_quorum(&fms)?))); } - Ok(if legacy_seen { - LegacyFormatOutcome::Incompatible - } else { - LegacyFormatOutcome::None + if let Some(err) = read_error { + return Err(err); + } + if !legacy_seen { + return Ok(LegacyFormatOutcome::None); + } + if invalid_legacy_format { + return Ok(LegacyFormatOutcome::Incompatible); + } + + let format = match get_format_erasure_in_quorum(&legacy_formats, 0) { + Ok(format) => format, + Err(_) => return Ok(LegacyFormatOutcome::Incompatible), + }; + if format.erasure.sets.len() != set_count + || check_format_erasure_value_for_topology(&format, disks.len(), set_drive_count).is_err() + || !formats_match_reference_slots(&legacy_formats, &format, 0) + || !formats_match_reference_slots(rustfs_formats, &format, 0) + { + return Ok(LegacyFormatOutcome::Incompatible); + } + + let mut formats_to_write = vec![None; disks.len()]; + for (index, disk) in disks.iter().enumerate() { + if disk.is_some() { + let mut disk_format = format.clone(); + disk_format.erasure.this = format.erasure.sets[index / set_drive_count][index % set_drive_count]; + formats_to_write[index] = Some(disk_format); + } + } + + let writes = disks + .iter() + .zip(&formats_to_write) + .zip(rustfs_formats) + .filter(|&((disk, _), existing)| disk.is_some() && existing.is_none()) + .map(|((disk, format), _)| save_format_file(disk, format)); + let mut write_error = None; + for result in join_all(writes).await { + if let Err(error) = result { + write_error.get_or_insert(error); + } + } + for disk in disks.iter().flatten() { + disk.set_disk_id(None).await?; + } + + let (persisted_formats, persisted_errors) = load_format_erasure_all(disks, false).await; + if let Some(error) = persisted_errors + .into_iter() + .flatten() + .find(|error| !matches!(error, DiskError::UnformattedDisk | DiskError::DiskNotFound)) + { + return match error { + DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::InconsistentDisk => { + Ok(LegacyFormatOutcome::Incompatible) + } + error => Err(error.into()), + }; + } + if !formats_match_reference_slots(&persisted_formats, &format, 0) { + return Ok(LegacyFormatOutcome::Incompatible); + } + let (persisted_format, quorum_members) = match select_format_erasure_in_quorum(&persisted_formats, 0) { + Ok(selected) => selected, + Err(error) => return Err(write_error.map_or(error, Into::into)), + }; + check_format_erasure_value_for_topology(&persisted_format, disks.len(), set_drive_count)?; + + Ok(LegacyFormatOutcome::Migrated { + format: Box::new(persisted_format), + quorum_members, }) } -pub fn get_format_erasure_in_quorum(formats: &[Option]) -> Result { - let mut countmap = HashMap::new(); +fn legacy_format_max_bytes(disk_count: usize) -> Result { + disk_count + .checked_mul(LEGACY_FORMAT_MAX_BYTES_PER_DISK) + .and_then(|size| size.checked_add(LEGACY_FORMAT_BASE_MAX_BYTES)) + .ok_or_else(|| Error::other("legacy format size limit overflow")) +} - for f in formats.iter() { - if f.is_some() { - let ds = f.as_ref().unwrap().drives(); - let v = countmap.entry(ds); - match v { - Entry::Occupied(mut entry) => *entry.get_mut() += 1, - Entry::Vacant(vacant) => { - vacant.insert(1); - } - }; +async fn read_legacy_format_bytes(reader: impl tokio::io::AsyncRead + Unpin, max_bytes: usize) -> disk::error::Result> { + let read_limit = max_bytes.checked_add(1).ok_or(DiskError::CorruptedFormat)?; + let mut data = Vec::with_capacity(read_limit.min(64 * 1024)); + reader + .take(u64::try_from(read_limit).map_err(|_| DiskError::CorruptedFormat)?) + .read_to_end(&mut data) + .await?; + if data.len() > max_bytes { + return Err(DiskError::CorruptedFormat); + } + Ok(data) +} + +pub(crate) fn formats_match_reference_slots(formats: &[Option], reference: &FormatV3, slot_offset: usize) -> bool { + let Ok(set_drive_count) = validate_format_erasure_layout(reference) else { + return false; + }; + + formats.iter().enumerate().all(|(index, format)| { + format.as_ref().is_none_or(|format| { + format.shared_identity() == reference.shared_identity() + && slot_offset + .checked_add(index) + .and_then(|slot| { + reference + .erasure + .sets + .get(slot / set_drive_count) + .and_then(|set| set.get(slot % set_drive_count)) + }) + .is_some_and(|expected| *expected == format.erasure.this) + }) + }) +} + +pub fn get_format_erasure_in_quorum(formats: &[Option], slot_offset: usize) -> Result { + select_format_erasure_in_quorum(formats, slot_offset).map(|(format, _)| format) +} + +pub(crate) fn select_format_erasure_in_quorum(formats: &[Option], slot_offset: usize) -> Result<(FormatV3, Vec)> { + let mut candidates = HashMap::new(); + let formats_present = formats.iter().flatten().count(); + let required_votes = formats.len() / 2 + 1; + + for (index, format) in formats + .iter() + .enumerate() + .filter_map(|(index, format)| Some((index, format.as_ref()?))) + { + let Some(slot) = slot_offset.checked_add(index) else { + continue; + }; + let (representative, set_drive_count, member_indices) = candidates + .entry(format.shared_identity()) + .or_insert_with(|| (format, validate_format_erasure_layout(format).ok(), Vec::new())); + let Some(set_drive_count) = *set_drive_count else { + continue; + }; + if representative + .erasure + .sets + .get(slot / set_drive_count) + .and_then(|set| set.get(slot % set_drive_count)) + .is_some_and(|expected| *expected == format.erasure.this) + { + member_indices.push(index); } } - let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0)); + let candidate_groups = candidates + .values() + .filter(|(_, _, member_indices)| !member_indices.is_empty()) + .count(); + let log_quorum_failure = |max_votes| { + warn!( + event = "format_quorum_failed", + component = "ecstore", + subsystem = "store_init", + state = "rejected", + formats_total = formats.len(), + formats_present, + candidate_groups, + max_votes, + required_votes, + "storage format quorum not reached" + ); + }; + let Some((format, _, member_indices)) = candidates + .into_values() + .max_by_key(|(_, _, member_indices)| member_indices.len()) + else { + log_quorum_failure(0); + return Err(Error::ErasureReadQuorum); + }; - if *max_drives == 0 || *max_count <= formats.len() / 2 { - warn!("get_format_erasure_in_quorum fi: {:?}", &formats); + let max_count = member_indices.len(); + if max_count < required_votes { + log_quorum_failure(max_count); return Err(Error::ErasureReadQuorum); } - let format = formats - .iter() - .find(|f| f.as_ref().is_some_and(|v| v.drives().eq(max_drives))) - .ok_or(Error::ErasureReadQuorum)?; - - let mut format = format.as_ref().unwrap().clone(); + let mut format = (*format).clone(); format.erasure.this = Uuid::nil(); + format.disk_info = None; - Ok(format) + let mut member_mask = vec![false; formats.len()]; + for index in member_indices { + member_mask[index] = true; + } + + Ok((format, member_mask)) } pub fn check_format_erasure_values( @@ -298,32 +513,34 @@ pub fn check_format_erasure_values( // disks: &Vec>, set_drive_count: usize, ) -> Result<()> { - for f in formats.iter() { - if f.is_none() { - continue; + let mut checked_identities = HashSet::new(); + for format in formats.iter().flatten() { + // `shared_identity` contains every persisted field inspected by the + // validators; only the per-slot UUID and runtime-only disk info differ. + if checked_identities.insert(format.shared_identity()) { + check_format_erasure_value_for_topology(format, formats.len(), set_drive_count)?; } + } + Ok(()) +} - let f = f.as_ref().unwrap(); - - check_format_erasure_value(f)?; - - let first_set = f.erasure.sets.first().ok_or_else(|| Error::other("erasure.sets is empty"))?; - - if formats.len() != f.erasure.sets.len() * first_set.len() { - return Err(Error::other(format!( - "formats length for erasure.sets does not match: got {}, expected {}", - formats.len(), - f.erasure.sets.len() * first_set.len() - ))); - } - - if first_set.len() != set_drive_count { - return Err(Error::other(format!( - "erasure set length for set_drive_count does not match: got {}, expected {}", - first_set.len(), - set_drive_count - ))); - } +fn check_format_erasure_value_for_topology(format: &FormatV3, format_count: usize, set_drive_count: usize) -> Result<()> { + let set_drive_count_in_format = validate_format_erasure_layout(format)?; + let format_drive_count = format + .erasure + .sets + .len() + .checked_mul(set_drive_count_in_format) + .ok_or_else(|| Error::other("erasure set drive count overflow"))?; + if format_count != format_drive_count { + return Err(Error::other(format!( + "formats length for erasure.sets does not match: got {format_count}, expected {format_drive_count}" + ))); + } + if set_drive_count_in_format != set_drive_count { + return Err(Error::other(format!( + "erasure set length for set_drive_count does not match: got {set_drive_count_in_format}, expected {set_drive_count}" + ))); } Ok(()) } @@ -333,12 +550,53 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> { return Err(Error::other("invalid FormatMetaVersion")); } + if !matches!(format.format, FormatBackend::Erasure | FormatBackend::ErasureSingle) { + return Err(Error::other("invalid FormatBackend")); + } + + if format.id.is_nil() || format.id == Uuid::max() { + return Err(Error::other("invalid deployment ID")); + } + if format.erasure.version != FormatErasureVersion::V3 { return Err(Error::other("invalid FormatErasureVersion")); } Ok(()) } +fn validate_format_erasure_layout(format: &FormatV3) -> Result { + check_format_erasure_value(format)?; + + let set_drive_count = format + .erasure + .sets + .first() + .map(Vec::len) + .filter(|count| *count > 0) + .ok_or_else(|| Error::other("erasure.sets must contain at least one drive"))?; + if format.erasure.sets.iter().any(|set| set.len() != set_drive_count) { + return Err(Error::other("erasure.sets must be rectangular")); + } + let drive_count = format + .erasure + .sets + .len() + .checked_mul(set_drive_count) + .ok_or_else(|| Error::other("erasure set drive count overflow"))?; + let mut disk_ids = HashSet::with_capacity(drive_count); + + for disk_id in format.erasure.sets.iter().flatten() { + if disk_id.is_nil() || *disk_id == Uuid::max() { + return Err(Error::other("erasure.sets contains an invalid disk UUID")); + } + if !disk_ids.insert(*disk_id) { + return Err(Error::other("erasure.sets contains a duplicate disk UUID")); + } + } + + Ok(set_drive_count) +} + // load_format_erasure_all reads all format.json files pub async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); @@ -356,13 +614,9 @@ pub async fn load_format_erasure_all(disks: &[Option], heal: bool) -> } let results = join_all(futures).await; - for (i, result) in results.into_iter().enumerate() { + for result in results { match result { Ok(s) => { - if !heal { - let _ = disks[i].as_ref().unwrap().set_disk_id(Some(s.erasure.this)).await; - } - datas.push(Some(s)); errors.push(None); } @@ -483,6 +737,116 @@ pub fn ec_drives_no_config(set_drive_count: usize) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::layout::endpoint::Endpoint; + use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id}; + use serial_test::serial; + + async fn local_disks(count: usize) -> (tempfile::TempDir, Vec>) { + let temp_dir = tempfile::tempdir().expect("temporary disk root should be created"); + let mut endpoints = Vec::with_capacity(count); + for disk_index in 0..count { + let path = temp_dir.path().join(format!("disk-{disk_index}")); + tokio::fs::create_dir_all(&path) + .await + .expect("temporary disk path should be created"); + let mut endpoint = + Endpoint::try_from(path.to_str().expect("temporary disk path should be UTF-8")).expect("endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + + let (disks, errors) = init_disks( + &Endpoints::from(endpoints), + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await; + assert!(errors.iter().all(Option::is_none), "local disk initialization failed: {errors:?}"); + + (temp_dir, disks) + } + + async fn two_local_disks_with_missing_third() -> (tempfile::TempDir, Vec>) { + let (temp_dir, mut disks) = local_disks(2).await; + disks.push(None); + + (temp_dir, disks) + } + + async fn write_legacy_format(disk: &Option, format: &FormatV3) { + write_legacy_bytes(disk, bytes::Bytes::from(format.to_json().expect("legacy format should serialize"))).await; + } + + async fn write_legacy_majority(disks: &[Option], format: &FormatV3) { + for (index, disk) in disks.iter().enumerate().take(disks.len() / 2 + 1) { + let mut disk_format = format.clone(); + disk_format.erasure.this = format.erasure.sets[0][index]; + write_legacy_format(disk, &disk_format).await; + } + } + + async fn write_legacy_bytes(disk: &Option, data: bytes::Bytes) { + let disk = disk.as_ref().expect("legacy disk should exist"); + if let Err(err) = disk.make_volume(MIGRATING_META_BUCKET).await { + assert_eq!(err, DiskError::VolumeExists, "legacy metadata volume should be created"); + } + disk.write_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE, data) + .await + .expect("legacy format should be written"); + } + + async fn write_rustfs_bytes(disk: &Option, data: bytes::Bytes) { + let disk = disk.as_ref().expect("RustFS disk should exist"); + if let Err(err) = disk.make_volume(RUSTFS_META_BUCKET).await { + assert_eq!(err, DiskError::VolumeExists, "RustFS metadata volume should be created"); + } + disk.write_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE, data) + .await + .expect("RustFS format should be written"); + } + + #[tokio::test] + async fn legacy_format_reader_rejects_oversized_input() { + let max_bytes = 32; + let accepted = read_legacy_format_bytes(std::io::Cursor::new(vec![0; max_bytes]), max_bytes) + .await + .expect("input at the limit should be accepted"); + assert_eq!(accepted.len(), max_bytes); + + let err = read_legacy_format_bytes(std::io::Cursor::new(vec![0; max_bytes + 1]), max_bytes) + .await + .expect_err("input above the limit must be rejected before parsing"); + assert_eq!(err, DiskError::CorruptedFormat); + } + + #[tokio::test] + async fn oversized_legacy_format_fails_closed_without_rustfs_writes() { + let (_temp_dir, mut disks) = local_disks(3).await; + let legacy = FormatV3::new(1, 3); + let oversized_len = legacy_format_max_bytes(disks.len()).expect("size limit should resolve") + 1; + for (index, disk) in disks.iter().enumerate().take(2) { + let mut disk_format = legacy.clone(); + disk_format.erasure.this = legacy.erasure.sets[0][index]; + let mut data = disk_format.to_json().expect("legacy format should serialize").into_bytes(); + data.resize(oversized_len, b' '); + write_legacy_bytes(disk, data.into()).await; + } + + let error = connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect_err("oversized legacy metadata must be rejected"); + assert!(matches!(error, Error::CorruptedFormat), "unexpected error: {error:?}"); + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "oversized legacy metadata must not be migrated"); + assert!( + errors.iter().all(|error| matches!(error, Some(DiskError::UnformattedDisk))), + "oversized legacy metadata must not create RustFS format files: {errors:?}" + ); + } #[test] fn ec_drives_no_config_uses_topology_defaults() { @@ -490,6 +854,579 @@ mod tests { assert_eq!(ec_drives_no_config(2).expect("two-drive topology should resolve"), 1); assert_eq!(ec_drives_no_config(6).expect("six-drive topology should resolve"), 3); } + + #[test] + fn format_quorum_rejects_duplicate_sentinel_and_ragged_layouts() { + let mut duplicate = FormatV3::new(1, 3); + duplicate.erasure.sets[0][1] = duplicate.erasure.sets[0][0]; + + let mut nil = FormatV3::new(1, 3); + nil.erasure.sets[0][2] = Uuid::nil(); + + let mut max = FormatV3::new(1, 3); + max.erasure.sets[0][2] = Uuid::max(); + + let mut ragged = FormatV3::new(2, 2); + ragged.erasure.sets[1].pop(); + + for (name, format, total_slots, voting_slots) in [ + ("duplicate", duplicate, 3, vec![0, 1]), + ("nil", nil, 3, vec![0, 1]), + ("max", max, 3, vec![0, 1]), + ("ragged", ragged, 4, vec![0, 1, 2]), + ] { + let set_drive_count = format.erasure.sets[0].len(); + let mut formats = vec![None; total_slots]; + for index in voting_slots { + let mut vote = format.clone(); + vote.erasure.this = format.erasure.sets[index / set_drive_count][index % set_drive_count]; + formats[index] = Some(vote); + } + + assert!( + matches!(get_format_erasure_in_quorum(&formats, 0), Err(Error::ErasureReadQuorum)), + "{name} layout must not form a format quorum" + ); + assert!( + check_format_erasure_values(&formats, set_drive_count).is_err(), + "{name} layout must fail startup format validation" + ); + } + } + + #[test] + fn format_quorum_rejects_unknown_backend_and_invalid_deployment_ids() { + let mut unknown_backend = FormatV3::new(1, 3); + unknown_backend.format = FormatBackend::Unknown; + + let mut nil_deployment = FormatV3::new(1, 3); + nil_deployment.id = Uuid::nil(); + + let mut max_deployment = FormatV3::new(1, 3); + max_deployment.id = Uuid::max(); + + for (name, format) in [ + ("unknown backend", unknown_backend), + ("nil deployment ID", nil_deployment), + ("max deployment ID", max_deployment), + ] { + let mut formats = vec![None; 3]; + for (index, slot) in formats.iter_mut().take(2).enumerate() { + let mut vote = format.clone(); + vote.erasure.this = format.erasure.sets[0][index]; + *slot = Some(vote); + } + + assert!( + matches!(get_format_erasure_in_quorum(&formats, 0), Err(Error::ErasureReadQuorum)), + "{name} must not form a format quorum" + ); + } + } + + #[test] + fn reference_slot_validation_rejects_wrong_slot_and_foreign_minorities() { + let reference = FormatV3::new(1, 3); + let mut formats = (0..3) + .map(|index| { + let mut format = reference.clone(); + format.erasure.this = reference.erasure.sets[0][index]; + Some(format) + }) + .collect::>(); + assert!(formats_match_reference_slots(&formats, &reference, 0)); + + formats[2].as_mut().expect("third format should exist").erasure.this = reference.erasure.sets[0][0]; + assert!(!formats_match_reference_slots(&formats, &reference, 0)); + + formats[2] = { + let mut foreign = reference.clone(); + foreign.id = Uuid::new_v4(); + foreign.erasure.this = foreign.erasure.sets[0][2]; + Some(foreign) + }; + assert!(!formats_match_reference_slots(&formats, &reference, 0)); + } + + #[tokio::test] + async fn existing_format_load_succeeds_with_a_strict_majority() { + let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await; + let mut format = FormatV3::new(1, 3); + let mut expected = format.clone(); + expected.erasure.this = Uuid::nil(); + + format.erasure.this = format.erasure.sets[0][0]; + save_format_file(&disks[0], &Some(format.clone())) + .await + .expect("existing format should be written to the first disk"); + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::ErasureReadQuorum) + )); + assert!(matches!( + load_format_erasure(disks[1].as_ref().expect("second disk should exist"), false).await, + Err(DiskError::UnformattedDisk) + )); + + format.erasure.this = format.erasure.sets[0][1]; + save_format_file(&disks[1], &Some(format)) + .await + .expect("existing format should be written to the second disk"); + + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("two existing formats should satisfy the production load path"), + expected + ); + } + + #[tokio::test] + async fn existing_format_load_rejects_conflicting_formats_without_a_majority() { + let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await; + let mut first = FormatV3::new(1, 3); + first.erasure.this = first.erasure.sets[0][0]; + save_format_file(&disks[0], &Some(first)) + .await + .expect("first existing format should be written"); + + let mut second = FormatV3::new(1, 3); + second.erasure.this = second.erasure.sets[0][1]; + save_format_file(&disks[1], &Some(second)) + .await + .expect("conflicting existing format should be written"); + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::ErasureReadQuorum) + )); + } + + #[tokio::test] + async fn existing_format_load_excludes_disks_outside_the_selected_quorum() { + for wrong_slot_id in [false, true] { + let (_temp_dir, mut disks) = local_disks(3).await; + let mut majority = FormatV3::new(1, 3); + let mut expected = majority.clone(); + expected.erasure.this = Uuid::nil(); + + for (index, disk) in disks.iter().take(2).enumerate() { + majority.erasure.this = majority.erasure.sets[0][index]; + save_format_file(disk, &Some(majority.clone())) + .await + .expect("majority format should be written"); + } + + let mut outlier = if wrong_slot_id { + majority.clone() + } else { + FormatV3::new(1, 3) + }; + outlier.erasure.this = if wrong_slot_id { + outlier.erasure.sets[0][0] + } else { + outlier.erasure.sets[0][2] + }; + save_format_file(&disks[2], &Some(outlier)) + .await + .expect("outlier format should be written"); + + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("two valid members should select the majority format"), + expected + ); + assert!(disks[0].is_some() && disks[1].is_some()); + assert!(disks[2].is_none(), "the outlier disk must not enter the selected erasure set"); + } + } + + #[tokio::test] + #[serial] + async fn existing_format_load_publishes_only_validated_quorum_disk_ids() { + clear_local_disk_id_map_for_test().await; + let (_temp_dir, mut disks) = local_disks(5).await; + let canonical = FormatV3::new(1, 5); + for (index, disk) in disks.iter().enumerate().take(3) { + let mut disk_format = canonical.clone(); + disk_format.erasure.this = canonical.erasure.sets[0][index]; + save_format_file(disk, &Some(disk_format)) + .await + .expect("canonical format should be written"); + } + let mut wrong_slot = canonical.clone(); + wrong_slot.erasure.this = wrong_slot.erasure.sets[0][0]; + save_format_file(&disks[3], &Some(wrong_slot)) + .await + .expect("wrong-slot format should be written"); + let mut foreign = FormatV3::new(1, 5); + foreign.erasure.this = foreign.erasure.sets[0][4]; + let foreign_id = foreign.erasure.this; + save_format_file(&disks[4], &Some(foreign)) + .await + .expect("foreign format should be written"); + let canonical_id = canonical.erasure.sets[0][0]; + let canonical_endpoint = disks[0].as_ref().expect("canonical disk should exist").endpoint().to_string(); + + connect_load_init_formats(true, &mut disks, 1, 5, None) + .await + .expect("the canonical strict majority should load"); + + assert!(disks[..3].iter().all(Option::is_some)); + assert!(disks[3..].iter().all(Option::is_none)); + assert_eq!(local_disk_path_by_id(&canonical_id).await, Some(canonical_endpoint)); + assert_eq!(local_disk_path_by_id(&foreign_id).await, None); + clear_local_disk_id_map_for_test().await; + } + + #[tokio::test] + async fn existing_format_load_excludes_a_malformed_outlier() { + let (_temp_dir, mut disks) = local_disks(3).await; + let mut majority = FormatV3::new(1, 3); + let mut expected = majority.clone(); + expected.erasure.this = Uuid::nil(); + + for (index, disk) in disks.iter().take(2).enumerate() { + majority.erasure.this = majority.erasure.sets[0][index]; + save_format_file(disk, &Some(majority.clone())) + .await + .expect("majority format should be written"); + } + let mut malformed = majority; + malformed.erasure.sets[0][1] = malformed.erasure.sets[0][0]; + malformed.erasure.this = malformed.erasure.sets[0][2]; + save_format_file(&disks[2], &Some(malformed)) + .await + .expect("malformed outlier should be written"); + + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("one malformed outlier must not block a valid strict majority"), + expected + ); + assert!(disks[0].is_some() && disks[1].is_some()); + assert!(disks[2].is_none(), "the malformed outlier must be isolated"); + } + + #[tokio::test] + async fn fresh_format_load_does_not_initialize_with_a_missing_disk() { + let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::FirstDiskWait) + )); + assert!(matches!( + connect_load_init_formats(false, &mut disks, 1, 3, None).await, + Err(Error::NotFirstDisk) + )); + + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none)); + assert!(matches!( + errors.as_slice(), + [ + Some(DiskError::UnformattedDisk), + Some(DiskError::UnformattedDisk), + Some(DiskError::DiskNotFound) + ] + )); + } + + #[tokio::test] + async fn compatible_legacy_format_migrates() { + let (_temp_dir, mut disks) = local_disks(3).await; + let legacy = FormatV3::new(1, 3); + write_legacy_majority(&disks, &legacy).await; + + let mut expected = legacy; + expected.erasure.this = Uuid::nil(); + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("compatible legacy format should migrate"), + expected + ); + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!( + errors.iter().all(Option::is_none), + "every available disk should receive the migrated format" + ); + for (index, format) in formats.iter().enumerate() { + assert_eq!( + format.as_ref().map(|format| format.erasure.this), + Some(expected.erasure.sets[0][index]), + "the migrated format must preserve each disk's physical slot" + ); + } + } + + #[tokio::test] + async fn compatible_legacy_format_migrates_when_the_file_is_missing() { + let (_temp_dir, mut disks) = local_disks(3).await; + let legacy = FormatV3::new(1, 3); + write_legacy_majority(&disks, &legacy).await; + disks[2] + .as_ref() + .expect("third legacy disk should exist") + .make_volume(MIGRATING_META_BUCKET) + .await + .expect("legacy metadata volume should be created without a format file"); + + connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("a missing legacy format file should be initialized from the majority"); + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!( + errors.iter().all(Option::is_none), + "every available disk should receive the migrated format" + ); + assert_eq!( + formats[2].as_ref().map(|format| format.erasure.this), + Some(legacy.erasure.sets[0][2]), + "the missing legacy format file must be replaced with the third slot" + ); + } + + #[tokio::test] + async fn legacy_format_migration_resumes_partial_rustfs_writes() { + for drive_count in [3, 4] { + let (_temp_dir, mut disks) = local_disks(drive_count).await; + let legacy = FormatV3::new(1, drive_count); + write_legacy_majority(&disks, &legacy).await; + for (index, disk) in disks.iter().enumerate().take(drive_count / 2) { + let mut disk_format = legacy.clone(); + disk_format.erasure.this = legacy.erasure.sets[0][index]; + save_format_file(disk, &Some(disk_format)) + .await + .expect("partial RustFS format should be written"); + } + + let mut expected = legacy; + expected.erasure.this = Uuid::nil(); + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, drive_count, None) + .await + .expect("a partial migration should resume from the legacy quorum"), + expected + ); + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!(errors.iter().all(Option::is_none), "the resumed migration should format every disk"); + for (index, format) in formats.iter().enumerate() { + assert_eq!( + format.as_ref().map(|format| format.erasure.this), + Some(expected.erasure.sets[0][index]), + "the resumed migration must preserve each physical slot" + ); + } + } + } + + #[tokio::test] + async fn legacy_format_migration_resumes_to_quorum_with_an_unavailable_disk() { + let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await; + let legacy = FormatV3::new(1, 3); + write_legacy_majority(&disks, &legacy).await; + let mut partial = legacy.clone(); + partial.erasure.this = partial.erasure.sets[0][0]; + save_format_file(&disks[0], &Some(partial)) + .await + .expect("partial RustFS format should be written"); + + let mut expected = legacy; + expected.erasure.this = Uuid::nil(); + assert_eq!( + connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect("the available blank disk should complete the migration quorum"), + expected + ); + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!(formats[0].is_some() && formats[1].is_some()); + assert!(formats[2].is_none()); + assert!(matches!(errors[2], Some(DiskError::DiskNotFound))); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() { + let (_temp_dir, mut disks) = local_disks(3).await; + let legacy = FormatV3::new(1, 3); + write_legacy_majority(&disks, &legacy).await; + let mut foreign = FormatV3::new(1, 3); + foreign.erasure.this = foreign.erasure.sets[0][0]; + let foreign_id = foreign.id; + save_format_file(&disks[0], &Some(foreign)) + .await + .expect("foreign partial RustFS format should be written"); + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id)); + assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended"); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_a_wrong_slot_partial_rustfs_format() { + let (_temp_dir, mut disks) = local_disks(3).await; + let legacy = FormatV3::new(1, 3); + write_legacy_majority(&disks, &legacy).await; + let mut wrong_slot = legacy; + wrong_slot.erasure.this = wrong_slot.erasure.sets[0][1]; + save_format_file(&disks[0], &Some(wrong_slot)) + .await + .expect("wrong-slot partial RustFS format should be written"); + + let error = connect_load_init_formats(true, &mut disks, 1, 3, None) + .await + .expect_err("a wrong-slot partial RustFS format must be rejected"); + assert!(matches!(error, Error::CorruptedFormat), "unexpected error: {error:?}"); + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!(matches!(errors[0], Some(DiskError::InconsistentDisk))); + assert!(formats[1..].iter().all(Option::is_none), "wrong-slot partial state must not be extended"); + } + + #[tokio::test] + async fn legacy_format_migration_does_not_extend_a_malformed_rustfs_format() { + let (_temp_dir, mut disks) = local_disks(3).await; + let legacy = FormatV3::new(1, 3); + write_legacy_majority(&disks, &legacy).await; + write_rustfs_bytes(&disks[0], bytes::Bytes::from_static(b"{not-json")).await; + + assert!(connect_load_init_formats(true, &mut disks, 1, 3, None).await.is_err()); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "malformed partial state must not be extended"); + } + + #[tokio::test] + async fn incompatible_legacy_format_fails_closed() { + let (_temp_dir, mut disks) = local_disks(3).await; + let mut legacy = FormatV3::new(1, 3); + legacy.id = Uuid::nil(); + write_legacy_majority(&disks, &legacy).await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + + let (formats, errors) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "incompatible legacy metadata must not be replaced"); + assert!( + errors.iter().all(|error| matches!(error, Some(DiskError::UnformattedDisk))), + "fresh RustFS formats must not be written after a rejected migration: {errors:?}" + ); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_a_foreign_minority() { + let (_temp_dir, mut disks) = local_disks(3).await; + let majority = FormatV3::new(1, 3); + write_legacy_majority(&disks, &majority).await; + let minority = FormatV3::new(1, 3); + let mut minority_disk = minority; + minority_disk.erasure.this = minority_disk.erasure.sets[0][2]; + write_legacy_format(&disks[2], &minority_disk).await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "a foreign legacy minority must not be overwritten"); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_a_wrong_slot_minority() { + let (_temp_dir, mut disks) = local_disks(3).await; + let majority = FormatV3::new(1, 3); + write_legacy_majority(&disks, &majority).await; + let mut wrong_slot = majority; + wrong_slot.erasure.this = wrong_slot.erasure.sets[0][1]; + write_legacy_format(&disks[2], &wrong_slot).await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!( + formats.iter().all(Option::is_none), + "a wrong-slot legacy minority must not be overwritten" + ); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_a_malformed_minority() { + let (_temp_dir, mut disks) = local_disks(3).await; + let majority = FormatV3::new(1, 3); + write_legacy_majority(&disks, &majority).await; + write_legacy_bytes(&disks[2], bytes::Bytes::from_static(b"{not-json")).await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "a malformed legacy minority must not be overwritten"); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_an_empty_minority() { + let (_temp_dir, mut disks) = local_disks(3).await; + let majority = FormatV3::new(1, 3); + write_legacy_majority(&disks, &majority).await; + write_legacy_bytes(&disks[2], bytes::Bytes::new()).await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "an empty legacy minority must not be overwritten"); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_a_parseable_invalid_minority() { + let (_temp_dir, mut disks) = local_disks(3).await; + let majority = FormatV3::new(1, 3); + write_legacy_majority(&disks, &majority).await; + let mut invalid = majority; + invalid.id = Uuid::nil(); + invalid.erasure.this = invalid.erasure.sets[0][2]; + write_legacy_format(&disks[2], &invalid).await; + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!( + formats.iter().all(Option::is_none), + "a parseable invalid legacy minority must not be overwritten" + ); + } + + #[tokio::test] + async fn legacy_format_migration_rejects_conflicting_candidates_without_a_quorum() { + let (_temp_dir, mut disks) = local_disks(3).await; + for (index, disk) in disks.iter().enumerate().take(2) { + let mut disk_format = FormatV3::new(1, 3); + disk_format.erasure.this = disk_format.erasure.sets[0][index]; + write_legacy_format(disk, &disk_format).await; + } + + assert!(matches!( + connect_load_init_formats(true, &mut disks, 1, 3, None).await, + Err(Error::CorruptedFormat) + )); + let (formats, _) = load_format_erasure_all(&disks, false).await; + assert!(formats.iter().all(Option::is_none), "a legacy split vote must not write RustFS formats"); + } } // #[derive(Debug, PartialEq, thiserror::Error)] diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 6ddb6d5bb..6c7f2523a 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -989,7 +989,7 @@ mod tests { init_local_disks(endpoint_pools.clone()).await.expect("init local disks"); - let (disks, errs) = init_disks( + let (mut disks, errs) = init_disks( &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints, &DiskOption { cleanup: true, @@ -999,7 +999,7 @@ mod tests { .await; assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}"); - connect_load_init_formats(true, &disks, 1, 4, None) + connect_load_init_formats(true, &mut disks, 1, 4, None) .await .expect("initialize format metadata"); diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs index ab05fea83..d9bf106f5 100644 --- a/crates/utils/src/net.rs +++ b/crates/utils/src/net.rs @@ -92,7 +92,6 @@ fn resolve_domain(domain: &str) -> std::io::Result> { (domain, 0) .to_socket_addrs() .map(|v| v.map(|v| v.ip()).collect::>()) - .map_err(Error::other) } #[cfg(test)] @@ -284,7 +283,7 @@ pub async fn get_host_ip(host: Host<&str>) -> std::io::Result> { } Err(err) => { error!("Failed to resolve domain {domain} using system resolver, err: {err}"); - Err(Error::other(err)) + Err(err) } } } @@ -608,6 +607,27 @@ mod test { assert!(get_host_ip(invalid_host).await.is_err()); } + #[tokio::test] + async fn test_get_host_ip_preserves_resolver_error_provenance() { + let _resolver_guard = set_mock_dns_resolver(|_| Err(IoError::from_raw_os_error(-3))); + + let err = get_host_ip(Host::Domain("temporarily-unavailable.example")) + .await + .unwrap_err(); + + assert_eq!(err.raw_os_error(), Some(-3)); + } + + #[test] + fn test_resolve_domain_preserves_system_resolver_error_provenance() { + let _resolver_lock = DNS_RESOLVER_TEST_LOCK.lock().unwrap(); + reset_dns_resolver_inner(); + + let err = resolve_domain("rustfs-resolver-provenance.invalid").unwrap_err(); + + assert_ne!(err.kind(), std::io::ErrorKind::Other, "system resolver error was wrapped: {err}"); + } + #[test] fn test_dns_cache_entry_expiry() { let ips: HashSet = [IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))].into_iter().collect(); diff --git a/crates/utils/src/string.rs b/crates/utils/src/string.rs index f8f798de9..c22dbf62e 100644 --- a/crates/utils/src/string.rs +++ b/crates/utils/src/string.rs @@ -360,7 +360,7 @@ pub fn find_ellipses_patterns(arg: &str) -> Result { Some(caps) => caps, None => { return Err(Error::other(format!( - "Invalid ellipsis format in ({arg}), Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}" + "Invalid ellipsis format. Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}" ))); } }; @@ -399,7 +399,7 @@ pub fn find_ellipses_patterns(arg: &str) -> Result { || p.suffix.contains(CLOSE_BRACES) { return Err(Error::other(format!( - "Invalid ellipsis format in ({arg}), Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}" + "Invalid ellipsis format. Ellipsis range must be provided in format {{N...M}} where N and M are decimal or hexadecimal positive integers, M must be greater than N, with a maximum expanded range size of {MAX_ELLIPSES_RANGE_SIZE}" ))); } } @@ -938,6 +938,15 @@ mod tests { assert!(err.to_string().contains("decimal or hexadecimal"), "unexpected error message: {err}"); } + #[test] + fn test_find_ellipses_patterns_leftover_brace_error_does_not_echo_input() { + let err = find_ellipses_patterns("http://:brace-secret@server/{1...2}}").unwrap_err(); + assert!( + !err.to_string().contains("brace-secret"), + "ellipsis error leaked endpoint credentials: {err}" + ); + } + #[test] fn test_parse_ellipses_range_rejects_oversized_ranges() { let err = parse_ellipses_range("{0...10000}").unwrap_err(); diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 90718159b..1bdc236a3 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,10 @@ for later deletion. ## Open Items +- `rustfs-5416` Helm distributed startup wait setting: charts that predate explicit local endpoint identity expose startupWaitTimeoutSeconds for their peer DNS/TCP init gate. The new chart keeps the value accepted but ignores it after moving startup convergence into RustFS. Remove the value and its documentation after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders the peer gate. +- `rustfs-5416-wait-mode` startup wait-mode validation: releases before explicit local endpoint identity treat an unknown RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE value as auto. New servers retain that fallback only when no explicit local endpoint host is configured; an anchor requires a recognized mode so a typo cannot bypass DNS locality. Remove the fallback and reject every unknown value after every supported direct-upgrade chart validates this setting before rollout. +- `rustfs-5416-kubernetes-alias-dns` Kubernetes endpoint identity fallback: deployments created before explicit local endpoint identity may use resolvable aliases that do not match the Pod hostname. An implicit auto-mode zero match retains legacy DNS locality with a bounded deadline, while ambiguous matches and invalid explicit anchors still fail closed. Remove the fallback after every supported direct-upgrade chart and deployment manifest provides a canonical RUSTFS_LOCAL_ENDPOINT_HOST for domain-based distributed topologies. +- `rustfs-5416-zero-retry-delay` startup retry-delay validation: releases before bounded topology convergence accept RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY values of 0 or 0ms. New servers replace those values with the safe nonzero default so a direct upgrade neither fails startup nor enters a busy loop. Reject zero after the minimum supported direct-upgrade release validates or rewrites this setting before rollout. - `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`. - `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version. - `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability. diff --git a/helm/README.md b/helm/README.md index 07df2792d..148d38cda 100644 --- a/helm/README.md +++ b/helm/README.md @@ -47,6 +47,60 @@ without manual intervention: If you previously set `replicaCount=16` and now want a different topology, set both `replicaCount` and `drivesPerNode` explicitly. +For distributed deployments that use the chart-generated +`RUSTFS_VOLUMES`, `localEndpointHost.autoInject` defaults to automatic +selection. Without `secret.existingSecret`, the chart injects a private +Downward API variable, `RUSTFS_CHART_POD_NAME`, and uses it to build the pod's +fully qualified hostname in +`RUSTFS_LOCAL_ENDPOINT_HOST`. RustFS can then identify local drives without +waiting for every peer's DNS record or TCP listener. A user-defined `POD_NAME` +is preserved and does not interfere with the private chart variable. Setting +`RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE` explicitly to `bounded`, `fail-fast`, +`failfast`, or `strict` also keeps legacy DNS-based locality discovery. A +dynamically sourced wait mode keeps the legacy path because its value cannot be +validated while rendering. Otherwise, Kubernetes auto-detection selects +orchestrated startup when RustFS consumes the generated anchor. + +An existing Secret is opaque to the chart and may historically contain more +than credentials, so the chart does not inject an anchor whenever +`secret.existingSecret` is set. In Kubernetes auto/orchestrated mode, RustFS +then derives a DNS-free identity from the kernel hostname when exactly one +domain endpoint at the server port has the same full hostname or first label. +All-IP topologies retain direct IP locality detection. A domain topology with +zero matches retains legacy DNS locality discovery; implicit auto mode bounds +that compatibility path by `RUSTFS_STARTUP_TOPOLOGY_WAIT_TIMEOUT` (180 seconds +by default). Multiple matching candidates remain an error. Set +`RUSTFS_LOCAL_ENDPOINT_HOST` explicitly to avoid DNS discovery. For a +credentials-only Secret, set +`localEndpointHost.autoInject=true` to add the chart anchor without changing +the historical ConfigMap-then-Secret `envFrom` precedence. If injection is +explicitly enabled with an incompatible hidden `RUSTFS_VOLUMES`, +`RUSTFS_ADDRESS`, or `RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE`, RustFS also fails +during endpoint construction; it does not silently fall back to a different +topology. + +When `config.rustfs.volumes` is set explicitly, the chart does not infer a +local endpoint identity. RustFS applies the same kernel-hostname inference to +custom domain topologies in Kubernetes auto/orchestrated mode; aliases that do +not match the Pod hostname retain legacy DNS locality, with the bounded auto +fallback described above. They may provide `RUSTFS_LOCAL_ENDPOINT_HOST` +through `extraEnv` for DNS-free startup. An explicit `RUSTFS_VOLUMES`, explicit +`RUSTFS_LOCAL_ENDPOINT_HOST`, bounded/dynamic or unrecognized startup mode, or +`localEndpointHost.autoInject=false`, also disables chart injection. A +`RUSTFS_ADDRESS` override alone does not disable it; the effective address and +generated topology must agree on the endpoint port. Custom anchor-based +configurations must resolve to `orchestrated` startup mode and must not receive +a conflicting mode from an `envFrom` source. +`startupWaitTimeoutSeconds` is retained for values-file compatibility but is +deprecated and ignored. +Historical `RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY` values of `0` or `0ms` +are replaced with the safe default retry cap instead of causing a busy loop or +blocking a direct upgrade. + +Upgrade the chart and RustFS image together. An older image that does not +recognize `RUSTFS_LOCAL_ENDPOINT_HOST` retains its previous DNS-based startup +behavior. + --- # Parameters Overview @@ -57,6 +111,7 @@ set both `replicaCount` and `drivesPerNode` explicitly. | affinity.podAntiAffinity.enabled | bool | `true` | | | affinity.podAntiAffinity.topologyKey | string | `"kubernetes.io/hostname"` | | | clusterDomain | string | `"cluster.local"` | Kubernetes cluster DNS domain used to build in-cluster FQDNs for `RUSTFS_VOLUMES` (distributed mode) and mTLS server certificate SANs. Override for clusters not using the default `cluster.local`. Provide the DNS root only, without a `svc.` prefix or leading/trailing dots. | +| localEndpointHost.autoInject | bool or null | `null` | Automatically inject `RUSTFS_LOCAL_ENDPOINT_HOST` for chart-generated distributed topologies unless `secret.existingSecret` is set. Use `true` for a credentials-only existing Secret or `false` to preserve legacy DNS locality explicitly. | | commonLabels | object | `{}` | Labels to add to all deployed objects. | | config.rustfs.address | string | `":9000"` | | | config.rustfs.console_address | string | `":9001"` | | @@ -66,7 +121,7 @@ set both `replicaCount` and `drivesPerNode` explicitly. | config.rustfs.obs_environment | string | `"development"` | | | config.rustfs.obs_log_directory | string | `"/logs"` | Log directory inside the RustFS container. Set to `""` to disable log PVCs and mounts. | | config.rustfs.region | string | `"us-east-1"` | | -| config.rustfs.volumes | string | `""` | | +| config.rustfs.volumes | string | `""` | Explicit distributed volume topology. When empty, the chart generates the topology and normally injects `RUSTFS_LOCAL_ENDPOINT_HOST`; custom topologies must configure local endpoint identity explicitly when needed. | | config.rustfs.log_rotation.size | int | `"100"` | Default log rotation size mb for rustfs. | | config.rustfs.log_rotation.time | string | `"hour"` | Default log rotation time for rustfs. | | config.rustfs.log_rotation.keep_files | int | `"30"` | Default log keep files for rustfs. | @@ -107,7 +162,7 @@ set both `replicaCount` and `drivesPerNode` explicitly. | config.rustfs.kms.vault.vault_token | string | `""`| The vault token. Rendered into a dedicated Secret (`-kms-secret`), never into the ConfigMap. | | config.rustfs.kms.vault.vault_mount_path | string | `"transit"`| The vault mount path, only works if `vault_backend` equals `vault-transit` . | | config.rustfs.kms.vault.default_key | string | `"transit"`| The master key id for RustFS. | -| extraEnv | map | `[]` | Extra environment variables for RustFS container. | +| extraEnv | list | `[]` | Extra environment variables for the RustFS container. An explicit `RUSTFS_LOCAL_ENDPOINT_HOST` or `RUSTFS_VOLUMES`, or a bounded, dynamic, or unrecognized startup mode, disables generated anchor injection. `POD_NAME` and `RUSTFS_ADDRESS` remain independent overrides. | | extraVolumes | list | `[]` | Extra volumes to add to the pod spec. Supported in both standalone (Deployment) and distributed (StatefulSet) modes. | | extraVolumeMounts | list | `[]` | Extra volume mounts to add to the RustFS container. Supported in both standalone (Deployment) and distributed (StatefulSet) modes. | | containerSecurityContext.capabilities.drop[0] | string | `"ALL"` | | @@ -190,7 +245,7 @@ uer. `ClusterIssuer` or `Issuer`. | | resources.limits.memory | string | `"512Mi"` | | | resources.requests.cpu | string | `"100m"` | | | resources.requests.memory | string | `"128Mi"` | | -| secret.existingSecret | string | `""` | Use existing secret with a credentials. | +| secret.existingSecret | string | `""` | Use an existing Secret. Automatic endpoint-anchor injection is disabled because the Secret is opaque; set `localEndpointHost.autoInject=true` only after confirming it contains credentials rather than runtime topology, address, or startup-mode overrides. | | secret.rustfs.access_key | string | `"rustfsadmin"` | RustFS Access Key ID | | secret.rustfs.secret_key | string | `"rustfsadmin"` | RustFS Secret Key ID | | service.type | string | `"ClusterIP"` | | @@ -202,6 +257,7 @@ uer. `ClusterIssuer` or `Issuer`. | | serviceAccount.automount | bool | `true` | | | serviceAccount.create | bool | `true` | | | serviceAccount.name | string | `""` | | +| startupWaitTimeoutSeconds | int | `300` | Deprecated and ignored; retained for values-file compatibility. | | storageclass.dataStorageSize | string | `"256Mi"` | The storage size for data PVC. | | storageclass.logStorageSize | string | `"256Mi"` | The storage size for logs PVC. | | storageclass.name | string | `"local-path"` | The name for StorageClass. | @@ -291,20 +347,17 @@ Notes: * **Pools are append-only.** The list index determines the StatefulSet name — never remove or reorder entries. Retire a pool with `rc admin decommission` before removing it from the list. -* During the expansion rollout, pods restart until every pod of every pool is - resolvable — the server refuses to start with unresolvable peers, so expect - a few crash/restart cycles before the cluster converges. This is harmless. +* With chart-generated volumes, each pod receives an explicit local endpoint + identity. An unavailable peer no longer blocks a pod from reaching RustFS's + own startup and quorum checks. * After the cluster converges, run `rc admin rebalance start ` to spread existing objects across the new pool. * Pod anti-affinity in pool mode is scoped per pool and **preferred** (soft), not required: two pools can share nodes, and each pool's own pods - spread across distinct nodes when capacity allows. Soft affinity is - load-bearing for in-place expansion — with required rules, the - not-yet-rolled pods of the existing pool block the new pool's pods from - their nodes while the rolled pods crash on the unresolvable (Pending) - peers, deadlocking the rollout on any cluster with fewer nodes than the - total pod count. Single-pool deployments (`pools.enabled=false`) keep the - chart's existing required anti-affinity unchanged. + spread across distinct nodes when capacity allows. Preferred affinity keeps + additional pools schedulable when the cluster has fewer nodes than total + pods. Single-pool deployments (`pools.enabled=false`) keep the chart's + existing required anti-affinity unchanged. * The PodDisruptionBudget spans all pools: with the default `pdb.maxUnavailable: 1`, at most one pod of the whole cluster may be evicted at a time. This is deliberately conservative — quorum safety @@ -315,7 +368,8 @@ Notes: ## Requirement * Helm V3 -* RustFS >= 1.0.0-alpha.69 +* The RustFS image from the same release as the chart. If `image.rustfs.tag` + is overridden, that image must support `RUSTFS_LOCAL_ENDPOINT_HOST`. Due to the traefik and ingress has different session sticky/affinity annotations, and rustfs support both those two controller, you should specify parameter `ingress.className` to select the right one which suits for you. diff --git a/helm/rustfs/templates/_helpers.tpl b/helm/rustfs/templates/_helpers.tpl index f9156b834..126e07e4f 100644 --- a/helm/rustfs/templates/_helpers.tpl +++ b/helm/rustfs/templates/_helpers.tpl @@ -322,21 +322,23 @@ Render RUSTFS_SERVER_DOMAINS {{- join "," $domains -}} {{- end -}} -{{/* Render probe command for liveness and readiness +{{/* Render an mTLS probe command */}} {{- define "rustfs.probeCommand" -}} -{{- $endpoint_port := .Values.service.endpoint.port | default 9000 -}} -{{- $console_port := .Values.service.console.port | default 9001 -}} +{{- $root := .root -}} +{{- $endpointPath := .endpointPath -}} +{{- $endpoint_port := $root.Values.service.endpoint.port | default 9000 -}} +{{- $console_port := $root.Values.service.console.port | default 9001 -}} {{- $args := "-skf" -}} -{{- if and .Values.mtls.enabled -}} - {{- $args = printf "%s --cert %s --key %s" $args .Values.mtls.clientCertPath .Values.mtls.clientKeyPath -}} +{{- if and $root.Values.mtls.enabled -}} + {{- $args = printf "%s --cert %s --key %s" $args $root.Values.mtls.clientCertPath $root.Values.mtls.clientKeyPath -}} {{- end -}} - /bin/sh - -c - | - curl {{ $args }} https://127.0.0.1:{{ $endpoint_port }}/health/ready && \ + curl {{ $args }} https://127.0.0.1:{{ $endpoint_port }}{{ $endpointPath }} && \ curl {{ $args }} https://127.0.0.1:{{ $console_port }}/rustfs/console/health {{- end -}} @@ -350,7 +352,7 @@ livenessProbe: {{- if .Values.mtls.enabled }} exec: command: -{{ include "rustfs.probeCommand" . | nindent 6 }} +{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health") | nindent 6 }} {{- else }} httpGet: path: /health @@ -369,7 +371,7 @@ readinessProbe: {{- if .Values.mtls.enabled }} exec: command: -{{ include "rustfs.probeCommand" . | nindent 6 }} +{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health/ready") | nindent 6 }} {{- else }} httpGet: path: /health/ready diff --git a/helm/rustfs/templates/statefulset.yaml b/helm/rustfs/templates/statefulset.yaml index 3954ad386..f2c90877d 100644 --- a/helm/rustfs/templates/statefulset.yaml +++ b/helm/rustfs/templates/statefulset.yaml @@ -1,12 +1,37 @@ {{- $logDir := .Values.config.rustfs.obs_log_directory }} {{- $logDirEnabled := ne $logDir "" }} {{- $poolsEnabled := and .Values.pools .Values.pools.enabled }} +{{- $generatedVolumes := empty .Values.config.rustfs.volumes }} +{{- $localEndpointHostAutoInject := empty .Values.secret.existingSecret }} +{{- with .Values.localEndpointHost }} +{{- if and (hasKey . "autoInject") (ne .autoInject nil) }} +{{- if not (kindIs "bool" .autoInject) }} +{{- fail "localEndpointHost.autoInject must be a boolean or null" }} +{{- end }} +{{- $localEndpointHostAutoInject = .autoInject }} +{{- end }} +{{- end }} +{{- $injectLocalEndpointHost := and $generatedVolumes $localEndpointHostAutoInject }} +{{- range $env := .Values.extraEnv -}} +{{- $name := default "" $env.name -}} +{{- if eq $name "RUSTFS_VOLUMES" -}} +{{- $injectLocalEndpointHost = false -}} +{{- else if eq $name "RUSTFS_LOCAL_ENDPOINT_HOST" -}} +{{- $injectLocalEndpointHost = false -}} +{{- else if eq $name "RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" -}} +{{- if hasKey $env "value" -}} +{{- $mode := lower (trim (toString $env.value)) -}} +{{- if and (ne $mode "") (ne $mode "auto") (ne $mode "orchestrated") -}} +{{- $injectLocalEndpointHost = false -}} +{{- end -}} +{{- else -}} +{{- $injectLocalEndpointHost = false -}} +{{- end -}} +{{- end -}} +{{- end -}} {{- if and .Values.mode.distributed.enabled (le (int .Values.replicaCount) 1) -}} {{- fail "Distributed mode requires replicaCount >= 2" -}} {{- end -}} -{{- if and .Values.mode.distributed.enabled (le (int .Values.startupWaitTimeoutSeconds) 0) -}} -{{- fail "startupWaitTimeoutSeconds must be greater than 0 in distributed mode" -}} -{{- end -}} {{- if .Values.mode.distributed.enabled }} {{- $pools := include "rustfs.pools" . | fromJsonArray }} @@ -74,13 +99,10 @@ spec: {{- if $.Values.affinity.podAntiAffinity.enabled }} podAntiAffinity: {{- if $poolsEnabled }} - {{- /* Pool-scoped and PREFERRED, not required: during in-place - expansion the not-yet-rolled pods' required rules block new - pods from their nodes, while the new pods' DNS must resolve - before the roll can proceed - required rules deadlock - expansion whenever the cluster has fewer nodes than total - pods. Soft anti-affinity converges (the scheduler still - spreads when capacity allows). */}} + {{- /* Pool-scoped and PREFERRED, not required: additional pools + must remain schedulable when the cluster has fewer nodes + than total pods. The scheduler still spreads them when + capacity allows. */}} preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: @@ -127,10 +149,6 @@ spec: env: - name: DRIVES_PER_NODE value: {{ $drivesPerNode | quote }} - - name: ENDPOINT_PORT - value: {{ $.Values.service.endpoint.port | quote }} - - name: STARTUP_WAIT_TIMEOUT_SECONDS - value: {{ $.Values.startupWaitTimeoutSeconds | quote }} command: - sh - -c @@ -146,44 +164,6 @@ spec: mkdir -p /mnt/rustfs/logs chmod 755 /mnt/rustfs/logs {{- end }} - deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS)) - wait_for_peer() { - description=$1 - shift - until "$@" >/dev/null 2>&1; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "Timed out after ${STARTUP_WAIT_TIMEOUT_SECONDS}s waiting for ${description}" >&2 - exit 1 - fi - sleep 2 - done - } - - echo "Waiting for distributed peer DNS records" - {{- range $peerPool := $pools }} - {{- range $i := until (int $peerPool.replicaCount) }} - wait_for_peer "DNS for {{ $peerPool.fullname }}-{{ $i }}" \ - nslookup "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" - {{- end }} - {{- end }} - - ordinal=${HOSTNAME##*-} - echo "Waiting for earlier distributed peers to listen on port ${ENDPOINT_PORT}" - {{- range $peerPool := $pools }} - {{- if lt (int $peerPool.index) (int $pool.index) }} - {{- range $i := until (int $peerPool.replicaCount) }} - wait_for_peer "{{ $peerPool.fullname }}-{{ $i }}:${ENDPOINT_PORT}" \ - nc -z -w 2 "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT" - {{- end }} - {{- else if eq (int $peerPool.index) (int $pool.index) }} - i=0 - while [ "$i" -lt "$ordinal" ]; do - wait_for_peer "{{ $peerPool.fullname }}-${i}:${ENDPOINT_PORT}" \ - nc -z -w 2 "{{ $peerPool.fullname }}-${i}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT" - i=$((i + 1)) - done - {{- end }} - {{- end }} volumeMounts: {{- if gt $drivesPerNode 1 }} {{- range $i := until $drivesPerNode }} @@ -210,9 +190,19 @@ spec: containerPort: {{ $.Values.service.endpoint.port }} - name: console containerPort: {{ $.Values.service.console.port }} - {{- with $.Values.extraEnv }} + {{- if or $injectLocalEndpointHost (not (empty $.Values.extraEnv)) }} env: + {{- if $injectLocalEndpointHost }} + - name: RUSTFS_CHART_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: RUSTFS_LOCAL_ENDPOINT_HOST + value: {{ printf "$(RUSTFS_CHART_POD_NAME).%s-headless.%s.svc.%s" (include "rustfs.fullname" $) $.Release.Namespace (include "rustfs.clusterDomain" $) | quote }} + {{- end }} + {{- with $.Values.extraEnv }} {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} envFrom: - configMapRef: diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 3ac64b9e3..7a254b869 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -30,11 +30,22 @@ drivesPerNode: null # Provide the DNS root only, without a `svc.` prefix or leading/trailing dots. clusterDomain: cluster.local -# Maximum time an init container waits for distributed peers to publish DNS -# and for earlier peers to start listening. The wait is shared across all -# peers, not applied once per peer. +# Deprecated and ignored. Retained so existing values files continue to +# render after peer startup coordination moved into RustFS. +# RUSTFS_COMPAT_TODO(rustfs-5416): Keep the removed init-gate setting visible to existing values files. Remove after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders peer gates. startupWaitTimeoutSeconds: 300 +# Inject a pod-specific endpoint identity for chart-generated distributed +# topologies. The default enables injection unless secret.existingSecret is +# set, because the chart cannot inspect runtime overrides hidden in that +# Secret. Without injection, the server infers a unique matching domain +# endpoint from its kernel hostname in Kubernetes auto/orchestrated mode; custom +# aliases retain bounded legacy DNS locality when no hostname matches. Configure +# RUSTFS_LOCAL_ENDPOINT_HOST explicitly for DNS-free startup. Set true for a +# credentials-only existing Secret or false to suppress chart injection. +localEndpointHost: + autoInject: null + # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: rustfs: # This sets the rustfs image repository and tag. @@ -223,7 +234,18 @@ config: default_key: "" -extraEnv: [] # This is for setting extra environment variables in the rustfs container. It should be a list of key value pairs. For example: +# Extra environment variables for the RustFS container. +# In distributed mode with chart-generated volumes (config.rustfs.volumes is +# empty), the chart injects a private Downward API pod name and +# RUSTFS_LOCAL_ENDPOINT_HOST to identify the pod's local endpoints. An explicit +# RUSTFS_LOCAL_ENDPOINT_HOST or RUSTFS_VOLUMES entry disables automatic anchor +# injection. An explicit bounded or fail-fast +# RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE, or an unrecognized static value, also keeps +# legacy DNS-based locality discovery. Otherwise Kubernetes auto-detection +# selects orchestrated startup when RustFS consumes the generated anchor. +# With custom volumes, the chart does not inject them and extraEnv may define +# the local endpoint identity explicitly. +extraEnv: [] # extraEnv: # - name: RUSTFS_ERASURE_SET_DRIVE_COUNT # value: "16" diff --git a/scripts/test_helm_templates.sh b/scripts/test_helm_templates.sh index 4bdccceb4..ecbaa4e83 100755 --- a/scripts/test_helm_templates.sh +++ b/scripts/test_helm_templates.sh @@ -36,6 +36,16 @@ render_distributed_statefulset() { ' } +statefulset_env_names() { + yq eval '[(.spec.template.spec.containers[0].env // [])[] | .name] | join(" ")' - +} + +statefulset_env_value() { + local rustfs_test_env_name=$1 + RUSTFS_TEST_ENV_NAME="$rustfs_test_env_name" \ + yq eval '.spec.template.spec.containers[0].env[] | select(.name == strenv(RUSTFS_TEST_ENV_NAME)) | .value' - +} + render_distributed_configmap() { helm template rustfs "$CHART_DIR" \ --namespace rustfs \ @@ -403,38 +413,308 @@ if grep -q "name: data$" <<<"$generic_eight_by_two"; then exit 1 fi -# Distributed startup coordination must use the configured endpoint port and -# stay transport-neutral so the same wait works with and without mTLS. -custom_port_startup=$(render_distributed_statefulset \ +# Issue #5416's exact three-node, one-drive topology must render all three +# ordinal hosts while each StatefulSet pod receives its own explicit anchor. +three_by_one_configmap=$(render_distributed_configmap --set replicaCount=3 --set drivesPerNode=1) +if ! grep -Fq 'RUSTFS_VOLUMES: "http://rustfs-{0...2}.rustfs-headless.rustfs.svc.cluster.local:9000/data"' <<<"$three_by_one_configmap"; then + echo "Three-node topology must render ordinals 0 through 2 in RUSTFS_VOLUMES" >&2 + exit 1 +fi +# The distributed init container only prepares directories. Peer DNS or TCP +# availability must not gate the RustFS process from applying its own quorum +# checks. The deprecated timeout remains accepted but is not rendered. +distributed_startup=$(render_distributed_statefulset \ + --set replicaCount=3 \ + --set drivesPerNode=1 \ + --set startupWaitTimeoutSeconds=0) +for removed_gate in nslookup 'nc -z' wait_for_peer STARTUP_WAIT_TIMEOUT_SECONDS ENDPOINT_PORT; do + if grep -Fq "$removed_gate" <<<"$distributed_startup"; then + echo "Distributed init must not contain the removed peer gate: $removed_gate" >&2 + exit 1 + fi +done +if ! grep -Fq 'mkdir -p /data/rustfs$i' <<<"$distributed_startup"; then + echo "Distributed init must keep per-drive directory initialization" >&2 + exit 1 +fi +if ! grep -Eq '^[[:space:]]+mkdir -p /data$' <<<"$distributed_startup"; then + echo "Distributed init must keep single-drive directory initialization" >&2 + exit 1 +fi + +# Chart-generated volumes use the Downward API pod name to construct the exact +# endpoint hostname. Order is load-bearing because Kubernetes only expands +# environment references that were defined earlier in the env list. +generated_env_names=$(statefulset_env_names <<<"$distributed_startup") +if [[ "$generated_env_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST" ]]; then + echo "Unexpected generated topology env order: $generated_env_names" >&2 + exit 1 +fi + +pod_name_field=$(yq eval '.spec.template.spec.containers[0].env[] | select(.name == "RUSTFS_CHART_POD_NAME") | .valueFrom.fieldRef.fieldPath' - <<<"$distributed_startup") +local_endpoint_host=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$distributed_startup") +if [[ "$pod_name_field" != "metadata.name" ]]; then + echo "Generated topology must source RUSTFS_CHART_POD_NAME from the Downward API" >&2 + exit 1 +fi +if [[ "$local_endpoint_host" != '$(RUSTFS_CHART_POD_NAME).rustfs-headless.rustfs.svc.cluster.local' ]]; then + echo "Unexpected generated RUSTFS_LOCAL_ENDPOINT_HOST: $local_endpoint_host" >&2 + exit 1 +fi + +# Non-default release identity, namespace, cluster domain, mTLS, and endpoint +# port must compose into one generated topology without restoring peer gates. +nondefault_generated=$(helm template tenant "$CHART_DIR" \ + --namespace object-storage \ + --set secret.rustfs.access_key=test-access-key \ + --set secret.rustfs.secret_key=test-secret-key \ + --set clusterDomain=cluster.corp \ + --set mtls.enabled=true \ --set service.endpoint.port=9443 \ --set config.rustfs.address=:9443) -grep -q 'name: ENDPOINT_PORT' <<<"$custom_port_startup" -grep -A1 'name: ENDPOINT_PORT' <<<"$custom_port_startup" | grep -q 'value: "9443"' -grep -q 'nc -z -w 2' <<<"$custom_port_startup" -if grep -q 'http.*9000/health' <<<"$custom_port_startup"; then - echo "Distributed startup wait must not hardcode the HTTP scheme or port 9000" >&2 +nondefault_statefulset=$(awk ' + /^# Source: rustfs\/templates\/statefulset.yaml$/ { in_statefulset = 1 } + in_statefulset && /^---$/ { exit } + in_statefulset { print } +' <<<"$nondefault_generated") +nondefault_local_host=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$nondefault_statefulset") +nondefault_namespace=$(yq eval '.metadata.namespace' - <<<"$nondefault_statefulset") +nondefault_endpoint_port=$(yq eval '.spec.template.spec.containers[0].ports[] | select(.name == "endpoint") | .containerPort' - <<<"$nondefault_statefulset") +mtls_liveness_command=$(yq eval '.spec.template.spec.containers[0].livenessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") +mtls_readiness_command=$(yq eval '.spec.template.spec.containers[0].readinessProbe.exec.command[-1]' - <<<"$nondefault_statefulset") +if [[ "$nondefault_local_host" != '$(RUSTFS_CHART_POD_NAME).tenant-rustfs-headless.object-storage.svc.cluster.corp' ]]; then + echo "Combined generated topology produced an unexpected local endpoint host: $nondefault_local_host" >&2 + exit 1 +fi +if [[ "$nondefault_namespace" != "object-storage" || "$nondefault_endpoint_port" != "9443" ]]; then + echo "Combined generated topology must retain its namespace and custom endpoint port" >&2 + exit 1 +fi +if ! grep -Fq 'https://127.0.0.1:9443/health' <<<"$mtls_liveness_command" || + grep -Fq '/health/ready' <<<"$mtls_liveness_command"; then + echo "mTLS liveness must use the process health endpoint, not readiness" >&2 + exit 1 +fi +if ! grep -Fq 'https://127.0.0.1:9443/health/ready' <<<"$mtls_readiness_command"; then + echo "mTLS readiness must use the ready endpoint" >&2 + exit 1 +fi +if ! grep -Fq 'RUSTFS_VOLUMES: "https://tenant-rustfs-{0...3}.tenant-rustfs-headless.object-storage.svc.cluster.corp:9443/data/rustfs{0...3}"' <<<"$nondefault_generated"; then + echo "Combined generated topology must use its release fullname, namespace, domain, mTLS scheme, and endpoint port" >&2 + exit 1 +fi +if grep -Eq 'nslookup|nc -z|wait_for_peer' <<<"$nondefault_generated"; then + echo "Combined generated topology must not render peer DNS or TCP gates" >&2 exit 1 fi -mtls_startup=$(render_distributed_statefulset --set mtls.enabled=true) -grep -q 'nc -z -w 2' <<<"$mtls_startup" -if grep -q 'wget .*http.*health' <<<"$mtls_startup"; then - echo "mTLS startup wait must not use a plaintext HTTP health probe" >&2 +default_scheme_port_generated=$(render_distributed_configmap \ + --set service.endpoint.port=80 \ + --set config.rustfs.address=:80) +if ! grep -Fq 'rustfs-headless.rustfs.svc.cluster.local:80/' <<<"$default_scheme_port_generated"; then + echo "Chart-generated topology must retain an explicit HTTP default port" >&2 exit 1 fi -# The timeout is one global deadline and invalid non-positive values fail -# during rendering instead of creating an unbounded init wait. -grep -q 'deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS))' <<<"$custom_port_startup" -invalid_startup_timeout_status=0 -render_distributed_statefulset --set startupWaitTimeoutSeconds=0 >/dev/null 2>&1 || invalid_startup_timeout_status=$? -if [[ $invalid_startup_timeout_status -eq 0 ]]; then - echo "Distributed mode with startupWaitTimeoutSeconds=0 must fail rendering" >&2 +# Unrelated extraEnv entries follow the generated entries so dependent +# environment expansion keeps working. +generated_with_extra_env=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=CUSTOM_ENV' \ + --set 'extraEnv[0].value=custom-value') +generated_with_extra_names=$(statefulset_env_names <<<"$generated_with_extra_env") +if [[ "$generated_with_extra_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST CUSTOM_ENV" ]]; then + echo "extraEnv must be appended after generated topology env entries" >&2 exit 1 fi -# Every pool waits for DNS across the complete normalized pool list, while -# port availability follows the stable pool-index/ordinal startup order. +# Explicit bounded/fail-fast modes retain their legacy DNS locality semantics, +# so the generated anchor must not make the runtime reject the configuration. +for legacy_wait_mode in bounded ' Strict '; do + bounded_wait_mode=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE' \ + --set-string "extraEnv[0].value=$legacy_wait_mode") + bounded_wait_env_names=$(statefulset_env_names <<<"$bounded_wait_mode") + bounded_wait_value=$(yq eval '.spec.template.spec.containers[0].env[0].value' - <<<"$bounded_wait_mode") + if [[ "$bounded_wait_env_names" != "RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" || "$bounded_wait_value" != "$legacy_wait_mode" ]]; then + echo "Startup mode $legacy_wait_mode must opt out of the generated local endpoint anchor" >&2 + exit 1 + fi +done + +unknown_wait_mode=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE' \ + --set 'extraEnv[0].value=invalid-mode') +unknown_wait_env_names=$(statefulset_env_names <<<"$unknown_wait_mode") +if [[ "$unknown_wait_env_names" != "RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" ]]; then + echo "An unknown explicit startup mode must not receive a generated local endpoint anchor" >&2 + exit 1 +fi + +dynamic_wait_mode=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE' \ + --set 'extraEnv[0].valueFrom.configMapKeyRef.name=startup-policy' \ + --set 'extraEnv[0].valueFrom.configMapKeyRef.key=mode') +dynamic_wait_env_names=$(statefulset_env_names <<<"$dynamic_wait_mode") +if [[ "$dynamic_wait_env_names" != "RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" ]]; then + echo "Dynamically sourced startup mode must opt out of the generated local endpoint anchor" >&2 + exit 1 +fi + +orchestrated_wait_mode=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE' \ + --set 'extraEnv[0].value=orchestrated') +orchestrated_wait_env_names=$(statefulset_env_names <<<"$orchestrated_wait_mode") +if [[ "$orchestrated_wait_env_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" ]]; then + echo "Explicit orchestrated startup mode must retain one generated local endpoint anchor" >&2 + exit 1 +fi + +# Existing POD_NAME remains independent from the chart-private Downward API +# variable, while an explicit anchor disables automatic anchor injection. +existing_pod_name=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=POD_NAME' \ + --set 'extraEnv[0].valueFrom.fieldRef.fieldPath=metadata.name') +existing_pod_name_names=$(statefulset_env_names <<<"$existing_pod_name") +existing_pod_name_count=$(yq eval '[.spec.template.spec.containers[0].env[] | select(.name == "POD_NAME")] | length' - <<<"$existing_pod_name") +existing_pod_name_field=$(yq eval '.spec.template.spec.containers[0].env[] | select(.name == "POD_NAME") | .valueFrom.fieldRef.fieldPath' - <<<"$existing_pod_name") +existing_chart_pod_name_field=$(yq eval '.spec.template.spec.containers[0].env[] | select(.name == "RUSTFS_CHART_POD_NAME") | .valueFrom.fieldRef.fieldPath' - <<<"$existing_pod_name") +existing_pod_name_anchor=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$existing_pod_name") +if [[ "$existing_pod_name_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST POD_NAME" || + "$existing_pod_name_count" != "1" || + "$existing_pod_name_field" != "metadata.name" || + "$existing_chart_pod_name_field" != "metadata.name" || + "$existing_pod_name_anchor" != '$(RUSTFS_CHART_POD_NAME).rustfs-headless.rustfs.svc.cluster.local' ]]; then + echo "An existing extraEnv POD_NAME must be preserved without interfering with the automatic anchor" >&2 + exit 1 +fi + +existing_local_anchor=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_LOCAL_ENDPOINT_HOST' \ + --set-string 'extraEnv[0].value=rustfs-0.rustfs-headless.rustfs.svc.cluster.local') +existing_local_anchor_names=$(statefulset_env_names <<<"$existing_local_anchor") +existing_local_anchor_value=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$existing_local_anchor") +if [[ "$existing_local_anchor_names" != "RUSTFS_LOCAL_ENDPOINT_HOST" || "$existing_local_anchor_value" != "rustfs-0.rustfs-headless.rustfs.svc.cluster.local" ]]; then + echo "An existing explicit local endpoint anchor must be preserved without a generated duplicate" >&2 + exit 1 +fi + +# `helm upgrade --reuse-values` can omit keys introduced by the new chart. +# A missing localEndpointHost map keeps the new safe default instead of +# failing template evaluation. +missing_local_endpoint_host_values=$(render_distributed_statefulset \ + --set-json 'localEndpointHost=null') +missing_local_endpoint_host_names=$(statefulset_env_names <<<"$missing_local_endpoint_host_values") +if [[ "$missing_local_endpoint_host_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST" ]]; then + echo "Missing localEndpointHost values from a reused release must default to automatic anchor injection" >&2 + exit 1 +fi + +# An opaque existing Secret keeps the historical DNS path by default because +# it may hide runtime topology or startup-mode overrides. A credentials-only +# Secret can explicitly opt in without changing envFrom precedence. +existing_secret_statefulset=$(render_distributed_statefulset \ + --set secret.existingSecret=legacy-rustfs-config) +existing_secret_env_names=$(statefulset_env_names <<<"$existing_secret_statefulset") +existing_secret_env_from=$(yq eval '[.spec.template.spec.containers[0].envFrom[] | (.configMapRef.name // .secretRef.name)] | join(" ")' - <<<"$existing_secret_statefulset") +if [[ -n "$existing_secret_env_names" || "$existing_secret_env_from" != "rustfs-config legacy-rustfs-config" ]]; then + echo "An opaque existingSecret must default to the legacy environment path" >&2 + exit 1 +fi + +existing_secret_opt_in=$(render_distributed_statefulset \ + --set secret.existingSecret=legacy-rustfs-config \ + --set localEndpointHost.autoInject=true) +existing_secret_opt_in_names=$(statefulset_env_names <<<"$existing_secret_opt_in") +existing_secret_opt_in_env_from=$(yq eval '[.spec.template.spec.containers[0].envFrom[] | (.configMapRef.name // .secretRef.name)] | join(" ")' - <<<"$existing_secret_opt_in") +if [[ "$existing_secret_opt_in_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST" || + "$existing_secret_opt_in_env_from" != "rustfs-config legacy-rustfs-config" ]]; then + echo "A credentials-only existingSecret must support explicit automatic anchor injection" >&2 + exit 1 +fi + +# A Secret that hides a custom topology can explicitly preserve the historical +# DNS path. No explicit env entries are added, and envFrom ordering is unchanged. +existing_secret_opt_out=$(render_distributed_statefulset \ + --set secret.existingSecret=legacy-rustfs-config \ + --set localEndpointHost.autoInject=false) +existing_secret_opt_out_env_names=$(statefulset_env_names <<<"$existing_secret_opt_out") +existing_secret_opt_out_env_from=$(yq eval '[.spec.template.spec.containers[0].envFrom[] | (.configMapRef.name // .secretRef.name)] | join(" ")' - <<<"$existing_secret_opt_out") +if [[ -n "$existing_secret_opt_out_env_names" || "$existing_secret_opt_out_env_from" != "rustfs-config legacy-rustfs-config" ]]; then + echo "localEndpointHost.autoInject=false must preserve the legacy existingSecret environment path" >&2 + exit 1 +fi + +quoted_local_endpoint_host_status=0 +quoted_local_endpoint_host_error=$(render_distributed_statefulset \ + --set secret.existingSecret=legacy-rustfs-config \ + --set-string localEndpointHost.autoInject=false 2>&1) || quoted_local_endpoint_host_status=$? +if [[ $quoted_local_endpoint_host_status -eq 0 || + "$quoted_local_endpoint_host_error" != *"localEndpointHost.autoInject must be a boolean or null"* ]]; then + echo "A quoted localEndpointHost.autoInject value must fail closed instead of becoming truthy" >&2 + exit 1 +fi + +missing_local_endpoint_host_with_secret=$(render_distributed_statefulset \ + --set secret.existingSecret=legacy-rustfs-config \ + --set-json 'localEndpointHost=null') +if [[ -n "$(statefulset_env_names <<<"$missing_local_endpoint_host_with_secret")" ]]; then + echo "A reused release without localEndpointHost values must not override an opaque existingSecret" >&2 + exit 1 +fi + +# An explicit topology disables inference, but an address override alone keeps +# the generated anchor so RustFS can validate the effective port before format +# initialization. +volumes_override_statefulset=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_VOLUMES' \ + --set-string 'extraEnv[0].value=http://legacy.example/data') +volumes_override_names=$(statefulset_env_names <<<"$volumes_override_statefulset") +volumes_override_value=$(yq eval '.spec.template.spec.containers[0].env[0].value' - <<<"$volumes_override_statefulset") +if [[ "$volumes_override_names" != "RUSTFS_VOLUMES" || "$volumes_override_value" != "http://legacy.example/data" ]]; then + echo "extraEnv RUSTFS_VOLUMES must retain precedence without a generated anchor" >&2 + exit 1 +fi + +address_override_statefulset=$(render_distributed_statefulset \ + --set 'extraEnv[0].name=RUSTFS_ADDRESS' \ + --set-string 'extraEnv[0].value=:9443') +address_override_names=$(statefulset_env_names <<<"$address_override_statefulset") +address_override_value=$(statefulset_env_value RUSTFS_ADDRESS <<<"$address_override_statefulset") +address_override_anchor_count=$(yq eval '[.spec.template.spec.containers[0].env[] | select(.name == "RUSTFS_LOCAL_ENDPOINT_HOST")] | length' - <<<"$address_override_statefulset") +if [[ "$address_override_names" != "RUSTFS_CHART_POD_NAME RUSTFS_LOCAL_ENDPOINT_HOST RUSTFS_ADDRESS" || + "$address_override_value" != ":9443" || + "$address_override_anchor_count" != "1" ]]; then + echo "extraEnv RUSTFS_ADDRESS must retain one generated local endpoint anchor" >&2 + exit 1 +fi + +# Explicit volumes remain authoritative: the chart must not infer any identity, +# while a user-supplied local endpoint anchor remains allowed. +custom_volumes_statefulset=$(render_distributed_statefulset \ + --set config.rustfs.volumes=http://example.test/data) +custom_volumes_env_names=$(statefulset_env_names <<<"$custom_volumes_statefulset") +if [[ -n "$custom_volumes_env_names" ]]; then + echo "Custom volumes must not receive generated topology env entries" >&2 + exit 1 +fi + +custom_volumes_with_anchor=$(render_distributed_statefulset \ + --set config.rustfs.volumes=http://example.test/data \ + --set 'extraEnv[0].name=RUSTFS_LOCAL_ENDPOINT_HOST' \ + --set 'extraEnv[0].value=example.test') +custom_anchor=$(statefulset_env_value RUSTFS_LOCAL_ENDPOINT_HOST <<<"$custom_volumes_with_anchor") +if [[ "$custom_anchor" != "example.test" ]]; then + echo "Custom volumes must allow a user-supplied local endpoint anchor" >&2 + exit 1 +fi + +if grep -q 'RUSTFS_LOCAL_ENDPOINT_HOST' <<<"$standalone_default_output"; then + echo "Standalone mode must not receive a distributed local endpoint anchor" >&2 + exit 1 +fi + +# Every generated server pool uses its runtime pod name with the shared root +# headless service. No pool waits for another pool's DNS or TCP listener. multi_pool_startup=$(helm template rustfs "$CHART_DIR" \ --namespace rustfs \ --set secret.rustfs.access_key=test-access-key \ @@ -442,14 +722,19 @@ multi_pool_startup=$(helm template rustfs "$CHART_DIR" \ --set pools.enabled=true \ --set 'pools.list[0].replicaCount=2' \ --set 'pools.list[1].replicaCount=2') -grep -q 'nslookup "rustfs-0.rustfs-headless.rustfs.svc.cluster.local"' <<<"$multi_pool_startup" -grep -q 'nslookup "rustfs-pool1-1.rustfs-headless.rustfs.svc.cluster.local"' <<<"$multi_pool_startup" -if grep -q 'rustfs-pool1-headless' <<<"$multi_pool_startup"; then - echo "Multi-pool startup wait must use the shared root headless service" >&2 +multi_pool_anchor_count=$(grep -c 'name: RUSTFS_LOCAL_ENDPOINT_HOST' <<<"$multi_pool_startup" || true) +if [[ $multi_pool_anchor_count -ne 2 ]]; then + echo "Every generated server pool must receive RUSTFS_LOCAL_ENDPOINT_HOST" >&2 + exit 1 +fi +if grep -Eq 'nslookup|nc -z|wait_for_peer' <<<"$multi_pool_startup"; then + echo "Multi-pool init must not wait for peer DNS or TCP" >&2 + exit 1 +fi +if grep -q 'rustfs-pool1-headless' <<<"$multi_pool_startup"; then + echo "Multi-pool local endpoint identity must use the shared root headless service" >&2 exit 1 fi -grep -q 'wait_for_peer "rustfs-1:${ENDPOINT_PORT}"' <<<"$multi_pool_startup" -grep -q 'wait_for_peer "rustfs-pool1-${i}:${ENDPOINT_PORT}"' <<<"$multi_pool_startup" # volumeClaimTemplates must not contain empty annotations when pvcAnnotations are unset, # because Kubernetes treats annotations: {} as a mutation of the immutable field. From b83c9c4663552da5b5359022107b10c9afdd3bea Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 11:28:48 +0800 Subject: [PATCH 07/52] ci(release): isolate preview releases from latest channels (#5462) --- .../skills/rustfs-release-publish/SKILL.md | 92 ++++++-- .../rustfs-release-version-bump/SKILL.md | 2 +- .github/workflows/audit.yml | 5 + .github/workflows/build.yml | 78 +++---- .github/workflows/docker.yml | 10 +- .github/workflows/helm-package.yml | 5 +- .../check_preview_release_workflow.sh | 205 ++++++++++++++++++ 7 files changed, 339 insertions(+), 58 deletions(-) create mode 100755 scripts/security/check_preview_release_workflow.sh diff --git a/.agents/skills/rustfs-release-publish/SKILL.md b/.agents/skills/rustfs-release-publish/SKILL.md index 1c64897c8..9c246398d 100644 --- a/.agents/skills/rustfs-release-publish/SKILL.md +++ b/.agents/skills/rustfs-release-publish/SKILL.md @@ -1,19 +1,21 @@ --- name: rustfs-release-publish -description: "End-to-end RustFS release pipeline: bump version files on main directly to the final target version, cut a preview tag on that commit, verify the CI build and release artifacts, run the downloaded binary locally and exercise the console, validate the server with the latest rc client, then publish the final tag on the SAME validated commit — never a new bump commit, never latest main. Use whenever the user wants to release/publish a RustFS version (发版/发布)." +description: "End-to-end RustFS release pipeline: first publish any merged-but-unreleased rustfs/console changes and wait for its latest Release asset, then bump RustFS version files on main directly to the final target, publish a visible GitHub prerelease from a preview tag without updating latest channels, validate it, and publish the final tag on the SAME commit. Use whenever the user wants to release/publish a RustFS version (发版/发布)." --- # RustFS Release Publish (preview-validated pipeline) This skill orchestrates a full release. It wraps `rustfs-release-version-bump` (which only edits version files and opens the PR) with a mandatory preview-tag validation loop before the final tag is published. -Core design: **version files never carry a `-preview.N` suffix**. The preview suffix exists only in tag names. This works because the binary self-reports the git tag it was built from (`build::TAG` via shadow_rs, see `rustfs/src/config/cli.rs` `SHORT_VERSION`), and `build.yml` derives artifact names and prerelease classification from the tag name — Cargo.toml's version is only a no-tag fallback. Therefore the preview tag and the final tag can (and MUST) point at the exact same commit: what you validated is byte-for-byte the source that ships. +Core design: **version files never carry a `-preview.N` suffix**. The preview suffix exists only in tag names. A preview tag creates a visible GitHub Release marked Prerelease and uploads versioned assets, but it never becomes GitHub Latest and never updates `*-latest`, `latest.json`, R2, Docker, or Helm channels. This works because the binary self-reports the git tag it was built from (`build::TAG` via shadow_rs, see `rustfs/src/config/cli.rs` `SHORT_VERSION`), and `build.yml` derives artifact names and preview classification from the tag name — Cargo.toml's version is only a no-tag fallback. Therefore the preview tag and the final tag can (and MUST) point at the exact same commit: what you validated is byte-for-byte the source that ships. Pipeline shape: ``` -bump version files to (final version, ONE commit) -> merge +check console main against its latest Release + -> if ahead: publish console -> wait for Release asset + latest API + -> bump RustFS version files to (final version, ONE commit) -> merge -> tag at that commit -> CI green - -> verify release artifacts -> run binary locally + console checks + -> verify preview Release assets -> run binary locally + console checks -> validate with latest rc client -> tag at the SAME commit (zero delta) -> re-verify CI/release ``` @@ -23,7 +25,7 @@ On validation failure: fix lands on main via normal PR (version files are alread ## Required inputs - Final target version, for example `1.0.0-beta.10`. -- Preview iteration `N` (default: next unused preview tag for that target; check with `git tag -l '-preview.*'` — and for stable targets `git tag -l '-rc.*'` — after `git fetch --tags`). +- Preview iteration `N` (default: next unused preview tag for that target; check with `git tag -l '-preview.*'` after `git fetch --tags`). If the target version is missing or ambiguous, stop and ask before doing anything (see the semver gate below). @@ -45,12 +47,14 @@ Rules: ## Preview tag naming -- Prerelease target (contains `alpha`/`beta`/`rc`): preview tag is `-preview.N`, e.g. `1.0.0-beta.10-preview.3`. It contains `beta`, so `build.yml`'s substring-based classification marks it prerelease — safe. -- **Stable** target (e.g. `1.1.0`): NEVER tag `1.1.0-preview.N` — `build.yml` marks a tag prerelease only if its name contains `alpha`, `beta`, or `rc`, so `1.1.0-preview.N` would be treated as a stable release and overwrite `latest.json` as stable. Use `1.1.0-rc.N` as the preview tag instead. +- Use `-preview.N` for every target, e.g. `1.0.0-beta.10-preview.3` or `1.1.0-preview.1`. +- The canonical suffix is exactly `-preview.`. `build.yml` recognizes it before alpha/beta/rc classification and routes it to the preview-only path; any other tag containing `-preview` fails closed instead of being treated as a release. +- A preview Release MUST be published with `isPrerelease=true` and `isLatest=false`. Any `*-latest` preview asset or preview-triggered `latest.json`, R2, Docker, or Helm publication is a pipeline failure. ## Hard rules - Version files (Cargo.toml, Cargo.lock, README, flake.nix, Chart.yaml, rustfs.spec) are bumped ONCE, directly to ``. Never write a `-preview.N` suffix into any version file. If `rustfs-release-version-bump` is ever asked for a `-preview` version, that is a pipeline bug — stop. +- Preview Release assets are versioned and intentionally visible on the Releases page. Do not label them Latest or use them to update any latest distribution channel. - Tags have no `v` prefix. Always annotated: `git tag -a -m "Release "`. - The final tag MUST point at exactly `PREVIEW_HASH` — the commit the validated preview tag points at. Never tag current `main` HEAD (commits merged after validation are unvalidated), and never create an extra version-bump commit between preview and final. - Phases run in order; a failure in any phase blocks everything after it. After the fix lands on main, restart from Phase 2 with the next preview iteration against the new `origin/main` hash — do not resume mid-pipeline against a stale hash. @@ -63,6 +67,61 @@ Rules: - `gh auth status` works; confirm you can view `gh release list -L 3`. - Confirm the exact final target version with the user if not explicit. +### Console release gate + +Complete this gate before changing any RustFS version file or creating any RustFS tag. RustFS `build.yml` downloads the asset returned by `repos/rustfs/console/releases/latest`, so a successful Console build alone is insufficient. + +1. Read the latest published Console tag and compare it with Console `main`: + +```bash +CONSOLE_REPO="rustfs/console" +CONSOLE_LATEST=$(gh api "repos/${CONSOLE_REPO}/releases/latest" --jq .tag_name) +gh api "repos/${CONSOLE_REPO}/compare/${CONSOLE_LATEST}...main" \ + --jq '{status, ahead_by, behind_by, commits: [.commits[] | {sha, message: .commit.message}]}' +``` + +- `ahead_by == 0`: no merged Console change is waiting for release. Still verify the current latest asset using step 4, then continue to Phase 1. +- `ahead_by > 0` and `behind_by == 0`: publish Console before continuing. Report the merged commits and select the next unused `vX.Y.Z` tag. Default to the next patch version when the changes are fixes or backward-compatible UI work; stop for confirmation if a minor/major bump is plausible. +- Any diverged history or `behind_by > 0`: stop and resolve the Console release baseline explicitly. Do not guess a range or publish RustFS. + +2. Clone/fetch `rustfs/console` into a scratch directory and record its exact `main` commit. Before creating a tag, check for a `v*` tag or Release workflow already associated with that hash. If one is in progress, wait for it instead of creating another version: + +```bash +CONSOLE_SCRATCH=$(mktemp -d) +gh repo clone "$CONSOLE_REPO" "$CONSOLE_SCRATCH/console" +git -C "$CONSOLE_SCRATCH/console" fetch origin main --tags +CONSOLE_HASH=$(git -C "$CONSOLE_SCRATCH/console" rev-parse origin/main) +git -C "$CONSOLE_SCRATCH/console" tag --points-at "$CONSOLE_HASH" 'v*' +gh run list -R "$CONSOLE_REPO" --workflow release.yml --commit "$CONSOLE_HASH" --limit 5 +``` + +If no release exists or is running for `CONSOLE_HASH`, create the selected annotated tag at that exact hash and push it: + +```bash +git -C "$CONSOLE_SCRATCH/console" tag -a "" -m "Release " "$CONSOLE_HASH" +git -C "$CONSOLE_SCRATCH/console" push origin "" +``` + +Console tags include the `v` prefix. Pushing the tag triggers `.github/workflows/release.yml` (`🚀 Release`). Remove `CONSOLE_SCRATCH` after the gate completes. + +3. Find the exact tag run and wait for completion: + +```bash +gh run list -R "$CONSOLE_REPO" --workflow release.yml --branch "" --limit 1 +gh run watch -R "$CONSOLE_REPO" "" --exit-status +``` + +4. Block until the published Release is non-draft, the latest endpoint returns the expected tag, and `rustfs-console-.zip` is uploaded, non-empty, and carries a `sha256:` digest: + +```bash +gh release view -R "$CONSOLE_REPO" "" --json isDraft,isPrerelease,assets,url +test "$(gh api "repos/${CONSOLE_REPO}/releases/latest" --jq .tag_name)" = "" +test "$(gh api "repos/${CONSOLE_REPO}/releases/tags/" \ + --jq '[.assets[] | select(.name == "rustfs-console-.zip" and .state == "uploaded" and .size > 0 and (.digest | startswith("sha256:")))] | length')" -eq 1 +``` + +Treat a missing/mismatched asset, digest, latest tag, or failed/cancelled workflow as BLOCKED. Do not start Phase 1 until the Console gate passes. Record `CONSOLE_TAG`, `CONSOLE_HASH`, Console run URL, and Release URL for the final report. + ## Phase 1 — Version bump to the final target (once) - If main's version files already read `` (e.g. this is a restart after a failed preview), verify with `rg -n "" Cargo.toml rustfs.spec helm/rustfs/Chart.yaml` and skip to Phase 2. @@ -85,16 +144,16 @@ git push origin "" Pushing the tag triggers `.github/workflows/build.yml` ("Build and Release"); `docker.yml` chains off it via `workflow_run`. +The preview run builds versioned artifacts and publishes them in a GitHub prerelease. Its latest-channel, R2, Docker, and Helm jobs must be skipped. Those publication paths run only after the final tag is pushed. + On a restart (N+1), refresh `PREVIEW_HASH=$(git rev-parse origin/main)` first — it must contain the fix — and re-report it. -## Phase 3 — CI and artifact verification +## Phase 3 — CI and preview Release verification -- Watch the tag build: `gh run list --workflow build.yml --limit 5` then `gh run watch `. Every matrix target must succeed (linux x86_64/aarch64 × musl/gnu, macos-aarch64, windows-x86_64) plus the release and latest.json jobs. -- Verify the GitHub release: `gh release view "" --json isPrerelease,assets` - - `isPrerelease` must be `true`. - - Assets must include all 6 platform zips in both versioned (`rustfs--v.zip`) and `-latest` forms, plus `SHA256SUMS`, `SHA512SUMS`, `rustfs-.sbom.cdx.json`, `rustfs-.provenance.json`. -- Verify the chained Docker run succeeded: `gh run list --workflow docker.yml --limit 3`. -- Checksum spot-check for the platform you will run locally: download the zip and `SHA256SUMS`, verify with `shasum -a 256 -c` (grep to one line). +- Find and watch the tag build: `gh run list --workflow build.yml --branch "" --limit 1` then `gh run watch `. Every build matrix target must succeed (linux x86_64/aarch64 × musl/gnu, macos-aarch64, windows-x86_64). +- Confirm the Release publication jobs (`create-release`, `upload-release-assets`, and `publish-release`) succeed while `update-latest-version` is skipped. +- Verify `gh release view "" --json isPrerelease,assets,url`: `isPrerelease` must be `true`, and the Release must contain all 6 versioned platform zips, checksums, SBOM, and provenance with no `-latest` assets. Confirm `gh api repos/{owner}/{repo}/releases/latest --jq .tag_name` does not return ``. +- Confirm preview-triggered Docker and Helm jobs are skipped. Preview validation covers the built RustFS binaries, embedded console, and rc compatibility; Docker image construction and Helm publication are deferred to the final tag because the Dockerfiles consume GitHub Release assets. ## Phase 4 — Run the artifact locally, verify the console @@ -157,13 +216,14 @@ git push origin "" ``` - CI rebuilds from the same source; the only changed input is the tag name, so the binary now self-reports ``. -- Re-run the Phase 3 verification against the final tag: all matrix jobs green; `gh release view ""` shows the full asset set; for a prerelease target `isPrerelease` is `true`, for a stable target it must be `false` and `latest.json` must be updated. +- Verify the final tag's complete publication path: all matrix and release jobs green; `gh release view ""` shows the full versioned and `-latest` asset set plus checksums, SBOM, and provenance; Docker and Helm workflows succeed; `latest.json` points to ``. A stable target must have `isPrerelease=false` and `isLatest=true`. An alpha/beta/rc target must have `isPrerelease=true`; GitHub does not permit prereleases to be Latest, but the project `latest.json` still advances to the final non-preview target. - Optionally spot-check `./rustfs --version` from a final-tag artifact — it must report ``. ## Output contract Always report: +- Console gate result: previous/latest Console tags, whether merged changes required a release, `CONSOLE_HASH`, and Console run/Release URLs when a release was published. - Target version, preview tag(s) used, `PREVIEW_HASH` (which both tags point at). -- Per-phase result (PASS/FAIL/BLOCKED) with key evidence: CI run URLs, release URLs, console check results, the rc command matrix. +- Per-phase result (PASS/FAIL/BLOCKED) with key evidence: preview and final Release URLs, preview `isPrerelease`/`isLatest` state, final latest-channel state, console check results, and the rc command matrix. - Any deviation from this pipeline and why the user approved it. diff --git a/.agents/skills/rustfs-release-version-bump/SKILL.md b/.agents/skills/rustfs-release-version-bump/SKILL.md index 0e39ce737..3c760f409 100644 --- a/.agents/skills/rustfs-release-version-bump/SKILL.md +++ b/.agents/skills/rustfs-release-version-bump/SKILL.md @@ -18,7 +18,7 @@ Validated baseline: release pattern used in PR `#2957`. If target version is missing or ambiguous, stop and ask before editing. -Reject any target version containing a `-preview.` suffix: preview identifiers are tag-only (see `rustfs-release-publish`) and must never be written into version files. If asked for one, stop and point to the release pipeline instead of editing. +Reject any target version containing `-preview`: preview identifiers are tag-only (see `rustfs-release-publish`) and must never be written into version files. If asked for one, stop and point to the release pipeline instead of editing. ## Read before editing diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index c630abc8b..f33240860 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -23,6 +23,7 @@ on: - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' + - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_workflow_pins.sh' pull_request: types: [ opened, synchronize, reopened, closed ] @@ -33,6 +34,7 @@ on: - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' + - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_workflow_pins.sh' schedule: - cron: '0 3 * * 0' # Weekly on Sunday 03:00 UTC (staggered after the midnight ci/build crons) @@ -96,6 +98,9 @@ jobs: - name: Report unpinned GitHub Actions run: ./scripts/security/check_workflow_pins.sh --enforce + - name: Check preview release workflow policy + run: ./scripts/security/check_preview_release_workflow.sh + dependency-review: name: Dependency Review runs-on: ubuntu-latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 331221545..2274763d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -107,13 +107,21 @@ jobs: # Determine build type based on trigger if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then - # Tag push - release or prerelease + # Tag push - preview, release, or prerelease should_build=true tag_name="${GITHUB_REF#refs/tags/}" version="${tag_name}" - # Check if this is a prerelease - if [[ "$tag_name" == *"alpha"* ]] || [[ "$tag_name" == *"beta"* ]] || [[ "$tag_name" == *"rc"* ]]; then + # Preview tags publish a GitHub prerelease for validation, but + # must not update any latest channel. + if [[ "$tag_name" =~ -preview\.[0-9]+$ ]]; then + build_type="preview" + is_prerelease=true + echo "🔍 Preview build detected: $tag_name" + elif [[ "$tag_name" == *"-preview"* ]]; then + echo "❌ Invalid preview tag: $tag_name (expected suffix: -preview.)" >&2 + exit 1 + elif [[ "$tag_name" == *"alpha"* ]] || [[ "$tag_name" == *"beta"* ]] || [[ "$tag_name" == *"rc"* ]]; then build_type="prerelease" is_prerelease=true echo "🚀 Prerelease build detected: $tag_name" @@ -714,6 +722,10 @@ jobs: echo "" case "$BUILD_TYPE" in + "preview") + echo "🔍 Preview artifacts are published in a GitHub prerelease" + echo "⏭️ Preview releases do not update latest channels" + ;; "development") echo "🛠️ Development build artifacts have been uploaded to OSS dev directory" echo "⚠️ This is a development build - not suitable for production use" @@ -732,7 +744,9 @@ jobs: echo "" echo "🐳 Docker Images:" - if [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then + if [[ "$BUILD_TYPE" == "preview" ]]; then + echo "⏭️ Preview tags do not publish Docker images" + elif [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then echo "⏭️ Docker image build was skipped (binary only build)" elif [[ "$BUILD_STATUS" == "success" ]]; then echo "🔄 Docker images will be built and pushed automatically via workflow_run event" @@ -740,11 +754,11 @@ jobs: echo "❌ Docker image build will be skipped due to build failure" fi - # Create GitHub Release (only for tag pushes) + # Create GitHub Release for every valid release tag, including previews create-release: name: Create GitHub Release needs: [ build-check, build-rustfs ] - if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development' + if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest permissions: contents: write @@ -769,7 +783,9 @@ jobs: BUILD_TYPE="${{ needs.build-check.outputs.build_type }}" # Determine release type for title - if [[ "$BUILD_TYPE" == "prerelease" ]]; then + if [[ "$BUILD_TYPE" == "preview" ]]; then + RELEASE_TYPE="preview" + elif [[ "$BUILD_TYPE" == "prerelease" ]]; then if [[ "$TAG" == *"alpha"* ]]; then RELEASE_TYPE="alpha" elif [[ "$TAG" == *"beta"* ]]; then @@ -816,6 +832,7 @@ jobs: --title "$TITLE" \ --notes "$RELEASE_NOTES" \ $PRERELEASE_FLAG \ + --latest=false \ --draft RELEASE_ID=$(gh release view "$TAG" --json databaseId --jq '.databaseId') @@ -830,7 +847,7 @@ jobs: upload-release-assets: name: Upload Release Assets needs: [ build-check, build-rustfs, create-release ] - if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development' + if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest permissions: contents: write @@ -920,8 +937,8 @@ jobs: # the pointed-to version is a prerelease. update-latest-version: name: Update Latest Version - needs: [ build-check, upload-release-assets ] - if: startsWith(github.ref, 'refs/tags/') + needs: [ build-check, publish-release ] + if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest steps: - name: Update latest.json @@ -980,7 +997,7 @@ jobs: publish-release: name: Publish Release needs: [ build-check, create-release, upload-release-assets ] - if: startsWith(github.ref, 'refs/tags/') && needs.build-check.outputs.build_type != 'development' + if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest permissions: contents: write @@ -994,37 +1011,22 @@ jobs: shell: bash run: | TAG="${{ needs.build-check.outputs.version }}" - VERSION="${{ needs.build-check.outputs.version }}" - IS_PRERELEASE="${{ needs.build-check.outputs.is_prerelease }}" BUILD_TYPE="${{ needs.build-check.outputs.build_type }}" + RELEASE_ID="${{ needs.create-release.outputs.release_id }}" - # Determine release type - if [[ "$BUILD_TYPE" == "prerelease" ]]; then - if [[ "$TAG" == *"alpha"* ]]; then - RELEASE_TYPE="alpha" - elif [[ "$TAG" == *"beta"* ]]; then - RELEASE_TYPE="beta" - elif [[ "$TAG" == *"rc"* ]]; then - RELEASE_TYPE="rc" - else - RELEASE_TYPE="prerelease" - fi + # Publish the release and correct its channel state on retries. + # Only a stable final release may become GitHub Latest. + if [[ "$BUILD_TYPE" == "release" ]]; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \ + -F draft=false \ + -F prerelease=false \ + -f make_latest=true >/dev/null else - RELEASE_TYPE="release" + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \ + -F draft=false \ + -F prerelease=true \ + -f make_latest=false >/dev/null fi - # Get original release notes from tag - ORIGINAL_NOTES=$(git tag -l --format='%(contents)' "${TAG}") - if [[ -z "$ORIGINAL_NOTES" || "$ORIGINAL_NOTES" =~ ^[[:space:]]*$ ]]; then - if [[ "$IS_PRERELEASE" == "true" ]]; then - ORIGINAL_NOTES="Pre-release ${VERSION} (${RELEASE_TYPE})" - else - ORIGINAL_NOTES="Release ${VERSION}" - fi - fi - - # Publish the release (remove draft status) - gh release edit "$TAG" --draft=false - echo "🎉 Released $TAG successfully!" echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a2995b618..75ef45a59 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -82,7 +82,8 @@ jobs: github.event_name == 'workflow_dispatch' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && - github.event.workflow_run.head_branch != 'main') + github.event.workflow_run.head_branch != 'main' && + !contains(github.event.workflow_run.head_branch, '-preview')) runs-on: ubuntu-latest outputs: should_build: ${{ steps.check.outputs.should_build }} @@ -220,6 +221,13 @@ jobs: create_latest=true echo "🚀 Building with latest stable release version" ;; + *-preview*) + build_type="preview" + is_prerelease=true + should_build=false + should_push=false + echo "⏭️ Preview tags do not publish Docker images" + ;; # Prerelease versions (must match first, more specific) v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*) build_type="prerelease" diff --git a/.github/workflows/helm-package.yml b/.github/workflows/helm-package.yml index 49d038069..7e313c9b6 100644 --- a/.github/workflows/helm-package.yml +++ b/.github/workflows/helm-package.yml @@ -33,11 +33,12 @@ jobs: build-helm-package: runs-on: ubuntu-latest if: | - github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) || ( github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && - contains(github.event.workflow_run.head_branch, '.') + contains(github.event.workflow_run.head_branch, '.') && + !contains(github.event.workflow_run.head_branch, '-preview') ) outputs: diff --git a/scripts/security/check_preview_release_workflow.sh b/scripts/security/check_preview_release_workflow.sh new file mode 100755 index 000000000..1ad081643 --- /dev/null +++ b/scripts/security/check_preview_release_workflow.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +set -euo pipefail + +build_workflow=".github/workflows/build.yml" +docker_workflow=".github/workflows/docker.yml" +helm_workflow=".github/workflows/helm-package.yml" + +require_line() { + local file="$1" + local line="$2" + local description="$3" + + if ! grep -Fxq "$line" "$file"; then + echo "missing preview release workflow contract: $description" >&2 + exit 1 + fi +} + +extract_job_if() { + local file="$1" + local job="$2" + + awk -v job="$job" ' + $0 == " " job ":" { in_job = 1 } + in_job && /^ if:/ { in_if = 1 } + in_if && /^ [A-Za-z0-9_-]+:/ && $0 !~ /^ if:/ { exit } + in_if { print } + in_job && $0 ~ /^ [A-Za-z0-9_-]+:$/ && $0 != " " job ":" { exit } + ' "$file" +} + +require_job_if() { + local file="$1" + local job="$2" + local expected="$3" + local actual + + actual=$(extract_job_if "$file" "$job") + if [[ "$actual" != "$expected" ]]; then + echo "missing preview release workflow contract: $job condition" >&2 + exit 1 + fi +} + +require_assignment() { + local block="$1" + local name="$2" + local expected="$3" + local count + + count=$(printf '%s\n' "$block" | grep -Ec "^[[:space:]]+${name}=") + if [[ "$count" -ne 1 || "$block" != *"${name}=${expected}"* ]]; then + echo "missing preview release workflow contract: $name=$expected" >&2 + exit 1 + fi +} + +require_no_assignment() { + local block="$1" + local name="$2" + + if printf '%s\n' "$block" | grep -Eq "^[[:space:]]+${name}="; then + echo "invalid preview release workflow contract: unexpected $name override" >&2 + exit 1 + fi +} + +tag_classification=$(awk ' + /if \[\[ "\$tag_name" =~ -preview\\\.\[0-9\]\+\$ \]\]; then/ { in_preview = 1 } + in_preview { print } + in_preview && $0 == " fi" { exit } +' "$build_workflow") +invalid_preview_condition="elif [[ \"\$tag_name\" == *\"-preview\"* ]]; then" +prerelease_condition="elif [[ \"\$tag_name\" == *\"alpha\"* ]] || [[ \"\$tag_name\" == *\"beta\"* ]] || [[ \"\$tag_name\" == *\"rc\"* ]]; then" +preview_branch=$(printf '%s\n' "$tag_classification" | awk -v boundary="$invalid_preview_condition" 'index($0, boundary) { exit } { print }') +invalid_preview_branch=$(printf '%s\n' "$tag_classification" | awk -v start="$invalid_preview_condition" -v boundary="$prerelease_condition" 'index($0, start) { in_branch = 1 } in_branch && index($0, boundary) { exit } in_branch { print }') +prerelease_branch=$(printf '%s\n' "$tag_classification" | awk -v start="$prerelease_condition" 'index($0, start) { in_branch = 1 } in_branch && $0 == " else" { exit } in_branch { print }') +release_branch=$(printf '%s\n' "$tag_classification" | awk '$0 == " else" { in_branch = 1; next } in_branch && $0 == " fi" { exit } in_branch { print }') + +if [[ -z "$preview_branch" || -z "$invalid_preview_branch" || -z "$prerelease_branch" || -z "$release_branch" ]]; then + echo "missing preview release workflow contract: ordered preview, invalid-preview, prerelease, and release branches" >&2 + exit 1 +fi +require_assignment "$preview_branch" "build_type" '"preview"' +require_assignment "$preview_branch" "is_prerelease" "true" +require_no_assignment "$invalid_preview_branch" "build_type" +if [[ "$invalid_preview_branch" != *"exit 1"* ]]; then + echo "missing preview release workflow contract: invalid preview tags must fail closed" >&2 + exit 1 +fi +require_assignment "$prerelease_branch" "build_type" '"prerelease"' +require_assignment "$prerelease_branch" "is_prerelease" "true" +require_assignment "$release_branch" "build_type" '"release"' +require_no_assignment "$release_branch" "is_prerelease" + +initial_prerelease_line=$(grep -Fn " is_prerelease=false" "$build_workflow") +preview_condition_line=$(grep -Fn " if [[ \"\$tag_name\" =~ -preview\\.[0-9]+\$ ]]; then" "$build_workflow") +if [[ $(printf '%s\n' "$initial_prerelease_line" | wc -l) -ne 1 || "${initial_prerelease_line%%:*}" -ge "${preview_condition_line%%:*}" ]]; then + echo "missing preview release workflow contract: is_prerelease=false must initialize before tag classification" >&2 + exit 1 +fi + +tag_branch_tail=$(awk ' + /if \[\[ "\$tag_name" =~ -preview\\\.\[0-9\]\+\$ \]\]; then/ { in_classification = 1 } + in_classification && $0 == " fi" { after_classification = 1; next } + after_classification && /^ elif / { exit } + after_classification { print } +' "$build_workflow") +post_strategy=$(awk ' + /# Determine build type based on trigger/ { in_strategy = 1 } + in_strategy && $0 == " fi" { after_strategy = 1; next } + after_strategy && $0 == " {" { exit } + after_strategy { print } +' "$build_workflow") +for block in "$tag_branch_tail" "$post_strategy"; do + require_no_assignment "$block" "build_type" + require_no_assignment "$block" "is_prerelease" +done +require_line "$build_workflow" " if [[ \"\$BUILD_TYPE\" == \"release\" ]] || [[ \"\$BUILD_TYPE\" == \"prerelease\" ]]; then" "latest artifact guard" +require_line "$build_workflow" " if: env.R2_ACCESS_KEY_ID != '' && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease' || needs.build-check.outputs.build_type == 'development')" "R2 publication guard" +release_guard="startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')" +for job in create-release upload-release-assets publish-release; do + require_job_if "$build_workflow" "$job" " if: $release_guard" +done +latest_guard="startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')" +require_job_if "$build_workflow" "update-latest-version" " if: $latest_guard" +require_line "$build_workflow" " needs: [ build-check, publish-release ]" "latest update must follow release publication" +require_line "$build_workflow" " --latest=false \\" "new releases must start outside the latest channel" +prerelease_flag_block=$(awk ' + $0 == " PRERELEASE_FLAG=\"\"" { in_block = 1 } + in_block { print } + in_block && $0 == " fi" { exit } +' "$build_workflow") +IFS= read -r -d '' expected_prerelease_flag_block <<'EOF' || true + PRERELEASE_FLAG="" + if [[ "$IS_PRERELEASE" == "true" ]]; then + PRERELEASE_FLAG="--prerelease" + fi +EOF +expected_prerelease_flag_block=${expected_prerelease_flag_block%$'\n'} +if [[ "$prerelease_flag_block" != "$expected_prerelease_flag_block" ]]; then + echo "missing preview release workflow contract: prerelease flag propagation" >&2 + exit 1 +fi +require_line "$build_workflow" " \$PRERELEASE_FLAG \\" "GitHub prerelease creation flag" + +release_channel_block=$(awk ' + $0 == " if [[ \"\$BUILD_TYPE\" == \"release\" ]]; then" { in_block = 1 } + in_block { print } + in_block && $0 == " fi" { exit } +' "$build_workflow") +IFS= read -r -d '' expected_release_channel_block <<'EOF' || true + if [[ "$BUILD_TYPE" == "release" ]]; then + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \ + -F draft=false \ + -F prerelease=false \ + -f make_latest=true >/dev/null + else + gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \ + -F draft=false \ + -F prerelease=true \ + -f make_latest=false >/dev/null + fi +EOF +expected_release_channel_block=${expected_release_channel_block%$'\n'} +if [[ "$release_channel_block" != "$expected_release_channel_block" ]]; then + echo "missing preview release workflow contract: only stable releases may become GitHub Latest" >&2 + exit 1 +fi + +IFS= read -r -d '' expected_docker_automatic_guard <<'EOF' || true + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch != 'main' && + !contains(github.event.workflow_run.head_branch, '-preview')) +EOF +expected_docker_automatic_guard=${expected_docker_automatic_guard%$'\n'} +require_job_if "$docker_workflow" "build-check" "$expected_docker_automatic_guard" + +docker_manual_guard=$(awk ' + $0 == " *-preview*)" { in_preview = 1 } + in_preview { print } + in_preview && $0 == " ;;" { exit } +' "$docker_workflow") +for assignment in 'build_type="preview"' 'is_prerelease=true' 'should_build=false' 'should_push=false'; do + name="${assignment%%=*}" + require_assignment "$docker_manual_guard" "$name" "${assignment#*=}" +done + +IFS= read -r -d '' expected_helm_guard <<'EOF' || true + if: | + (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) || + ( + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + contains(github.event.workflow_run.head_branch, '.') && + !contains(github.event.workflow_run.head_branch, '-preview') + ) +EOF +expected_helm_guard=${expected_helm_guard%$'\n'} +require_job_if "$helm_workflow" "build-helm-package" "$expected_helm_guard" + +echo "Preview release workflow contract ok." From 8601179c3989d131fb68fa311fd517fe281270fe Mon Sep 17 00:00:00 2001 From: cxymds Date: Thu, 30 Jul 2026 12:09:08 +0800 Subject: [PATCH 08/52] fix(ecstore): quarantine rejected format members (#5463) --- crates/ecstore/src/disk/disk_store.rs | 4 + crates/ecstore/src/disk/mod.rs | 78 +++++++ crates/ecstore/src/runtime/sources.rs | 118 +++++++++- crates/ecstore/src/store/init.rs | 22 +- crates/ecstore/src/store/init_format.rs | 285 +++++++++++++++++++----- crates/ecstore/src/store/peer.rs | 75 ++++++- 6 files changed, 514 insertions(+), 68 deletions(-) diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 1183d7c18..29c76765d 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -1049,6 +1049,10 @@ impl LocalDiskWrapper { Ok(()) } + pub(crate) async fn set_disk_id_state(&self, id: Option) { + *self.disk_id.write().await = id; + } + /// Get the current disk ID pub async fn get_current_disk_id(&self) -> Option { *self.disk_id.read().await diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 4ac894373..a8074ddd9 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -132,6 +132,18 @@ pub enum Disk { Remote(Box), } +impl Disk { + pub(crate) async fn set_disk_id_state(&self, id: Option) -> Result<()> { + match self { + Disk::Local(local_disk) => { + local_disk.set_disk_id_state(id).await; + Ok(()) + } + Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await, + } + } +} + #[async_trait::async_trait] impl DiskAPI for Disk { fn to_string(&self) -> String { @@ -1552,6 +1564,72 @@ mod tests { let _ = fs::remove_dir_all(&test_dir).await; } + #[tokio::test] + #[serial_test::serial] + async fn local_disk_id_state_does_not_publish_to_the_process_registry() { + let local_dir = tempfile::tempdir().expect("local disk tempdir should be created"); + let mut endpoint = + Endpoint::try_from(local_dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let local_disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize"); + let disk = Disk::Local(Box::new(LocalDiskWrapper::new(Arc::new(local_disk), false))); + let disk_id = Uuid::new_v4(); + + disk.set_disk_id_state(Some(disk_id)) + .await + .expect("local wrapper state should accept a disk ID"); + + let Disk::Local(local_disk) = &disk else { + panic!("test disk should remain local"); + }; + assert_eq!(local_disk.get_current_disk_id().await, Some(disk_id)); + assert!( + !crate::runtime::global::current_ctx() + .local_disk_id_map() + .read() + .await + .contains_key(&disk_id), + "state-only startup publication must not update the process disk-ID registry" + ); + + disk.set_disk_id_state(None) + .await + .expect("local wrapper state should clear a disk ID"); + assert_eq!(local_disk.get_current_disk_id().await, None); + } + + #[tokio::test] + async fn remote_disk_id_state_delegates_some_and_none() { + let mut endpoint = Endpoint::try_from("http://remote-server:9000/data").expect("remote endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let remote_disk = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport), + ) + .await + .expect("remote disk should initialize"); + let disk = Disk::Remote(Box::new(remote_disk)); + let disk_id = Uuid::new_v4(); + + disk.set_disk_id_state(Some(disk_id)) + .await + .expect("remote state should accept a disk ID"); + assert_eq!(disk.get_disk_id().await.expect("remote disk ID should be readable"), Some(disk_id)); + + disk.set_disk_id_state(None) + .await + .expect("remote state should clear a disk ID"); + assert_eq!(disk.get_disk_id().await.expect("remote disk ID should be readable"), None); + } + #[tokio::test] async fn reset_health_for_store_init_retry_delegates_to_disk_variants() { let local_dir = tempfile::tempdir().unwrap(); diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index f2df75d2a..8a16f0c26 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -27,7 +27,7 @@ use crate::{ bucket::replication::{DynReplicationPool, ReplicationStats}, config::{get_global_storage_class, get_global_storage_class_snapshot, set_global_storage_class, storageclass}, disk::{DiskAPI, DiskOption, DiskStore, new_disk}, - error::Result, + error::{Error, Result}, layout::endpoints::{EndpointServerPools, SetupType}, runtime::global::{ GLOBAL_BOOT_TIME, GLOBAL_LIFECYCLE_SYS, GLOBAL_LOCAL_NODE_NAME_FALLBACK, GLOBAL_ROOT_DISK_THRESHOLD, @@ -417,10 +417,6 @@ pub(crate) async fn clear_local_disk_id_map_for_test() { local_disk_id_map_handle().write().await.clear(); } -pub(crate) async fn record_local_disk_id(instance_ctx: &Arc, disk_id: Uuid, endpoint: String) { - instance_ctx.local_disk_id_map().write().await.insert(disk_id, endpoint); -} - pub(crate) async fn replace_local_disk_id(previous: Option, current: Option, endpoint: String) { let id_map = local_disk_id_map_handle(); let mut disk_id_map = id_map.write().await; @@ -436,6 +432,53 @@ pub(crate) async fn replace_local_disk_id(previous: Option, current: Optio } } +pub(crate) async fn reconcile_local_disk_ids( + instance_ctx: &InstanceContext, + pool_endpoints: &[String], + selected: &[(Uuid, String)], +) { + let pool_endpoints = pool_endpoints.iter().map(String::as_str).collect::>(); + let disk_id_map = instance_ctx.local_disk_id_map(); + let mut disk_ids = disk_id_map.write().await; + disk_ids.retain(|_, registered_endpoint| !pool_endpoints.contains(registered_endpoint.as_str())); + disk_ids.extend(selected.iter().cloned()); +} + +pub(crate) async fn quarantine_local_disks(instance_ctx: &InstanceContext, endpoints: &[Endpoint]) -> Result<()> { + let slots = endpoints + .iter() + .map(|endpoint| { + Ok(( + usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?, + usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?, + usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?, + )) + }) + .collect::>>()?; + + let local_disk_map = instance_ctx.local_disk_map(); + let mut local_disks = local_disk_map.write().await; + for endpoint in endpoints { + local_disks.insert(endpoint.to_string(), None); + } + drop(local_disks); + + let set_drives = instance_ctx.local_disk_set_drives(); + let mut local_set_drives = set_drives.write().await; + if local_set_drives.is_empty() { + return Ok(()); + } + for (pool_idx, set_idx, disk_idx) in slots { + let disk = local_set_drives + .get_mut(pool_idx) + .and_then(|sets| sets.get_mut(set_idx)) + .and_then(|disks| disks.get_mut(disk_idx)) + .ok_or(Error::CorruptedFormat)?; + *disk = None; + } + Ok(()) +} + pub(crate) async fn record_local_disks(instance_ctx: &Arc, disks: Vec) { let map = instance_ctx.local_disk_map(); let mut global_local_disk_map = map.write().await; @@ -563,8 +606,8 @@ pub(crate) async fn init_tier_config_mgr(store: Arc) -> Result<()> { #[cfg(test)] mod tests { use super::{ - LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, replace_local_disk_id, - set_local_node_name, + LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, reconcile_local_disk_ids, + replace_local_disk_id, set_local_node_name, }; use crate::disk::endpoint::Endpoint; use rustfs_lock::{LocalClient, LockClient}; @@ -628,4 +671,65 @@ mod tests { assert_eq!(local_disk_path_by_id(&disk_id).await, Some("endpoint-a".to_string())); clear_local_disk_id_map_for_test().await; } + + #[tokio::test] + #[serial_test::serial] + async fn reconciling_pool_disk_ids_preserves_other_endpoints() { + let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let process_ctx = crate::runtime::global::current_ctx(); + let bootstrap_ctx = crate::runtime::instance::bootstrap_ctx(); + let retained_id = Uuid::new_v4(); + let removed_id = Uuid::new_v4(); + let selected_id = Uuid::new_v4(); + let process_sentinel = Uuid::new_v4(); + let bootstrap_sentinel = Uuid::new_v4(); + instance_ctx.local_disk_id_map().write().await.extend([ + (retained_id, "endpoint-a".to_string()), + (removed_id, "endpoint-b".to_string()), + ]); + process_ctx + .local_disk_id_map() + .write() + .await + .insert(process_sentinel, "endpoint-b".to_string()); + bootstrap_ctx + .local_disk_id_map() + .write() + .await + .insert(bootstrap_sentinel, "endpoint-b".to_string()); + + reconcile_local_disk_ids( + &instance_ctx, + &["endpoint-b".to_string(), "endpoint-c".to_string()], + &[(selected_id, "endpoint-c".to_string())], + ) + .await; + + let disk_ids = instance_ctx.local_disk_id_map(); + let disk_ids = disk_ids.read().await; + assert_eq!(disk_ids.get(&retained_id).map(String::as_str), Some("endpoint-a")); + assert_eq!(disk_ids.get(&removed_id), None); + assert_eq!(disk_ids.get(&selected_id).map(String::as_str), Some("endpoint-c")); + drop(disk_ids); + assert_eq!( + process_ctx + .local_disk_id_map() + .read() + .await + .get(&process_sentinel) + .map(String::as_str), + Some("endpoint-b") + ); + assert_eq!( + bootstrap_ctx + .local_disk_id_map() + .read() + .await + .get(&bootstrap_sentinel) + .map(String::as_str), + Some("endpoint-b") + ); + process_ctx.local_disk_id_map().write().await.remove(&process_sentinel); + bootstrap_ctx.local_disk_id_map().write().await.remove(&bootstrap_sentinel); + } } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 4e7128188..c89320908 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -313,7 +313,8 @@ impl ECStore { let mut times = 0; let mut interval = 1; loop { - match init_format::connect_load_init_formats( + match init_format::connect_load_init_formats_with_instance_ctx( + &instance_ctx, pool_first_is_local, &mut disks, pool_eps.set_count, @@ -1259,6 +1260,16 @@ mod tests { let registered: Vec = instance_ctx.local_disk_map().read().await.keys().cloned().collect(); assert_eq!(registered.len(), 4, "the passed context must register all four local disks"); + let registered_disk_ids = instance_ctx.local_disk_id_map(); + let registered_disk_ids = registered_disk_ids.read().await; + assert_eq!(registered_disk_ids.len(), 4, "the passed context must publish all four disk IDs"); + for endpoint in registered_disk_ids.values() { + assert!( + registered.contains(endpoint), + "every disk ID in the passed context must resolve to one of its registered endpoints" + ); + } + drop(registered_disk_ids); let bootstrap = crate::runtime::instance::bootstrap_ctx(); assert_ne!( bootstrap.deployment_id(), @@ -1273,6 +1284,15 @@ mod tests { "the bootstrap context must not absorb the fresh store's disks" ); } + drop(bootstrap_map); + let bootstrap_disk_ids = bootstrap.local_disk_id_map(); + let bootstrap_disk_ids = bootstrap_disk_ids.read().await; + for endpoint in bootstrap_disk_ids.values() { + assert!( + !registered.contains(endpoint), + "the bootstrap context must not absorb the fresh store's disk IDs" + ); + } } #[tokio::test] diff --git a/crates/ecstore/src/store/init_format.rs b/crates/ecstore/src/store/init_format.rs index bd4ccb301..95cf17c66 100644 --- a/crates/ecstore/src/store/init_format.rs +++ b/crates/ecstore/src/store/init_format.rs @@ -16,6 +16,8 @@ use crate::config::storageclass; use crate::disk::error_reduce::{count_errs, reduce_write_quorum_errs}; use crate::disk::{self, DiskAPI}; use crate::error::{Error, Result}; +use crate::runtime::instance::InstanceContext; +use crate::runtime::sources::{quarantine_local_disks, reconcile_local_disk_ids}; use crate::{ disk::{ DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET, @@ -26,7 +28,10 @@ use crate::{ layout::endpoints::Endpoints, }; use futures::{future::join_all, stream, stream::StreamExt}; -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use tokio::io::AsyncReadExt; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -62,12 +67,25 @@ pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec], set_count: usize, set_drive_count: usize, deployment_id: Option, +) -> Result { + let instance_ctx = crate::runtime::global::current_ctx(); + connect_load_init_formats_with_instance_ctx(&instance_ctx, first_disk, disks, set_count, set_drive_count, deployment_id).await +} + +pub(crate) async fn connect_load_init_formats_with_instance_ctx( + instance_ctx: &Arc, + first_disk: bool, + disks: &mut [Option], + set_count: usize, + set_drive_count: usize, + deployment_id: Option, ) -> Result { let (formats, errs) = load_format_erasure_all(disks, false).await; @@ -98,7 +116,7 @@ pub async fn connect_load_init_formats( match try_migrate_format(disks, &formats, set_count, set_drive_count).await { Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => { info!("Migrated format from MinIO config"); - retain_format_quorum_members(disks, &format, &quorum_members, set_drive_count).await?; + retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?; return Ok(*format); } Ok(LegacyFormatOutcome::Incompatible) => { @@ -115,7 +133,7 @@ pub async fn connect_load_init_formats( Err(e) => return Err(e), } if all_unformatted { - let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?; + let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?; return Ok(fm); } } @@ -140,12 +158,13 @@ pub async fn connect_load_init_formats( None => select_format_erasure_in_quorum(&formats, 0)?, }; check_format_erasure_value_for_topology(&fm, formats.len(), set_drive_count)?; - retain_format_quorum_members(disks, &fm, &quorum_members, set_drive_count).await?; + retain_format_quorum_members(instance_ctx, disks, &fm, &quorum_members, set_drive_count).await?; Ok(fm) } async fn retain_format_quorum_members( + instance_ctx: &Arc, disks: &mut [Option], format: &FormatV3, quorum_members: &[bool], @@ -154,26 +173,73 @@ async fn retain_format_quorum_members( if set_drive_count == 0 || quorum_members.len() != disks.len() { return Err(Error::CorruptedFormat); } - for (disk, belongs_to_quorum) in disks.iter().zip(quorum_members) { - if !belongs_to_quorum && let Some(disk) = disk { - disk.set_disk_id(None).await?; + + let pool_idx = disks + .iter() + .flatten() + .map(|disk| disk.endpoint().pool_idx) + .find_map(|pool_idx| usize::try_from(pool_idx).ok()); + let registered_endpoints = if let Some(pool_idx) = pool_idx { + instance_ctx + .local_disk_set_drives() + .read() + .await + .get(pool_idx) + .map(|sets| { + sets.iter() + .flat_map(|set| set.iter()) + .map(|disk| disk.as_ref().map(|disk| disk.endpoint())) + .collect::>() + }) + .filter(|endpoints| endpoints.len() == disks.len()) + } else { + None + }; + let endpoints = disks + .iter() + .enumerate() + .map(|(index, disk)| { + disk.as_ref() + .map(|disk| disk.endpoint()) + .or_else(|| registered_endpoints.as_ref()?.get(index)?.clone()) + }) + .collect::>(); + + let mut member_disk_ids = vec![None; disks.len()]; + let mut local_pool_endpoints = Vec::new(); + let mut selected_local_disk_ids = Vec::new(); + let mut quarantined_endpoints = Vec::new(); + for (index, ((disk, endpoint), belongs_to_quorum)) in disks.iter().zip(&endpoints).zip(quorum_members).enumerate() { + if let Some(endpoint) = endpoint.as_ref().filter(|endpoint| endpoint.is_local) { + local_pool_endpoints.push(endpoint.to_string()); + if !belongs_to_quorum { + quarantined_endpoints.push(endpoint.clone()); + } } - } - for (index, (disk, belongs_to_quorum)) in disks.iter().zip(quorum_members).enumerate() { if !belongs_to_quorum { continue; } + let disk = disk.as_ref().ok_or(Error::CorruptedFormat)?; let disk_id = format .erasure .sets .get(index / set_drive_count) .and_then(|set| set.get(index % set_drive_count)) + .copied() .ok_or(Error::CorruptedFormat)?; - disk.as_ref() - .ok_or(Error::CorruptedFormat)? - .set_disk_id(Some(*disk_id)) - .await?; + member_disk_ids[index] = Some(disk_id); + if disk.is_local() { + selected_local_disk_ids.push((disk_id, disk.endpoint().to_string())); + } } + quarantine_local_disks(instance_ctx, &quarantined_endpoints).await?; + + for (disk, disk_id) in disks.iter().zip(member_disk_ids) { + if let Some(disk) = disk { + disk.set_disk_id_state(disk_id).await?; + } + } + reconcile_local_disk_ids(instance_ctx, &local_pool_endpoints, &selected_local_disk_ids).await; for (disk, belongs_to_quorum) in disks.iter_mut().zip(quorum_members) { if !belongs_to_quorum { *disk = None; @@ -207,7 +273,8 @@ pub fn check_disk_fatal_errs(errs: &[Option]) -> disk::error::Result< } async fn init_format_erasure( - disks: &[Option], + instance_ctx: &Arc, + disks: &mut [Option], set_count: usize, set_drive_count: usize, deployment_id: Option, @@ -227,9 +294,11 @@ async fn init_format_erasure( } } - save_format_file_all(disks, &fms).await?; - - get_format_erasure_in_quorum(&fms, 0) + write_format_file_all(disks, &fms).await?; + let format = get_format_erasure_in_quorum(&fms, 0)?; + let quorum_members = vec![true; disks.len()]; + retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?; + Ok(format) } /// Outcome of attempting to migrate an on-disk MinIO `format.json`. @@ -348,44 +417,59 @@ async fn try_migrate_format( .iter() .zip(&formats_to_write) .zip(rustfs_formats) - .filter(|&((disk, _), existing)| disk.is_some() && existing.is_none()) - .map(|((disk, format), _)| save_format_file(disk, format)); + .filter_map(|((disk, format), existing)| { + if existing.is_some() { + return None; + } + Some(write_format_file(disk.as_ref()?, format.as_ref()?)) + }); let mut write_error = None; for result in join_all(writes).await { if let Err(error) = result { write_error.get_or_insert(error); } } - for disk in disks.iter().flatten() { - disk.set_disk_id(None).await?; - } - let (persisted_formats, persisted_errors) = load_format_erasure_all(disks, false).await; + let Some((persisted_format, quorum_members)) = + select_persisted_migration_format(&persisted_formats, persisted_errors, &format, set_drive_count, write_error)? + else { + return Ok(LegacyFormatOutcome::Incompatible); + }; + + Ok(LegacyFormatOutcome::Migrated { + format: Box::new(persisted_format), + quorum_members, + }) +} + +fn select_persisted_migration_format( + persisted_formats: &[Option], + persisted_errors: Vec>, + reference: &FormatV3, + set_drive_count: usize, + write_error: Option, +) -> Result)>> { + if !formats_match_reference_slots(persisted_formats, reference, 0) { + return Ok(None); + } + let quorum_error = match select_format_erasure_in_quorum(persisted_formats, 0) { + Ok((format, quorum_members)) => { + check_format_erasure_value_for_topology(&format, persisted_formats.len(), set_drive_count)?; + return Ok(Some((format, quorum_members))); + } + Err(error) => error, + }; if let Some(error) = persisted_errors .into_iter() .flatten() .find(|error| !matches!(error, DiskError::UnformattedDisk | DiskError::DiskNotFound)) { return match error { - DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::InconsistentDisk => { - Ok(LegacyFormatOutcome::Incompatible) - } + DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::InconsistentDisk => Ok(None), error => Err(error.into()), }; } - if !formats_match_reference_slots(&persisted_formats, &format, 0) { - return Ok(LegacyFormatOutcome::Incompatible); - } - let (persisted_format, quorum_members) = match select_format_erasure_in_quorum(&persisted_formats, 0) { - Ok(selected) => selected, - Err(error) => return Err(write_error.map_or(error, Into::into)), - }; - check_format_erasure_value_for_topology(&persisted_format, disks.len(), set_drive_count)?; - - Ok(LegacyFormatOutcome::Migrated { - format: Box::new(persisted_format), - quorum_members, - }) + Err(write_error.map_or(quorum_error, Into::into)) } fn legacy_format_max_bytes(disk_count: usize) -> Result { @@ -676,13 +760,12 @@ pub async fn load_format_erasure(disk: &DiskStore, heal: bool) -> disk::error::R Ok(fm) } -async fn save_format_file_all(disks: &[Option], formats: &[Option]) -> disk::error::Result<()> { - let mut futures = Vec::with_capacity(disks.len()); - - for (i, disk) in disks.iter().enumerate() { - futures.push(save_format_file(disk, &formats[i])); - } - +async fn write_format_file_all(disks: &[Option], formats: &[Option]) -> disk::error::Result<()> { + let futures = disks.iter().zip(formats).map(|(disk, format)| async move { + let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?; + let format = format.as_ref().ok_or_else(|| DiskError::other("format is none"))?; + write_format_file(disk, format).await + }); let mut errors = Vec::with_capacity(disks.len()); let results = join_all(futures).await; @@ -713,6 +796,13 @@ pub async fn save_format_file(disk: &Option, format: &Option disk::error::Result<()> { let json_data = format.to_json()?; let tmpfile = Uuid::new_v4().to_string(); @@ -723,8 +813,6 @@ pub async fn save_format_file(disk: &Option, format: &Option Result { mod tests { use super::*; use crate::layout::endpoint::Endpoint; - use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id}; + use crate::runtime::global::{current_ctx, reset_local_disk_test_state}; + use crate::runtime::sources::{local_disk_path_by_id, record_local_disks}; + use crate::store::peer::{find_local_disk_by_ref, prewarm_local_disk_id_map_with_instance_ctx}; use serial_test::serial; async fn local_disks(count: usize) -> (tempfile::TempDir, Vec>) { @@ -1045,7 +1135,7 @@ mod tests { #[tokio::test] #[serial] async fn existing_format_load_publishes_only_validated_quorum_disk_ids() { - clear_local_disk_id_map_for_test().await; + reset_local_disk_test_state().await; let (_temp_dir, mut disks) = local_disks(5).await; let canonical = FormatV3::new(1, 5); for (index, disk) in disks.iter().enumerate().take(3) { @@ -1055,19 +1145,56 @@ mod tests { .await .expect("canonical format should be written"); } - let mut wrong_slot = canonical.clone(); - wrong_slot.erasure.this = wrong_slot.erasure.sets[0][0]; - save_format_file(&disks[3], &Some(wrong_slot)) - .await - .expect("wrong-slot format should be written"); + let canonical_id = canonical.erasure.sets[0][0]; let mut foreign = FormatV3::new(1, 5); foreign.erasure.this = foreign.erasure.sets[0][4]; let foreign_id = foreign.erasure.this; save_format_file(&disks[4], &Some(foreign)) .await .expect("foreign format should be written"); - let canonical_id = canonical.erasure.sets[0][0]; + let mut collision = FormatV3::new(1, 5); + collision.erasure.sets[0][3] = canonical_id; + collision.erasure.this = canonical_id; + save_format_file(&disks[3], &Some(collision)) + .await + .expect("foreign collision format should be written"); let canonical_endpoint = disks[0].as_ref().expect("canonical disk should exist").endpoint().to_string(); + let collision_endpoint = disks[3].as_ref().expect("collision disk should exist").endpoint().to_string(); + let foreign_endpoint = disks[4].as_ref().expect("foreign disk should exist").endpoint().to_string(); + let prewarm_endpoints = Endpoints::from( + disks + .iter() + .map(|disk| disk.as_ref().expect("test disk should exist").endpoint()) + .collect::>(), + ); + for disk in disks.iter().flatten() { + disk.set_disk_id(None).await.expect("test disk ID should be reset"); + } + let (prewarm_disks, prewarm_errors) = init_disks( + &prewarm_endpoints, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await; + assert!(prewarm_errors.iter().all(Option::is_none)); + let prewarm_disks = prewarm_disks + .into_iter() + .map(|disk| disk.expect("prewarm disk should exist")) + .collect::>(); + let instance_ctx = current_ctx(); + record_local_disks(&instance_ctx, prewarm_disks.clone()).await; + *instance_ctx.local_disk_set_drives().write().await = vec![vec![prewarm_disks.into_iter().map(Some).collect()]]; + prewarm_local_disk_id_map_with_instance_ctx(&instance_ctx).await; + instance_ctx + .local_disk_id_map() + .write() + .await + .insert(canonical_id, collision_endpoint.clone()); + assert!(local_disk_path_by_id(&foreign_id).await.is_some(), "foreign ID should be prewarmed"); + assert_eq!(local_disk_path_by_id(&canonical_id).await, Some(collision_endpoint)); + disks[4] = None; connect_load_init_formats(true, &mut disks, 1, 5, None) .await @@ -1075,9 +1202,51 @@ mod tests { assert!(disks[..3].iter().all(Option::is_some)); assert!(disks[3..].iter().all(Option::is_none)); - assert_eq!(local_disk_path_by_id(&canonical_id).await, Some(canonical_endpoint)); + assert_eq!(local_disk_path_by_id(&canonical_id).await, Some(canonical_endpoint.clone())); assert_eq!(local_disk_path_by_id(&foreign_id).await, None); - clear_local_disk_id_map_for_test().await; + assert!( + instance_ctx + .local_disk_map() + .read() + .await + .get(&foreign_endpoint) + .is_some_and(Option::is_none) + ); + assert!(instance_ctx.local_disk_set_drives().read().await[0][0][4].is_none()); + assert!(find_local_disk_by_ref(&foreign_id.to_string()).await.is_none()); + assert_eq!( + find_local_disk_by_ref(&canonical_id.to_string()) + .await + .map(|disk| disk.endpoint().to_string()), + Some(canonical_endpoint) + ); + reset_local_disk_test_state().await; + } + + #[test] + fn persisted_migration_quorum_ignores_nonmember_read_errors() { + for drive_count in [3, 4] { + let reference = FormatV3::new(1, drive_count); + let required = drive_count / 2 + 1; + let mut formats = vec![None; drive_count]; + let mut errors = (0..drive_count).map(|_| None).collect::>>(); + for (index, format) in formats.iter_mut().enumerate().take(required) { + let mut disk_format = reference.clone(); + disk_format.erasure.this = reference.erasure.sets[0][index]; + *format = Some(disk_format); + } + errors[drive_count - 1] = Some(DiskError::FaultyDisk); + + let (selected, members) = + select_persisted_migration_format(&formats, errors, &reference, drive_count, Some(DiskError::FaultyDisk)) + .expect("a persisted strict majority must outrank a nonmember read error") + .expect("the persisted formats should be compatible"); + + assert_eq!(selected.shared_identity(), reference.shared_identity()); + assert_eq!(members.iter().filter(|member| **member).count(), required); + assert!(members[..required].iter().all(|member| *member)); + assert!(members[required..].iter().all(|member| !*member)); + } } #[tokio::test] diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index e827c7fa4..d8ba54d1f 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -28,8 +28,26 @@ async fn remember_local_disk_id(disk: &DiskStore) -> Option { async fn remember_local_disk_id_with_instance_ctx(instance_ctx: &Arc, disk: &DiskStore) -> Option { let disk_id = disk.get_disk_id().await.ok().flatten()?; - runtime_sources::record_local_disk_id(instance_ctx, disk_id, disk.endpoint().to_string()).await; - Some(disk_id) + record_local_disk_id_if_active(instance_ctx, disk, disk_id) + .await + .then_some(disk_id) +} + +async fn record_local_disk_id_if_active(instance_ctx: &Arc, disk: &DiskStore, disk_id: Uuid) -> bool { + let endpoint = disk.endpoint().to_string(); + let local_disk_map = instance_ctx.local_disk_map(); + let local_disks = local_disk_map.read().await; + let Some(active_disk) = local_disks.get(&endpoint).and_then(Option::as_ref) else { + return false; + }; + if !Arc::ptr_eq(active_disk, disk) { + return false; + } + + // Lock order is local_disk_map -> local_disk_id_map so quarantine is the + // linearization point for rejecting an in-flight stale disk snapshot. + instance_ctx.local_disk_id_map().write().await.insert(disk_id, endpoint); + true } pub async fn find_local_disk(disk_path: &str) -> Option { @@ -228,6 +246,7 @@ pub async fn get_disk_infos(disks: &[Option]) -> Vec #[cfg(test)] mod tests { use super::*; + use crate::disk::new_disk; use crate::layout::endpoints::{Endpoints, PoolEndpoints}; fn single_local_disk_pools(dir: &std::path::Path) -> EndpointServerPools { @@ -314,4 +333,56 @@ mod tests { ); } } + + #[tokio::test] + async fn stale_local_disk_snapshot_cannot_repopulate_the_id_registry() { + let temp_dir = tempfile::tempdir().expect("create temp disk dir"); + let endpoint_pools = single_local_disk_pools(temp_dir.path()); + let instance_ctx = Arc::new(InstanceContext::new()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools) + .await + .expect("local disk should be registered"); + let disk = instance_ctx + .local_disk_map() + .read() + .await + .values() + .find_map(|disk| disk.clone()) + .expect("registered local disk"); + let endpoint = disk.endpoint().to_string(); + let disk_id = Uuid::new_v4(); + + let local_disk_map = instance_ctx.local_disk_map(); + let mut quarantine = local_disk_map.write().await; + let replacement = new_disk( + &disk.endpoint(), + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("replacement disk should initialize"); + assert!(!Arc::ptr_eq(&disk, &replacement)); + let task_ctx = instance_ctx.clone(); + let task_disk = disk.clone(); + let remember = tokio::spawn(async move { record_local_disk_id_if_active(&task_ctx, &task_disk, disk_id).await }); + tokio::task::yield_now().await; + quarantine.insert(endpoint.clone(), Some(replacement.clone())); + drop(quarantine); + + assert!(!remember.await.expect("stale lookup task should complete")); + assert!(!instance_ctx.local_disk_id_map().read().await.contains_key(&disk_id)); + let active = instance_ctx + .local_disk_map() + .read() + .await + .get(&endpoint) + .cloned() + .flatten() + .expect("replacement disk should remain registered"); + assert!(Arc::ptr_eq(&active, &replacement)); + assert!(record_local_disk_id_if_active(&instance_ctx, &replacement, disk_id).await); + assert_eq!(instance_ctx.local_disk_id_map().read().await.get(&disk_id), Some(&endpoint)); + } } From 6e6b38ad8ece4de0800ecb726165a6c77649bfa4 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 30 Jul 2026 12:38:30 +0800 Subject: [PATCH 09/52] test(e2e): accept backpressure unknown terminal status (#5464) Co-authored-by: heihutu --- .../e2e_test/src/inline_fast_path_cluster_test.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index 9e737c382..d0585b307 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -2136,11 +2136,20 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat assert_eq!(terminal["bucket"].as_str(), Some(bucket.as_str())); assert_eq!(terminal["prefix"].as_str(), Some(prefix)); assert_eq!(terminal["dry_run"].as_bool(), Some(false)); - assert_eq!( - terminal["status"].as_str(), - Some("partial"), + let terminal_status = terminal["status"].as_str(); + assert!( + matches!(terminal_status, Some("partial" | "unknown")), "small transition queue should surface terminal backpressure: {terminal}" ); + if terminal_status == Some("unknown") { + let failure_reason = terminal["failure_reason"] + .as_str() + .ok_or_else(|| format!("unknown terminal status omitted failure_reason: {terminal}"))?; + assert!( + failure_reason.contains("worker result was not persisted before the transition queue drained"), + "unknown terminal status should identify lost worker-result persistence: {terminal}" + ); + } let skipped_queue_full = terminal["report"]["skipped_queue_full"] .as_u64() .ok_or_else(|| format!("terminal status omitted report.skipped_queue_full: {terminal}"))?; From ad7663afd184b1990490465b5cb29df51df7ed35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Thu, 30 Jul 2026 12:39:25 +0800 Subject: [PATCH 10/52] refactor(sse): decouple ecstore and harden KMS lifecycle (#5435) * refactor(sse): decouple encryption from ecstore * feat(kms): enhance KMS service manager with runtime state and persistence support * feat(kms): add local key export functionality for SSE-S3 migration tests * fix(kms): keep local key export narrowly scoped * fix(sse): validate copy source customer algorithm --------- Co-authored-by: Zhengchao An --- Cargo.lock | 3 - crates/ecstore/Cargo.toml | 3 - crates/ecstore/src/api/mod.rs | 10 +- crates/ecstore/src/client/utils.rs | 16 +- crates/ecstore/src/object_api/encryption.rs | 80 + crates/ecstore/src/object_api/mod.rs | 5 + crates/ecstore/src/object_api/readers.rs | 1646 ++--------------- crates/ecstore/src/object_api/types.rs | 63 +- crates/ecstore/src/runtime/instance.rs | 17 + crates/ecstore/src/runtime/sources.rs | 5 - crates/ecstore/src/set_disk/metadata.rs | 4 +- crates/ecstore/src/set_disk/mod.rs | 13 +- crates/ecstore/src/set_disk/ops/object.rs | 73 +- crates/ecstore/tests/README.md | 4 +- crates/kms/AGENTS.md | 22 + crates/kms/examples/local_kms_key_decrypt.rs | 112 ++ crates/kms/src/backends/local.rs | 100 + crates/kms/src/lib.rs | 2 +- crates/kms/src/service_manager.rs | 406 +++- .../rio-v2/tests/minio_fixture_lab/README.md | 2 +- crates/utils/src/http/header_compat.rs | 76 +- .../ecstore-validation-suite-design.md | 2 +- rustfs/src/admin/handlers/kms_dynamic.rs | 275 +-- rustfs/src/app/context/global.rs | 16 +- rustfs/src/app/context/handles.rs | 38 +- .../src/storage}/minio_generated_read_test.rs | 20 +- rustfs/src/storage/mod.rs | 2 + rustfs/src/storage/sse.rs | 379 +++- rustfs/src/storage/storage_api.rs | 39 +- scripts/run_ecstore_validation_suite.sh | 4 +- 30 files changed, 1486 insertions(+), 1951 deletions(-) create mode 100644 crates/ecstore/src/object_api/encryption.rs create mode 100644 crates/kms/examples/local_kms_key_decrypt.rs rename {crates/ecstore/tests => rustfs/src/storage}/minio_generated_read_test.rs (96%) diff --git a/Cargo.lock b/Cargo.lock index dd420db90..3cde4a2ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9099,7 +9099,6 @@ dependencies = [ name = "rustfs-ecstore" version = "1.0.0-beta.12" dependencies = [ - "aes-gcm", "arc-swap", "async-channel", "async-recursion", @@ -9115,7 +9114,6 @@ dependencies = [ "byteorder", "bytes", "bytesize", - "chacha20poly1305", "chrono", "criterion", "enumset", @@ -9169,7 +9167,6 @@ dependencies = [ "rustfs-erasure-codec", "rustfs-filemeta", "rustfs-io-metrics", - "rustfs-kms", "rustfs-lifecycle", "rustfs-lock", "rustfs-madmin", diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 31a5446d1..78186b52a 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -57,7 +57,6 @@ rustfs-policy.workspace = true rustfs-protos.workspace = true rustfs-replication.workspace = true rustfs-lifecycle.workspace = true -rustfs-kms.workspace = true rustfs-s3-types = { workspace = true } rustfs-data-usage.workspace = true rustfs-object-capacity.workspace = true @@ -125,8 +124,6 @@ libc.workspace = true rustix = { workspace = true, features = ["process", "fs"] } rustfs-madmin.workspace = true reqwest = { workspace = true } -aes-gcm = { workspace = true, features = ["rand_core"] } -chacha20poly1305.workspace = true aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } urlencoding = { workspace = true } smallvec = { workspace = true, features = ["serde"] } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index d5bed138b..15aebe9d9 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -391,10 +391,12 @@ pub mod notification { pub mod object { pub use crate::object_api::{ - BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, - GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer, - get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, - register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, + GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo, + ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, + ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, + register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook, + unregister_object_mutation_hook, }; pub use crate::store::PreparedGetObjectReader; } diff --git a/crates/ecstore/src/client/utils.rs b/crates/ecstore/src/client/utils.rs index 5d6c975be..3a05e7cf3 100644 --- a/crates/ecstore/src/client/utils.rs +++ b/crates/ecstore/src/client/utils.rs @@ -46,16 +46,6 @@ lazy_static! { m.insert("x-amz-replication-status".to_string(), true); m }; - static ref SSE_HEADERS: HashMap = { - let mut m = HashMap::new(); - m.insert("x-amz-server-side-encryption".to_string(), true); - m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); - m.insert("x-amz-server-side-encryption-context".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); - m - }; } pub fn is_standard_query_value(qs_key: &str) -> bool { @@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool { *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) } -pub fn is_sse_header(header_key: &str) -> bool { - *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) -} - pub fn is_amz_header(header_key: &str) -> bool { let key = header_key.to_lowercase(); key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" - || is_sse_header(header_key) + || rustfs_utils::http::is_sse_header(header_key) || key.starts_with("x-amz-checksum-") } diff --git a/crates/ecstore/src/object_api/encryption.rs b/crates/ecstore/src/object_api/encryption.rs new file mode 100644 index 000000000..07413cdd9 --- /dev/null +++ b/crates/ecstore/src/object_api/encryption.rs @@ -0,0 +1,80 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use async_trait::async_trait; +use http::{HeaderMap, HeaderValue}; +use std::collections::HashMap; +use std::error::Error; +use std::fmt::{Display, Formatter}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadEncryptionMode { + Direct { base_nonce: [u8; 12] }, + Object, +} + +pub struct ReadEncryptionMaterial { + pub key_bytes: [u8; 32], + pub mode: ReadEncryptionMode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncryptionResolutionErrorKind { + InvalidRequest, + InvalidMetadata, + ServiceUnavailable, + DecryptionFailed, +} + +#[derive(Debug)] +pub struct EncryptionResolutionError { + kind: EncryptionResolutionErrorKind, + message: String, +} + +impl EncryptionResolutionError { + pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } + + pub fn kind(&self) -> EncryptionResolutionErrorKind { + self.kind + } +} + +impl Display for EncryptionResolutionError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl Error for EncryptionResolutionError {} + +pub struct ReadEncryptionRequest<'a> { + pub bucket: &'a str, + pub object: &'a str, + pub metadata: &'a HashMap, + pub headers: &'a HeaderMap, +} + +#[async_trait] +pub trait ObjectEncryptionResolver: Send + Sync { + async fn resolve_read_material( + &self, + request: ReadEncryptionRequest<'_>, + ) -> Result, EncryptionResolutionError>; +} diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 5c57b69b2..28d567e75 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool { } mod body_cache_hook; +mod encryption; mod hook_slot; mod object_mutation_hook; mod readers; @@ -98,6 +99,10 @@ pub use body_cache_hook::{ pub(crate) use body_cache_hook::{ get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook, }; +pub use encryption::{ + EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial, + ReadEncryptionMode, ReadEncryptionRequest, +}; pub(crate) use object_mutation_hook::notify_object_mutation; pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook}; pub use readers::*; diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 1330cc10b..ff82e2a4b 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -13,110 +13,13 @@ // limitations under the License. use super::*; -#[cfg(feature = "rio-v2")] -use aes_gcm::aead::Payload; -use aes_gcm::{ - Aes256Gcm, Key, Nonce, - aead::{Aead, KeyInit}, -}; -use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; -#[cfg(feature = "rio-v2")] -use chacha20poly1305::ChaCha20Poly1305; -#[cfg(feature = "rio-v2")] -use hmac::{Hmac, Mac}; -use md5::{Digest, Md5}; -use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext}; -use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; -use rustfs_utils::path::path_join_buf; -use serde::Deserialize; -#[cfg(feature = "rio-v2")] -use sha2::Sha256; -use std::collections::HashMap; -use std::env; use crate::io_support::rio::Index; -const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; -const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key"; -const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context"; -const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv"; -const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size"; -const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size"; -const DEFAULT_SSE_ALGORITHM: &str = "AES256"; -const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1; #[cfg(feature = "rio-v2")] const DARE_PAYLOAD_SIZE: i64 = 64 * 1024; #[cfg(feature = "rio-v2")] const DARE_PACKAGE_SIZE: i64 = DARE_PAYLOAD_SIZE + 32; -const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256"; -#[cfg(feature = "rio-v2")] -const DARE_VERSION_20: u8 = 0x20; -#[cfg(feature = "rio-v2")] -const DARE_CIPHER_AES_256_GCM: u8 = 0x00; -#[cfg(feature = "rio-v2")] -const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01; -#[cfg(feature = "rio-v2")] -const DARE_HEADER_SIZE: usize = 16; -#[cfg(feature = "rio-v2")] -const DARE_TAG_SIZE: usize = 16; -#[cfg(feature = "rio-v2")] -const SEALED_KEY_IV_SIZE: usize = 32; -#[cfg(feature = "rio-v2")] -const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE; -#[cfg(feature = "rio-v2")] -const MINIO_SECRET_KEY_RANDOM_SIZE: usize = 28; -#[cfg(feature = "rio-v2")] -const MINIO_SECRET_KEY_IV_SIZE: usize = 16; -#[cfg(feature = "rio-v2")] -const MINIO_SECRET_KEY_NONCE_SIZE: usize = 12; - -#[cfg(feature = "rio-v2")] -type HmacSha256 = Hmac; - -fn canonical_kms_bucket_path(bucket: &str, object: &str) -> String { - path_join_buf(&[bucket, object]) -} - -fn build_object_encryption_context( - bucket: &str, - object: &str, - provided_context: Option<&HashMap>, -) -> ObjectEncryptionContext { - let mut context = provided_context.cloned().unwrap_or_default(); - context - .entry(bucket.to_string()) - .or_insert_with(|| canonical_kms_bucket_path(bucket, object)); - - let mut object_context = ObjectEncryptionContext::new(bucket.to_string(), object.to_string()); - for (ctx_key, ctx_value) in context { - object_context = object_context.with_encryption_context(ctx_key, ctx_value); - } - object_context -} - -#[cfg(feature = "rio-v2")] -fn is_legacy_rustfs_managed_metadata(metadata: &HashMap) -> bool { - metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some() - && metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER).is_some() - && metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_none() - && metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_none() -} fn part_plaintext_size(part: &ObjectPartInfo) -> i64 { if part.actual_size > 0 { @@ -544,21 +447,6 @@ impl GetObjectReader { } } -#[derive(Debug, Clone, Copy)] -struct EncryptionMaterial { - key_bytes: [u8; 32], - base_nonce: [u8; 12], - key_kind: EncryptionKeyKind, - reader_backend: crate::io_support::rio::ReadEncryptionBackend, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum EncryptionKeyKind { - Direct, - Object, -} - -#[derive(Debug, Clone)] enum ReadTransform { Plain { visible_offset: usize, @@ -572,7 +460,7 @@ enum ReadTransform { total_plaintext_size: usize, }, Encrypted { - material: EncryptionMaterial, + material: ReadEncryptionMaterial, is_multipart: bool, part_numbers: Vec, sequence_number: u32, @@ -584,7 +472,6 @@ enum ReadTransform { }, } -#[derive(Debug, Clone)] struct ReadPlan { storage_offset: usize, storage_length: i64, @@ -593,7 +480,18 @@ struct ReadPlan { } impl ReadPlan { + #[cfg(test)] async fn build(rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result { + Self::build_with_resolver(rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await + } + + async fn build_with_resolver( + rs: Option, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, + resolver: Option<&dyn ObjectEncryptionResolver>, + ) -> Result { let mut rs = rs; if let Some(part_number) = opts.part_number && rs.is_none() @@ -657,9 +555,20 @@ impl ReadPlan { } if is_encrypted { - let material = resolve_encryption_material(oi, h).await?; + let resolver = resolver.ok_or_else(|| Error::other("object encryption resolver is unavailable"))?; + let resolved = resolver + .resolve_read_material(ReadEncryptionRequest { + bucket: &oi.bucket, + object: &oi.name, + metadata: &oi.user_defined, + headers: h, + }) + .await + .map_err(Error::other)? + .ok_or_else(|| Error::other("encrypted object metadata is incomplete"))?; + let material = resolved; #[cfg(feature = "rio-v2")] - let encryption_backend = material.reader_backend; + let uses_legacy_encryption = matches!(material.mode, ReadEncryptionMode::Direct { .. }); let is_multipart = is_multipart_encrypted_object(&oi.parts, oi.etag.as_deref()); let recorded_plaintext_size = oi.encryption_original_size()?; let plaintext_size = encrypted_plaintext_size(oi, is_multipart, is_compressed, recorded_plaintext_size)?; @@ -678,7 +587,7 @@ impl ReadPlan { let (requested_offset, requested_length) = rs.get_offset_length(plaintext_size)?; #[cfg(feature = "rio-v2")] { - if encryption_backend == crate::io_support::rio::ReadEncryptionBackend::Legacy { + if uses_legacy_encryption { legacy_encrypted_range_plan( oi, is_multipart, @@ -869,32 +778,32 @@ impl ReadPlan { #[cfg(not(feature = "rio-v2"))] let _ = sequence_number; let decrypted_reader: Box = if is_multipart { - match material.key_kind { - EncryptionKeyKind::Object => crate::io_support::rio::decrypt_multipart_reader_with_object_key( + match material.mode { + ReadEncryptionMode::Object => crate::io_support::rio::decrypt_multipart_reader_with_object_key( reader, material.key_bytes, part_numbers, sequence_number, ), - EncryptionKeyKind::Direct => crate::io_support::rio::decrypt_multipart_reader( + ReadEncryptionMode::Direct { base_nonce } => crate::io_support::rio::decrypt_multipart_reader( reader, material.key_bytes, - material.base_nonce, + base_nonce, part_numbers, - material.reader_backend, + crate::io_support::rio::ReadEncryptionBackend::Legacy, sequence_number, ), } } else { - match material.key_kind { - EncryptionKeyKind::Object => { + match material.mode { + ReadEncryptionMode::Object => { crate::io_support::rio::decrypt_reader_with_object_key(reader, material.key_bytes, sequence_number) } - EncryptionKeyKind::Direct => crate::io_support::rio::decrypt_reader( + ReadEncryptionMode::Direct { base_nonce } => crate::io_support::rio::decrypt_reader( reader, material.key_bytes, - material.base_nonce, - material.reader_backend, + base_nonce, + crate::io_support::rio::ReadEncryptionBackend::Legacy, sequence_number, ), } @@ -962,14 +871,28 @@ impl ReadPlan { } impl GetObjectReader { - pub async fn new( + #[cfg(test)] + pub(crate) async fn new( reader: Box, rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap, ) -> Result<(Self, usize, i64)> { - ReadPlan::build(rs, oi, opts, h).await?.into_reader(reader, oi) + Self::new_with_resolver(reader, rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await + } + + pub async fn new_with_resolver( + reader: Box, + rs: Option, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, + resolver: Option<&dyn ObjectEncryptionResolver>, + ) -> Result<(Self, usize, i64)> { + ReadPlan::build_with_resolver(rs, oi, opts, h, resolver) + .await? + .into_reader(reader, oi) } pub async fn read_all(&mut self) -> Result> { let mut data = Vec::new(); @@ -1327,599 +1250,80 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec { parts.iter().map(|part| part.number).collect() } -fn metadata_get<'a>(metadata: &'a HashMap, key: &str) -> Option<&'a str> { - metadata.get(key).map(String::as_str).or_else(|| { - metadata - .iter() - .find_map(|(candidate, value)| candidate.eq_ignore_ascii_case(key).then_some(value.as_str())) - }) -} - -#[cfg(feature = "rio-v2")] -fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool { - matches!(cipher, DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305) -} - -#[cfg(feature = "rio-v2")] -fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result> { - let nonce = &header[4..16]; - let ciphertext = &sealed_key[DARE_HEADER_SIZE..]; - let aad = &header[..4]; - match header[1] { - DARE_CIPHER_AES_256_GCM => { - let cipher = Aes256Gcm::new_from_slice(&sealing_key) - .map_err(|err| Error::other(format!("invalid AES-GCM sealing key: {err}")))?; - let nonce = Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?; - cipher.decrypt(&nonce, Payload { msg: ciphertext, aad }) - } - DARE_CIPHER_CHACHA20_POLY1305 => { - let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key) - .map_err(|err| Error::other(format!("invalid ChaCha20-Poly1305 sealing key: {err}")))?; - let nonce = - chacha20poly1305::Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?; - cipher.decrypt(&nonce, Payload { msg: ciphertext, aad }) - } - _ => return Err(Error::other("unsupported sealed object-key DARE header")), - } - .map_err(|err| Error::other(format!("failed to unseal object key: {err}"))) -} - -async fn resolve_encryption_material(oi: &ObjectInfo, headers: &HeaderMap) -> Result { - if metadata_get(&oi.user_defined, SSEC_ALGORITHM_HEADER).is_some() { - return resolve_ssec_material(oi, headers); - } - - if contains_managed_encryption_metadata(&oi.user_defined) { - return resolve_managed_material(&oi.bucket, &oi.name, &oi.user_defined).await; - } - - Err(Error::other("encrypted object metadata is incomplete")) -} - -fn contains_managed_encryption_metadata(metadata: &HashMap) -> bool { - if metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some() { - return true; - } - - #[cfg(feature = "rio-v2")] - { - metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_some() - || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_some() - || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER).is_some() - } - - #[cfg(not(feature = "rio-v2"))] - { - false - } -} - -#[cfg(feature = "rio-v2")] -fn canonical_sse_path(bucket: &str, object: &str) -> String { - let bucket = bucket.trim_matches('/'); - let object = object.trim_matches('/'); - if object.is_empty() { - bucket.to_string() - } else if bucket.is_empty() { - object.to_string() - } else { - format!("{bucket}/{object}") - } -} - -#[cfg(feature = "rio-v2")] -fn managed_sse_domain(metadata: &HashMap) -> &'static str { - if metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_some() - || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER).is_some() - || matches!(metadata_get(metadata, "x-amz-server-side-encryption"), Some("aws:kms")) - { - "SSE-KMS" - } else if metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER).is_some() { - "SSE-C" - } else { - "SSE-S3" - } -} - -#[cfg(feature = "rio-v2")] -fn derive_sealing_key( - external_key: [u8; 32], - iv: [u8; SEALED_KEY_IV_SIZE], - domain: &str, - bucket: &str, - object: &str, -) -> [u8; 32] { - let mut mac = HmacSha256::new_from_slice(&external_key).expect("32-byte HMAC key"); - mac.update(&iv); - mac.update(domain.as_bytes()); - mac.update(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.as_bytes()); - mac.update(canonical_sse_path(bucket, object).as_bytes()); - - let mut sealing_key = [0u8; 32]; - sealing_key.copy_from_slice(mac.finalize().into_bytes().as_slice()); - sealing_key -} - -#[cfg(feature = "rio-v2")] -fn try_decode_minio_sealed_key(bytes: &str) -> Result> { - let decoded = BASE64_STANDARD - .decode(bytes) - .map_err(|e| Error::other(format!("failed to decode sealed object key: {e}")))?; - match decoded.as_slice().try_into() { - Ok(sealed_key) => Ok(Some(sealed_key)), - Err(_) => Ok(None), - } -} - -#[cfg(feature = "rio-v2")] -fn try_decode_minio_sealing_iv(bytes: &str) -> Result> { - let decoded = BASE64_STANDARD - .decode(bytes) - .map_err(|e| Error::other(format!("failed to decode sealing IV: {e}")))?; - match decoded.as_slice().try_into() { - Ok(iv) => Ok(Some(iv)), - Err(_) => Ok(None), - } -} - -#[cfg(feature = "rio-v2")] -fn try_unseal_minio_object_key( - metadata: &HashMap, - bucket: &str, - object: &str, - external_key: [u8; 32], -) -> Result> { - let Some(algorithm) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) else { - return Ok(None); - }; - if algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM { - return Ok(None); - } - - let Some(iv_b64) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER) else { - return Ok(None); - }; - let Some(iv) = try_decode_minio_sealing_iv(iv_b64)? else { - return Ok(None); - }; - - let sealed_key_b64 = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)); - let Some(sealed_key_b64) = sealed_key_b64 else { - return Ok(None); - }; - let Some(sealed_key) = try_decode_minio_sealed_key(sealed_key_b64)? else { - return Ok(None); - }; - let header = &sealed_key[..DARE_HEADER_SIZE]; - if header[0] != DARE_VERSION_20 || !is_supported_sealed_object_key_cipher(header[1]) { - return Err(Error::other("unsupported sealed object-key DARE header")); - } - if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 { - return Err(Error::other("invalid sealed object-key payload header")); - } - - let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object); - let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed_key)?; - let object_key: [u8; 32] = plaintext - .as_slice() - .try_into() - .map_err(|_| Error::other("sealed object key must decrypt to 32 bytes"))?; - Ok(Some(object_key)) -} - -fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap) -> Result { - let algorithm = headers - .get(SSEC_ALGORITHM_HEADER) - .ok_or_else(|| Error::other("missing SSE-C algorithm header"))? - .to_str() - .map_err(|_| Error::other("invalid SSE-C algorithm header"))?; - if algorithm != DEFAULT_SSE_ALGORITHM { - return Err(Error::other(format!("unsupported SSE-C algorithm {algorithm}"))); - } - - let key_b64 = headers - .get(SSEC_KEY_HEADER) - .ok_or_else(|| Error::other("missing SSE-C key header"))? - .to_str() - .map_err(|_| Error::other("invalid SSE-C key header"))?; - let key_md5 = headers - .get(SSEC_KEY_MD5_HEADER) - .ok_or_else(|| Error::other("missing SSE-C key md5 header"))? - .to_str() - .map_err(|_| Error::other("invalid SSE-C key md5 header"))?; - - let key_bytes_vec = BASE64_STANDARD - .decode(key_b64) - .map_err(|_| Error::other("failed to decode SSE-C key"))?; - let key_bytes: [u8; 32] = key_bytes_vec - .try_into() - .map_err(|_| Error::other("SSE-C key must be 32 bytes"))?; - - let expected_md5 = BASE64_STANDARD.encode(md5_bytes(key_bytes)); - if expected_md5 != key_md5 { - return Err(Error::other("SSE-C key MD5 mismatch")); - } - - let stored_md5 = - metadata_get(&oi.user_defined, SSEC_KEY_MD5_HEADER).ok_or_else(|| Error::other("missing stored SSE-C key md5"))?; - if stored_md5 != expected_md5 { - return Err(Error::other("SSE-C key does not match object metadata")); - } - - #[cfg(feature = "rio-v2")] - if let Some(object_key) = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)? { - return Ok(EncryptionMaterial { - key_bytes: object_key, - base_nonce: [0u8; 12], - key_kind: EncryptionKeyKind::Object, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2, - }); - } - - Ok(EncryptionMaterial { - key_bytes, - base_nonce: read_stored_ssec_nonce(&oi.user_defined, &oi.bucket, &oi.name), - key_kind: EncryptionKeyKind::Direct, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy, - }) -} - -/// Resolve the SSE-C Direct base nonce for decryption. -/// -/// Since #4576 the encrypt side uses a fresh random nonce per encryption and -/// persists it under `x-rustfs-encryption-iv` (plus the MinIO interop key); -/// this reader-side resolver must read that stored value back or every SSE-C -/// GET fails its first AEAD block. Legacy objects written before random -/// nonces were persisted carry no stored IV and were encrypted with the -/// deterministic `(bucket, key)` nonce, so fall back to recomputing it. Must -/// stay in lockstep with `read_stored_ssec_nonce` in rustfs/src/storage/sse.rs -/// (the API-layer twin of this resolver). -fn read_stored_ssec_nonce(metadata: &HashMap, bucket: &str, key: &str) -> [u8; 12] { - metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER)) - .and_then(|encoded| BASE64_STANDARD.decode(encoded).ok()) - .and_then(|bytes| <[u8; 12]>::try_from(bytes.as_slice()).ok()) - .unwrap_or_else(|| generate_ssec_nonce(bucket, key)) -} - -async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap) -> Result { - let normalized_metadata = normalize_managed_metadata(metadata); - let encrypted_dek = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_HEADER) - .ok_or_else(|| Error::other("missing managed encrypted DEK"))?; - let encrypted_dek = BASE64_STANDARD - .decode(encrypted_dek) - .map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?; - - let kms_key_id = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).unwrap_or("default"); - #[cfg(feature = "rio-v2")] - let kms_context = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_CONTEXT_HEADER) - .map(|value| { - serde_json::from_str::>(value) - .map_err(|e| Error::other(format!("failed to parse managed KMS context: {e}"))) - }) - .transpose()?; - #[cfg(not(feature = "rio-v2"))] - let kms_context: Option> = None; - let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref()); - - // Persisted wrapping format is the read-side source of truth. The - // advertised SSE scheme and current KMS availability are write policy - // and runtime state, neither of which identifies the historical provider. - let decrypted_key = if is_data_key_envelope(&encrypted_dek) { - let service = crate::runtime::sources::object_encryption_service() - .await - .ok_or_else(|| Error::other(KmsUnavailableError))?; - #[cfg(feature = "rio-v2")] - let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) { - service.decrypt_legacy_data_key(&encrypted_dek).await - } else { - service.decrypt_data_key(&encrypted_dek, &object_context).await - }; - #[cfg(not(feature = "rio-v2"))] - let data_key = service.decrypt_data_key(&encrypted_dek, &object_context).await; - - data_key - .map_err(|e| Error::other(format!("failed to decrypt managed data key: {e}")))? - .plaintext_key - } else { - decrypt_local_sse_dek(&encrypted_dek, kms_key_id, &object_context)? - }; - - #[cfg(feature = "rio-v2")] - if let Some(object_key) = try_unseal_minio_object_key(&normalized_metadata, bucket, object, decrypted_key)? { - return Ok(EncryptionMaterial { - key_bytes: object_key, - base_nonce: [0u8; 12], - key_kind: EncryptionKeyKind::Object, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2, - }); - } - - let iv_b64 = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_IV_HEADER) - .ok_or_else(|| Error::other("missing managed encryption IV"))?; - let iv = BASE64_STANDARD - .decode(iv_b64) - .map_err(|e| Error::other(format!("failed to decode managed encryption IV: {e}")))?; - let base_nonce: [u8; 12] = iv - .as_slice() - .try_into() - .map_err(|_| Error::other("managed encryption IV must be 12 bytes"))?; - - Ok(EncryptionMaterial { - key_bytes: decrypted_key, - base_nonce, - key_kind: EncryptionKeyKind::Direct, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy, - }) -} - -fn normalize_managed_metadata(metadata: &HashMap) -> HashMap { - #[cfg(feature = "rio-v2")] - { - let mut normalized = metadata.clone(); - if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) - { - normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.to_string()); - } - - if metadata_get(&normalized, INTERNAL_ENCRYPTION_IV_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.to_string()); - } - - if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_ID_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.to_string()); - } - - if metadata_get(&normalized, INTERNAL_ENCRYPTION_CONTEXT_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) - && let Ok(decoded) = BASE64_STANDARD.decode(value) - && let Ok(context) = serde_json::from_slice::>(&decoded) - && let Ok(encoded) = serde_json::to_string(&context) - { - normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded); - } - - normalized - } - - #[cfg(not(feature = "rio-v2"))] - { - metadata.clone() - } -} - -fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> { - if let Ok(plaintext) = decrypt_rustfs_local_sse_dek(encrypted_dek) { - return Ok(plaintext); - } - - #[cfg(feature = "rio-v2")] - { - decrypt_minio_secret_key_dek(encrypted_dek, object_context) - } - - #[cfg(not(feature = "rio-v2"))] - { - let _ = object_context; - Err(Error::other("invalid managed DEK format")) - } -} - -fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> { - let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?; - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct LocalSseDekEnvelope<'a> { - version: u8, - nonce: &'a str, - ciphertext: &'a str, - } - - let (nonce, ciphertext) = match serde_json::from_str::>(encrypted_dek) { - Ok(envelope) => { - if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION { - return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version))); - } - (envelope.nonce, envelope.ciphertext) - } - Err(_) => { - // DEPRECATED: read-only compatibility for persisted colon-delimited DEKs. - // RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs. - let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else { - return Err(Error::other("invalid managed DEK format")); - }; - if ciphertext.contains(':') { - return Err(Error::other("invalid managed DEK format")); - } - (nonce, ciphertext) - } - }; - - let nonce_vec = BASE64_STANDARD - .decode(nonce) - .map_err(|_| Error::other("invalid managed DEK nonce"))?; - let ciphertext = BASE64_STANDARD - .decode(ciphertext) - .map_err(|_| Error::other("invalid managed DEK ciphertext"))?; - - let nonce_array: [u8; 12] = nonce_vec - .as_slice() - .try_into() - .map_err(|_| Error::other("invalid managed DEK nonce length"))?; - - let key = Key::::from(local_sse_master_key()?); - let cipher = Aes256Gcm::new(&key); - let plaintext = cipher - .decrypt(&Nonce::from(nonce_array), ciphertext.as_slice()) - .map_err(|e| Error::other(format!("failed to decrypt managed DEK: {e}")))?; - - plaintext - .as_slice() - .try_into() - .map_err(|_| Error::other("managed DEK has invalid plaintext length")) -} - -#[cfg(feature = "rio-v2")] -#[derive(Deserialize)] -struct MinioLegacyCiphertext { - #[serde(rename = "aead")] - algorithm: String, - iv: Vec, - nonce: Vec, - bytes: Vec, -} - -#[cfg(feature = "rio-v2")] -fn decrypt_minio_secret_key_dek(encrypted_dek: &[u8], object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> { - let key = local_sse_master_key()?; - let (ciphertext, iv, nonce) = parse_minio_secret_key_ciphertext(encrypted_dek)?; - let associated_data = marshal_minio_kms_context(&object_context.encryption_context); - - let mut mac = HmacSha256::new_from_slice(&key).map_err(|err| Error::other(format!("invalid local SSE master key: {err}")))?; - mac.update(&iv); - let sealing_key = mac.finalize().into_bytes(); - let cipher = Aes256Gcm::new_from_slice(sealing_key.as_slice()) - .map_err(|err| Error::other(format!("invalid MinIO sealing key: {err}")))?; - let nonce = Nonce::try_from(&nonce[..]).map_err(|_| Error::other("invalid MinIO managed DEK nonce"))?; - let plaintext = cipher - .decrypt( - &nonce, - aes_gcm::aead::Payload { - msg: &ciphertext, - aad: &associated_data, - }, - ) - .map_err(|err| Error::other(format!("failed to decrypt MinIO managed DEK: {err}")))?; - - plaintext - .as_slice() - .try_into() - .map_err(|_| Error::other("MinIO managed DEK has invalid plaintext length")) -} - -#[cfg(feature = "rio-v2")] -fn parse_minio_secret_key_ciphertext( - encrypted_dek: &[u8], -) -> Result<(Vec, [u8; MINIO_SECRET_KEY_IV_SIZE], [u8; MINIO_SECRET_KEY_NONCE_SIZE])> { - if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') { - let legacy: MinioLegacyCiphertext = serde_json::from_slice(encrypted_dek) - .map_err(|err| Error::other(format!("failed to parse MinIO legacy managed DEK: {err}")))?; - if legacy.algorithm != "AES-256-GCM-HMAC-SHA-256" { - return Err(Error::other(format!( - "unsupported MinIO legacy managed DEK algorithm {}", - legacy.algorithm - ))); - } - let iv = legacy - .iv - .as_slice() - .try_into() - .map_err(|_| Error::other("invalid MinIO legacy managed DEK IV length"))?; - let nonce = legacy - .nonce - .as_slice() - .try_into() - .map_err(|_| Error::other("invalid MinIO legacy managed DEK nonce length"))?; - return Ok((legacy.bytes, iv, nonce)); - } - - if encrypted_dek.len() <= MINIO_SECRET_KEY_RANDOM_SIZE { - return Err(Error::other("invalid MinIO managed DEK length")); - } - - let split_at = encrypted_dek.len() - MINIO_SECRET_KEY_RANDOM_SIZE; - let (ciphertext, random) = encrypted_dek.split_at(split_at); - let iv = random[..MINIO_SECRET_KEY_IV_SIZE] - .try_into() - .map_err(|_| Error::other("invalid MinIO managed DEK IV length"))?; - let nonce = random[MINIO_SECRET_KEY_IV_SIZE..] - .try_into() - .map_err(|_| Error::other("invalid MinIO managed DEK nonce length"))?; - Ok((ciphertext.to_vec(), iv, nonce)) -} - -#[cfg(feature = "rio-v2")] -fn marshal_minio_kms_context(context: &HashMap) -> Vec { - let mut entries: Vec<_> = context.iter().collect(); - entries.sort_by_key(|(left, _)| *left); - - let mut json = String::from("{"); - for (index, (key, value)) in entries.into_iter().enumerate() { - if index > 0 { - json.push(','); - } - json.push_str(&serde_json::to_string(key).expect("string key serializes")); - json.push(':'); - json.push_str(&serde_json::to_string(value).expect("string value serializes")); - } - json.push('}'); - json.into_bytes() -} - -fn local_sse_master_key() -> Result<[u8; 32]> { - if let Some(key) = decode_master_key_env("__RUSTFS_SSE_SIMPLE_CMK")? { - return Ok(key); - } - - if let Some(key) = decode_master_key_env("RUSTFS_SSE_S3_MASTER_KEY")? { - return Ok(key); - } - - Ok([0u8; 32]) -} - -fn decode_master_key_env(name: &str) -> Result> { - let Ok(value) = env::var(name) else { - return Ok(None); - }; - - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - - let decoded = BASE64_STANDARD - .decode(value) - .map_err(|e| Error::other(format!("{name} is not valid base64: {e}")))?; - let key = - <[u8; 32]>::try_from(decoded.as_slice()).map_err(|_| Error::other(format!("{name} must decode to exactly 32 bytes")))?; - - Ok(Some(key)) -} - -fn generate_ssec_nonce(bucket: &str, key: &str) -> [u8; 12] { - let digest = md5_bytes(format!("{bucket}-{key}").as_bytes()); - let mut nonce = [0u8; 12]; - nonce.copy_from_slice(&digest[..12]); - nonce -} - -fn md5_bytes(data: impl AsRef<[u8]>) -> [u8; 16] { - let digest = Md5::digest(data.as_ref()); - let mut out = [0u8; 16]; - out.copy_from_slice(&digest); - out -} - #[cfg(test)] mod tests { use super::*; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use md5::{Digest, Md5}; + use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER}; + use std::collections::HashMap; use std::io::Cursor; use temp_env::async_with_vars; use tokio::io::AsyncReadExt; + const TEST_DIRECT_KEY_HEADER: &str = "x-rustfs-test-direct-key"; + const TEST_OBJECT_KEY_HEADER: &str = "x-rustfs-test-object-key"; + const TEST_NONCE_HEADER: &str = "x-rustfs-test-nonce"; + + pub(super) static TEST_RESOLVER: TestObjectEncryptionResolver = TestObjectEncryptionResolver; + + pub(super) struct TestObjectEncryptionResolver; + + #[async_trait::async_trait] + impl ObjectEncryptionResolver for TestObjectEncryptionResolver { + async fn resolve_read_material( + &self, + request: ReadEncryptionRequest<'_>, + ) -> std::result::Result, EncryptionResolutionError> { + if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) { + let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key") + })?; + let key_bytes = decoded.try_into().map_err(|_| { + EncryptionResolutionError::new( + EncryptionResolutionErrorKind::InvalidMetadata, + "invalid test object key length", + ) + })?; + return Ok(Some(ReadEncryptionMaterial { + key_bytes, + mode: ReadEncryptionMode::Object, + })); + } + + let encoded = request + .headers + .get(TEST_DIRECT_KEY_HEADER) + .ok_or_else(|| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "missing test direct key") + })? + .to_str() + .map_err(|_| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key") + })?; + let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key") + })?; + let key_bytes = decoded.try_into().map_err(|_| { + EncryptionResolutionError::new( + EncryptionResolutionErrorKind::InvalidRequest, + "invalid test encryption key length", + ) + })?; + let base_nonce = request + .metadata + .get(TEST_NONCE_HEADER) + .and_then(|encoded| BASE64_STANDARD.decode(encoded).ok()) + .and_then(|bytes| bytes.try_into().ok()) + .unwrap_or_else(|| fixture_nonce(request.bucket, request.object)); + Ok(Some(ReadEncryptionMaterial { + key_bytes, + mode: ReadEncryptionMode::Direct { base_nonce }, + })) + } + } + fn md5_bytes(data: impl AsRef<[u8]>) -> [u8; 16] { let digest = Md5::digest(data.as_ref()); let mut bytes = [0u8; 16]; @@ -1927,6 +1331,22 @@ mod tests { bytes } + fn fixture_nonce(bucket: &str, object: &str) -> [u8; 12] { + let digest = md5_bytes(format!("{bucket}-{object}")); + let mut nonce = [0; 12]; + nonce.copy_from_slice(&digest[..12]); + nonce + } + + fn ssec_headers_from_key(key_bytes: [u8; 32]) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + TEST_DIRECT_KEY_HEADER, + HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"), + ); + headers + } + #[tokio::test] async fn cache_body_uses_plaintext_length_for_compressed_metadata() { let mut metadata = HashMap::new(); @@ -1961,104 +1381,6 @@ mod tests { assert_eq!(restored, body); } - /// Regression for the #4576 fallout: the encrypt side persists a random - /// SSE-C nonce, and this reader-side resolver must read it back — falling - /// back to the deterministic legacy nonce only when no IV was stored. - /// Reverting the stored-nonce lookup breaks the first two cases. - #[test] - fn read_stored_ssec_nonce_prefers_persisted_iv_and_falls_back_for_legacy() { - let stored = [7u8; 12]; - let deterministic = generate_ssec_nonce("bucket", "object"); - assert_ne!(stored, deterministic, "test nonce must differ from the deterministic value"); - - let mut metadata = HashMap::new(); - metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(stored)); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), stored); - - // MinIO interop key only, in non-canonical casing: the lookup is - // case-insensitive like every other internal-metadata read here. - let mut metadata = HashMap::new(); - metadata.insert(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_ascii_lowercase(), BASE64_STANDARD.encode(stored)); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), stored); - - // Legacy object: no stored IV → deterministic fallback. - assert_eq!(read_stored_ssec_nonce(&HashMap::new(), "bucket", "object"), deterministic); - - // Corrupt values (bad base64 / wrong length) also fall back instead of erroring. - let mut metadata = HashMap::new(); - metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "not-base64!!".to_string()); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), deterministic); - let mut metadata = HashMap::new(); - metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([1u8; 8])); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), deterministic); - } - - fn ssec_headers_from_key(key_bytes: [u8; 32]) -> HeaderMap { - let mut headers = HeaderMap::new(); - headers.insert(SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); - headers.insert( - SSEC_KEY_HEADER, - HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("valid base64 header"), - ); - headers.insert( - SSEC_KEY_MD5_HEADER, - HeaderValue::from_str(&BASE64_STANDARD.encode(md5_bytes(key_bytes))).expect("valid md5 header"), - ); - headers - } - - #[cfg(feature = "rio-v2")] - #[test] - fn test_legacy_managed_metadata_excludes_sealed_keys() { - let legacy_metadata = HashMap::from([ - (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()), - ]); - assert!(is_legacy_rustfs_managed_metadata(&legacy_metadata)); - - let sealed_metadata = HashMap::from([ - (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()), - (MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), "sealed-key".to_string()), - ]); - - assert!(!is_legacy_rustfs_managed_metadata(&sealed_metadata)); - } - - #[cfg(feature = "rio-v2")] - fn seal_ssec_object_key_for_test( - bucket: &str, - object: &str, - customer_key: [u8; 32], - object_key: [u8; 32], - ) -> ([u8; 32], Vec) { - let iv = [0x23u8; SEALED_KEY_IV_SIZE]; - let sealing_key = derive_sealing_key(customer_key, iv, "SSE-C", bucket, object); - let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key"); - - let mut header = [0u8; DARE_HEADER_SIZE]; - header[0] = DARE_VERSION_20; - header[1] = DARE_CIPHER_AES_256_GCM; - header[2..4].copy_from_slice(&31u16.to_le_bytes()); - header[4] = 0x80; - header[5..16].copy_from_slice(&[0x45u8; 11]); - - let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce"); - let mut sealed = header.to_vec(); - sealed.extend_from_slice( - &cipher - .encrypt( - &nonce, - aes_gcm::aead::Payload { - msg: &object_key, - aad: &header[..4], - }, - ) - .expect("seal object key"), - ); - (iv, sealed) - } - #[tokio::test] async fn test_ranged_decompress_reader() { // Create test data @@ -2342,261 +1664,6 @@ mod tests { assert_eq!(actual, b"fghijkl"); } - fn encrypt_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String { - let key = Key::::from(master_key); - let cipher = Aes256Gcm::new(&key); - let nonce = Nonce::from([0u8; 12]); - let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek"); - serde_json::json!({ - "version": LOCAL_SSE_DEK_FORMAT_VERSION, - "nonce": BASE64_STANDARD.encode(nonce), - "ciphertext": BASE64_STANDARD.encode(ciphertext), - }) - .to_string() - } - - fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String { - let key = Key::::from(master_key); - let cipher = Aes256Gcm::new(&key); - let nonce = Nonce::from([0u8; 12]); - let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek"); - format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext)) - } - - #[test] - fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() { - let envelope = serde_json::json!({ - "version": LOCAL_SSE_DEK_FORMAT_VERSION + 1, - "nonce": BASE64_STANDARD.encode([0u8; 12]), - "ciphertext": BASE64_STANDARD.encode([0u8; 48]), - }) - .to_string(); - - let error = - decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed"); - assert!(error.to_string().contains("unsupported managed DEK format version")); - } - - #[cfg(feature = "rio-v2")] - fn seal_managed_s3_object_key_for_test( - bucket: &str, - object: &str, - data_key: [u8; 32], - object_key: [u8; 32], - ) -> ([u8; 32], Vec) { - seal_managed_s3_object_key_for_test_with_cipher(bucket, object, data_key, object_key, DARE_CIPHER_AES_256_GCM) - } - - #[cfg(feature = "rio-v2")] - fn seal_managed_s3_object_key_for_test_with_cipher( - bucket: &str, - object: &str, - data_key: [u8; 32], - object_key: [u8; 32], - cipher_id: u8, - ) -> ([u8; 32], Vec) { - let iv = [0x24u8; SEALED_KEY_IV_SIZE]; - let sealing_key = derive_sealing_key(data_key, iv, "SSE-S3", bucket, object); - - let mut header = [0u8; DARE_HEADER_SIZE]; - header[0] = DARE_VERSION_20; - header[1] = cipher_id; - header[2..4].copy_from_slice(&31u16.to_le_bytes()); - header[4] = 0x80; - header[5..16].copy_from_slice(&[0x46u8; 11]); - - let ciphertext = match cipher_id { - DARE_CIPHER_AES_256_GCM => { - let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key"); - let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce"); - cipher - .encrypt( - &nonce, - Payload { - msg: &object_key, - aad: &header[..4], - }, - ) - .expect("seal managed object key") - } - DARE_CIPHER_CHACHA20_POLY1305 => { - let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key).expect("valid sealing key"); - let nonce = chacha20poly1305::Nonce::try_from(&header[4..16]).expect("valid nonce"); - cipher - .encrypt( - &nonce, - Payload { - msg: &object_key, - aad: &header[..4], - }, - ) - .expect("seal managed object key") - } - _ => panic!("unsupported test cipher"), - }; - let mut sealed = header.to_vec(); - sealed.extend_from_slice(&ciphertext); - (iv, sealed) - } - - #[cfg(feature = "rio-v2")] - #[test] - fn test_supported_sealed_object_key_cipher_accepts_current_minio_fixture_value() { - assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_AES_256_GCM)); - assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_CHACHA20_POLY1305)); - assert!(!is_supported_sealed_object_key_cipher(0x02)); - } - - #[tokio::test] - async fn resolve_managed_material_accepts_case_insensitive_metadata_keys() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let data_key = [0x24; 32]; - let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let metadata = HashMap::from([ - ("X-Rustfs-Encryption-Key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("X-Rustfs-Encryption-IV".to_string(), BASE64_STANDARD.encode(base_nonce)), - ]); - - let material = resolve_managed_material("", "", &metadata) - .await - .expect("managed material should resolve mixed-case metadata keys"); - - assert_eq!(material.key_bytes, data_key); - assert_eq!(material.base_nonce, base_nonce); - }) - .await; - } - - #[tokio::test] - async fn resolve_managed_material_selects_provider_from_persisted_dek() { - use rustfs_kms::KmsConfig; - use tempfile::TempDir; - - let key_dir = TempDir::new().expect("create KMS key directory"); - let manager = rustfs_kms::init_global_kms_service_manager(); - manager - .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("start test KMS service"); - - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async { - let data_key = [0x24; 32]; - let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]); - let metadata = HashMap::from([ - ( - INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(base_nonce)), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "legacy-local-key".to_string()), - ]); - - let material = resolve_managed_material("bucket", "object", &metadata) - .await - .expect("legacy local DEK should not be routed to the running KMS"); - assert_eq!(material.key_bytes, data_key); - assert_eq!(material.base_nonce, base_nonce); - }) - .await; - - manager.stop().await.expect("stop test KMS service"); - - let kms_envelope = br#"{ - "key_id": "test-key-id", - "master_key_id": "master-key-id", - "key_spec": "AES_256", - "encrypted_key": [1, 2, 3, 4], - "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - "encryption_context": {}, - "created_at": "2024-01-01T00:00:00+00:00" - }"#; - let metadata = HashMap::from([ - (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(kms_envelope)), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x14; 12])), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "test-key-id".to_string()), - ]); - let error = match resolve_managed_material("bucket", "object", &metadata).await { - Ok(_) => panic!("KMS envelope must not fall back to the local provider"), - Err(error) => error, - }; - let Error::Io(io_error) = error else { - panic!("KMS absence should retain its typed source"); - }; - assert!( - io_error - .get_ref() - .and_then(|source| source.downcast_ref::()) - .is_some() - ); - } - - #[cfg(feature = "rio-v2")] - #[tokio::test] - async fn resolve_managed_material_accepts_chacha20_poly1305_header_variant() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let data_key = [0x24; 32]; - let object_key = [0x33; 32]; - let (iv, sealed_key) = seal_managed_s3_object_key_for_test_with_cipher( - "bucket", - "object", - data_key, - object_key, - DARE_CIPHER_CHACHA20_POLY1305, - ); - - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let metadata = HashMap::from([ - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(iv)), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - ( - MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), - ]); - - let material = resolve_managed_material("bucket", "object", &metadata) - .await - .expect("managed material should accept current MinIO header variant"); - assert_eq!(material.key_kind, EncryptionKeyKind::Object); - assert_eq!(material.key_bytes, object_key); - }) - .await; - } - - #[tokio::test] - async fn resolve_encryption_material_accepts_case_insensitive_metadata_keys() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let data_key = [0x24; 32]; - let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let metadata = HashMap::from([ - ("X-Rustfs-Encryption-Key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("X-Rustfs-Encryption-IV".to_string(), BASE64_STANDARD.encode(base_nonce)), - ]); - let object_info = ObjectInfo { - user_defined: Arc::new(metadata), - ..Default::default() - }; - let material = resolve_encryption_material(&object_info, &HeaderMap::new()) - .await - .expect("resolve_encryption_material should accept mixed-case managed metadata"); - - assert_eq!(material.key_bytes, data_key); - assert_eq!(material.base_nonce, base_nonce); - }) - .await; - } - #[tokio::test] async fn test_get_object_reader_rejects_ssec_read_without_headers() { let object_info = ObjectInfo { @@ -2791,310 +1858,6 @@ mod tests { )); } - #[tokio::test] - async fn test_get_object_reader_allows_encrypted_full_object_passthrough() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let plaintext = b"managed-full-object".to_vec(); - let data_key = [0x21; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-full-object"; - - let mut encrypted = Vec::new(); - #[cfg(feature = "rio-v2")] - let user_defined = { - let object_key = [0x41; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - ]) - }; - #[cfg(not(feature = "rio-v2"))] - let user_defined = { - let base_nonce = [0x11; 12]; - crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ]) - }; - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(user_defined), - ..Default::default() - }; - - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted.clone())), - None, - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted full-object reads should decrypt inside ecstore"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed plaintext"); - - assert_eq!(offset, 0); - assert_eq!(length, object_info.size); - assert_eq!(reader.object_info.size, plaintext.len() as i64); - assert_eq!(actual, plaintext); - }) - .await; - } - - #[tokio::test] - async fn test_get_object_reader_decrypts_managed_sse_range_on_plaintext_semantics() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let plaintext = b"0123456789abcdefghijklmnopqrstuvwxyz".to_vec(); - let data_key = [0x23; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-range-object"; - - let mut encrypted = Vec::new(); - #[cfg(feature = "rio-v2")] - let user_defined = { - let object_key = [0x43; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed ranged object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - ]) - }; - #[cfg(not(feature = "rio-v2"))] - let user_defined = { - let base_nonce = [0x13; 12]; - crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed ranged object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ]) - }; - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(user_defined), - ..Default::default() - }; - let range = HTTPRangeSpec { - is_suffix_length: false, - start: 5, - end: 11, - }; - - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted.clone())), - Some(range), - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted range reads should decrypt inside ecstore"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed ranged plaintext"); - - assert_eq!(offset, 0); - assert_eq!(length, encrypted.len() as i64); - assert_eq!(reader.object_info.size, 7); - assert_eq!(actual, b"56789ab"); - }) - .await; - } - - #[tokio::test] - async fn test_get_object_reader_uses_local_managed_fallback_with_explicit_sse_s3_key() { - async_with_vars( - [ - ("__RUSTFS_SSE_SIMPLE_CMK", None::), - ("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode([0u8; 32]))), - ], - async { - let plaintext = b"managed-local-fallback".to_vec(); - let data_key = [0x22; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-local-fallback"; - - let mut encrypted = Vec::new(); - #[cfg(feature = "rio-v2")] - let user_defined = { - let object_key = [0x42; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object with local fallback key"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - ]) - }; - #[cfg(not(feature = "rio-v2"))] - let user_defined = { - let base_nonce = [0x12; 12]; - crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object with local fallback key"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ]) - }; - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(user_defined), - ..Default::default() - }; - - let (mut reader, _, _) = GetObjectReader::new( - Box::new(Cursor::new(encrypted)), - None, - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted reads should use the configured local SSE-S3 key"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed plaintext"); - - assert_eq!(reader.object_info.size, plaintext.len() as i64); - assert_eq!(actual, plaintext); - }, - ) - .await; - } - - #[cfg(feature = "rio-v2")] - #[tokio::test] - async fn test_get_object_reader_accepts_minio_only_managed_metadata() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let plaintext = b"managed-minio-metadata".to_vec(); - let data_key = [0x23; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-minio-metadata"; - let object_key = [0x44; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - - let mut encrypted = Vec::new(); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), - ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()), - ])), - ..Default::default() - }; - - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted.clone())), - None, - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted reads should accept MinIO-style metadata"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed plaintext"); - - assert_eq!(offset, 0); - assert_eq!(length, object_info.size); - assert_eq!(reader.object_info.size, plaintext.len() as i64); - assert_eq!(actual, plaintext); - }) - .await; - } - #[tokio::test] async fn test_get_object_reader_compressed_range_returns_physical_offset_from_index() { let mut index = Index::new(); @@ -3384,8 +2147,6 @@ mod tests { let object_key = [0x67; 32]; let bucket = "bucket"; let object = "sealed-object"; - let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); - let mut encrypted = Vec::new(); crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) @@ -3397,6 +2158,7 @@ mod tests { name: object.to_string(), size: encrypted.len() as i64, user_defined: Arc::new(HashMap::from([ + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), @@ -3406,15 +2168,6 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), plaintext.len().to_string(), ), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), ])), ..Default::default() }; @@ -3623,10 +2376,7 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), total_plaintext.to_string(), ), - ( - INTERNAL_ENCRYPTION_IV_HEADER.to_string(), - BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE), - ), + (TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)), ]) } @@ -4005,65 +2755,6 @@ mod tests { .await; } - #[tokio::test] - async fn test_legacy_managed_multipart_range_seek_byte_exact() { - async_with_vars( - [ - ("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32]))), - (ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true".to_string())), - ], - async { - let data_key = [0x74; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let total_plaintext: usize = 20_000 + 9_000 + 5_000; - let metadata = HashMap::from([ - ( - INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()), - ( - INTERNAL_ENCRYPTION_IV_HEADER.to_string(), - BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE), - ), - (INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), total_plaintext.to_string()), - ]); - let fixture = - build_legacy_multipart_fixture("bucket", "managed-multipart", data_key, &[20_000, 9_000, 5_000], metadata) - .await; - let headers = HeaderMap::new(); - let opts = ObjectOptions::default(); - - for (rs, expected_offset, expected_length, label) in [ - ( - range(33_900, 33_999), - fixture.physical_part_start(2), - fixture.part_physical_sizes[2] as i64, - "managed tail range", - ), - ( - range(19_990, 20_010), - 0, - (fixture.part_physical_sizes[0] + fixture.part_physical_sizes[1]) as i64, - "managed boundary straddle", - ), - ] { - let (start, len) = rs.get_offset_length(total_plaintext as i64).expect("valid managed range"); - let expected_body = - &fixture.plaintext[start..start + usize::try_from(len).expect("valid managed range length fits usize")]; - - let (body, offset, length, reported_size) = read_via_seek_window(&fixture, Some(rs), &opts, &headers).await; - - assert_eq!(offset, expected_offset, "{label}: physical offset"); - assert_eq!(length, expected_length, "{label}: physical length"); - assert_eq!(reported_size, len, "{label}: reported plaintext size"); - assert_eq!(body, expected_body, "{label}: body bytes"); - } - }, - ) - .await; - } - #[tokio::test] async fn test_legacy_single_part_multipart_object_keeps_full_read_shape() { async_with_vars([(ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { @@ -4299,50 +2990,6 @@ mod tests { .await; } - #[tokio::test] - async fn test_legacy_ssec_multipart_range_rejects_wrong_or_missing_key() { - async_with_vars([(ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { - let key_bytes = [0x79; 32]; - let fixture = build_legacy_ssec_multipart_fixture(key_bytes, &[20_000, 9_000, 5_000]).await; - let rs = range(33_900, 33_999); - - let missing = match GetObjectReader::new( - Box::new(Cursor::new(fixture.ciphertext.clone())), - Some(rs.clone()), - &fixture.object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - { - Ok(_) => panic!("missing SSE-C key must fail before any body is produced"), - Err(err) => err, - }; - assert!( - missing.to_string().contains("SSE-C"), - "missing-key failure must come from SSE-C validation: {missing}" - ); - - let wrong = match GetObjectReader::new( - Box::new(Cursor::new(fixture.ciphertext.clone())), - Some(rs), - &fixture.object_info, - &ObjectOptions::default(), - &ssec_headers_from_key([0x00; 32]), - ) - .await - { - Ok(_) => panic!("wrong SSE-C key must fail before any body is produced"), - Err(err) => err, - }; - assert!( - wrong.to_string().contains("SSE-C key does not match object metadata"), - "wrong-key failure must come from the stored key check: {wrong}" - ); - }) - .await; - } - #[tokio::test] async fn test_legacy_ssec_multipart_seek_tamper_fails_hard_with_no_plaintext() { async_with_vars([(ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { @@ -4394,8 +3041,6 @@ mod tests { let object_key = [0x68; 32]; let bucket = "bucket"; let object = "large-range-object"; - let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); - let mut encrypted = Vec::new(); crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) @@ -4407,6 +3052,7 @@ mod tests { name: object.to_string(), size: encrypted.len() as i64, user_defined: Arc::new(HashMap::from([ + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), @@ -4416,15 +3062,6 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), plaintext.len().to_string(), ), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), ])), ..Default::default() }; @@ -4536,7 +3173,6 @@ mod tests { let object_key = [0x74; 32]; let bucket = "bucket"; let object = "compressed-large-object"; - let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); let mut compressor = crate::io_support::rio::CompressReader::with_encrypted_padding( Cursor::new(plaintext.clone()), CompressionAlgorithm::default(), @@ -4629,6 +3265,7 @@ mod tests { ..Default::default() }]), user_defined: Arc::new(HashMap::from([ + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), @@ -4638,15 +3275,6 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), plaintext.len().to_string(), ), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), ( "x-minio-internal-compression".to_string(), crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), @@ -4680,7 +3308,7 @@ mod tests { assert_eq!(plaintext_offset as i64, range.start - uncomp_off); assert_eq!(plaintext_length, 64); } - other => panic!("expected encrypted read plan, got {other:?}"), + _ => panic!("expected encrypted read plan"), } let (mut reader, offset, length) = GetObjectReader::new( diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index dfd403e18..0248d76b2 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -273,29 +273,9 @@ impl ObjectInfo { } pub fn is_encrypted(&self) -> bool { - // Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function - use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; - - self.user_defined.keys().any(|key| { - let lower = key.to_ascii_lowercase(); - lower.starts_with("x-minio-encryption-") - || lower.starts_with("x-minio-internal-server-side-encryption-") - || matches!( - lower.as_str(), - "x-minio-internal-encrypted-multipart" - | "x-rustfs-encryption-key" - | "x-rustfs-encryption-algorithm" - | "x-rustfs-encryption-iv" - | "x-rustfs-encryption-key-id" - | "x-rustfs-encryption-context" - | "x-rustfs-encryption-tag" - | "x-amz-server-side-encryption-aws-kms-key-id" - | SSEC_ALGORITHM_HEADER - | SSEC_KEY_HEADER - | SSEC_KEY_MD5_HEADER - | "x-amz-server-side-encryption" - ) - }) + self.user_defined + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) } /// Maximum inline size for non-versioned objects (128 KiB). @@ -339,26 +319,7 @@ impl ObjectInfo { } pub fn encryption_original_size(&self) -> std::io::Result> { - let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE); - if let Some(size_str) = self - .user_defined - .get("x-rustfs-encryption-original-size") - .map(String::as_str) - .or_else(|| { - self.user_defined - .get("x-amz-server-side-encryption-customer-original-size") - .map(String::as_str) - }) - .or(actual_size.as_deref()) - && !size_str.is_empty() - { - let size = size_str - .parse::() - .map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?; - return Ok(Some(size)); - } - - Ok(None) + rustfs_utils::http::get_object_encryption_original_size(&self.user_defined) } pub fn decrypted_size(&self) -> std::io::Result { @@ -388,9 +349,6 @@ impl ObjectInfo { return Ok(actual_size); } - // Check if object is encrypted - // Managed SSE stores original size in x-rustfs-encryption-original-size metadata - // SSE-C stores original size in x-amz-server-side-encryption-customer-original-size if let Some(size) = self.encryption_original_size()? { return Ok(size); } @@ -881,6 +839,19 @@ mod tests { assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back"); } + #[test] + fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() { + let object = ObjectInfo { + user_defined: Arc::new(HashMap::from([( + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(), + "sealed".to_string(), + )])), + ..Default::default() + }; + + assert!(object.is_encrypted()); + } + #[test] fn versions_after_marker_handles_null_version_marker() { let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap(); diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 2c8ab72c6..ed0bda1dd 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys; use crate::bucket::replication::{DynReplicationPool, ReplicationStats}; use crate::disk::DiskStore; use crate::layout::endpoints::{EndpointServerPools, SetupType}; +use crate::object_api::ObjectEncryptionResolver; use crate::services::event_notification::EventNotifier; use crate::services::tier::tier::TierConfigMgr; use rustfs_lock::{GlobalLockManager, get_global_lock_manager}; @@ -159,6 +160,8 @@ pub struct InstanceContext { /// workers (scanner/heal/tier/lifecycle) without touching another instance. /// Replaces the process-global cancel-token static. background_cancel_token: OnceLock, + /// Resolves object-encryption material at the application boundary. + object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, transition_transaction_recovery_stores: std::sync::Mutex>, #[cfg(test)] @@ -197,6 +200,7 @@ impl InstanceContext { local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), bucket_metadata_sys: std::sync::Mutex::new(None), background_cancel_token: OnceLock::new(), + object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), #[cfg(test)] @@ -209,6 +213,19 @@ impl InstanceContext { self.lock_manager.clone() } + /// Install the application-owned object-encryption resolver once. + pub fn set_object_encryption_resolver( + &self, + resolver: Arc, + ) -> Result<(), Arc> { + self.object_encryption_resolver.set(resolver) + } + + /// Return the configured object-encryption resolver, if startup installed one. + pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> { + self.object_encryption_resolver.get().map(Arc::as_ref) + } + /// Set this instance's S3 region. /// /// Write-once: panics on a second write, preserving the startup fail-fast diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index 8a16f0c26..b85d6c395 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -46,7 +46,6 @@ use crate::{ use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; -use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service}; use rustfs_lock::client::LockClient; use s3s::dto::BucketLifecycleConfiguration; use s3s::region::Region; @@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_ global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error); } -pub(crate) async fn object_encryption_service() -> Option> { - get_global_encryption_service().await -} - pub fn object_store_handle() -> Option> { resolve_object_store_handle() } diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs index e1cfea0e0..b19d502a0 100644 --- a/crates/ecstore/src/set_disk/metadata.rs +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -505,9 +505,7 @@ impl SetDisks { } fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool { - meta.metadata - .keys() - .any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name)) + meta.metadata.keys().any(|name| http::is_object_encryption_marker(name)) } fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index d356108d3..6ff7488a2 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -147,15 +147,17 @@ use rustfs_object_capacity::capacity_scope::{ CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope, }; use rustfs_s3_types::EventName; +#[cfg(test)] +use rustfs_utils::http::SSEC_ALGORITHM_HEADER; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; use rustfs_utils::http::{ - SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, - SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str, - get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map, + SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, + SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker, + remove_header_map, }; use rustfs_utils::{ HashAlgorithm, @@ -670,10 +672,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) -> bool { - metadata.keys().any(|key| is_encryption_metadata_key(key)) - || metadata.contains_key(SSEC_ALGORITHM_HEADER) - || metadata.contains_key(SSEC_KEY_HEADER) - || metadata.contains_key(SSEC_KEY_MD5_HEADER) + metadata.keys().any(|key| is_object_encryption_marker(key)) } /// Per-set memoized capacity dirty scope. diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 90f84db95..30cd9f94a 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -42,6 +42,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; use crate::store::ECStore; use futures::FutureExt as _; +use http::HeaderValue; use std::future::Future; fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result { @@ -49,6 +50,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result, + range: Option, + object_info: &ObjectInfo, + opts: &ObjectOptions, + headers: &HeaderMap, +) -> Result<(GetObjectReader, usize, i64)> { + GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await +} + /// Length of the full plaintext body when — and only when — this read's output /// is exactly the object's complete plaintext, so the app-layer body cache may /// serve it in place of the erasure read. @@ -713,7 +725,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { size_bucket, ); record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket); - let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?; + let (mut reader, _offset, _length) = + get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?; // Carry the hook probe result so the app layer skips its // now-redundant lookup on the streaming miss path (ODC-16). reader.body_source = body_source; @@ -745,7 +758,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let (rd, wd) = tokio::io::duplex(duplex_buffer_size); debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer"); - let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?; + let (mut reader, offset, length) = + get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?; // Carry the hook probe result so the app layer skips its now-redundant // lookup on the streaming miss path (ODC-16). reader.body_source = body_source; @@ -4536,6 +4550,61 @@ mod erasure_construction_tests { } } +#[cfg(test)] +mod object_encryption_resolver_wiring_tests { + use super::*; + use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest}; + use std::io::Cursor; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CountingResolver { + calls: AtomicUsize, + } + + #[async_trait::async_trait] + impl ObjectEncryptionResolver for CountingResolver { + async fn resolve_read_material( + &self, + _request: ReadEncryptionRequest<'_>, + ) -> std::result::Result, EncryptionResolutionError> { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok(None) + } + } + + #[tokio::test] + async fn get_object_reader_forwards_instance_resolver() { + let resolver = Arc::new(CountingResolver { + calls: AtomicUsize::new(0), + }); + let ctx = InstanceContext::new(); + assert!( + ctx.set_object_encryption_resolver(resolver.clone()).is_ok(), + "fresh context should accept resolver" + ); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 1, + user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])), + ..Default::default() + }; + + let result = get_object_reader_with_context( + &ctx, + Box::new(Cursor::new(Vec::::new())), + None, + &object_info, + &ObjectOptions::default(), + &HeaderMap::new(), + ) + .await; + + assert!(result.is_err(), "resolver returning no material must fail closed"); + assert_eq!(resolver.calls.load(Ordering::Relaxed), 1); + } +} + #[cfg(test)] pub(in crate::set_disk::ops) mod hermetic_set_disks_support { //! Shared hermetic `SetDisks` construction for the ops tests below: the diff --git a/crates/ecstore/tests/README.md b/crates/ecstore/tests/README.md index 0833a467d..1514ba7f3 100644 --- a/crates/ecstore/tests/README.md +++ b/crates/ecstore/tests/README.md @@ -2,7 +2,7 @@ ## MinIO-generated encrypted fixtures -`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by +`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by `.\rustfs\scripts\minio_fixture_lab\lab.py`. It currently covers multipart fixtures for: @@ -20,5 +20,5 @@ Example: ```powershell $env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key' $env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '' -cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored +cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored ``` diff --git a/crates/kms/AGENTS.md b/crates/kms/AGENTS.md index 3f6cb014a..5555c8071 100644 --- a/crates/kms/AGENTS.md +++ b/crates/kms/AGENTS.md @@ -25,3 +25,25 @@ For local KMS end-to-end tests, keep proxy bypass settings: NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1 ``` + +## Local Key Export for SSE-S3 Migration Tests + +Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local +KMS key as the base64 value expected by `RUSTFS_SSE_S3_MASTER_KEY`: + +```bash +export RUSTFS_KMS_LOCAL_MASTER_KEY='' +export RUSTFS_SSE_S3_MASTER_KEY="$( + cargo run -q -p rustfs-kms --example local_kms_key_decrypt -- \ + /absolute/path/to/.key +)" +``` + +For a `plaintext-dev-only` Local KMS key file, +`RUSTFS_KMS_LOCAL_MASTER_KEY` is not required. + +The example writes only the base64-encoded 32-byte key to stdout. Diagnostics +go to stderr. Never paste its output into logs, shell history, issue comments, +or committed configuration. The export path must remain read-only and must +reuse `LocalKmsClient` decoding so current Argon2id and legacy key-file +compatibility stay aligned with the backend. diff --git a/crates/kms/examples/local_kms_key_decrypt.rs b/crates/kms/examples/local_kms_key_decrypt.rs new file mode 100644 index 000000000..37f174ccc --- /dev/null +++ b/crates/kms/examples/local_kms_key_decrypt.rs @@ -0,0 +1,112 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use zeroize::Zeroizing; + +const LOCAL_KMS_MASTER_KEY_ENV: &str = "RUSTFS_KMS_LOCAL_MASTER_KEY"; + +fn usage(program: &str) -> String { + format!( + "Usage: {program} \n\ + Reads {LOCAL_KMS_MASTER_KEY_ENV} when the key file is encrypted.\n\ + Writes only the base64-encoded 32-byte key to stdout." + ) +} + +fn resolve_key_file(path: &Path) -> Result<(PathBuf, String), String> { + let canonical = std::fs::canonicalize(path).map_err(|error| format!("cannot open Local KMS key file: {error}"))?; + if canonical.extension().and_then(|extension| extension.to_str()) != Some("key") { + return Err("Local KMS key file must have a .key extension".to_string()); + } + let key_dir = canonical + .parent() + .ok_or_else(|| "Local KMS key file must have a parent directory".to_string())? + .to_path_buf(); + let key_id = canonical + .file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| !stem.is_empty()) + .ok_or_else(|| "Local KMS key file name must contain a valid UTF-8 key ID".to_string())? + .to_string(); + Ok((key_dir, key_id)) +} + +async fn run() -> Result<(), String> { + let mut args = std::env::args(); + let program = args.next().unwrap_or_else(|| "local_kms_key_decrypt".to_string()); + let Some(key_file) = args.next() else { + return Err(usage(&program)); + }; + if args.next().is_some() { + return Err(usage(&program)); + } + + let (key_dir, key_id) = resolve_key_file(Path::new(&key_file))?; + let master_key = std::env::var(LOCAL_KMS_MASTER_KEY_ENV).ok().filter(|value| !value.is_empty()); + let client = LocalKmsClient::new_for_key_export(LocalConfig { + key_dir, + master_key, + file_permissions: Some(0o600), + }) + .await + .map_err(|error| error.to_string())?; + let key_material = client + .decrypt_key_material_for_export(&key_id) + .await + .map_err(|error| error.to_string())?; + let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref())); + + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}")) +} + +#[tokio::main] +async fn main() { + if let Err(error) = run().await { + let _ = writeln!(io::stderr().lock(), "local_kms_key_decrypt: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolve_key_file_extracts_directory_and_key_id() { + let directory = tempfile::tempdir().expect("create temporary directory"); + let key_file = directory.path().join("migration-key.key"); + std::fs::write(&key_file, b"{}").expect("create key file"); + + let (key_dir, key_id) = resolve_key_file(&key_file).expect("resolve key file"); + + assert_eq!(key_dir, directory.path().canonicalize().expect("canonical directory")); + assert_eq!(key_id, "migration-key"); + } + + #[test] + fn resolve_key_file_rejects_non_key_extension() { + let directory = tempfile::tempdir().expect("create temporary directory"); + let key_file = directory.path().join("migration-key.json"); + std::fs::write(&key_file, b"{}").expect("create key file"); + + let error = resolve_key_file(&key_file).expect_err("non-key file must be rejected"); + + assert!(error.contains(".key")); + } +} diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index ada43f00b..60081963f 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -36,6 +36,7 @@ use std::path::{Component, Path, PathBuf}; use std::time::Duration; use tokio::fs; use tracing::{debug, warn}; +use zeroize::Zeroizing; /// Reject key identifiers that would not name a single file directly inside the key /// directory. @@ -146,6 +147,45 @@ impl LocalKmsClient { Ok(client) } + /// Open a Local KMS key directory without creating or modifying any files. + /// + /// This constructor is restricted to explicit key-export tooling. Normal + /// backend operation must use [`Self::new`]. + pub async fn new_for_key_export(config: LocalConfig) -> Result { + if !fs::try_exists(&config.key_dir).await? { + return Err(KmsError::configuration_error("Local KMS key directory does not exist")); + } + + let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key { + let legacy_key = Self::derive_legacy_master_key(master_key)?; + let legacy_master_cipher = Aes256Gcm::new(&legacy_key); + let salt_path = Self::master_key_salt_path(&config); + let master_cipher = if fs::try_exists(&salt_path).await? { + let salt = fs::read(&salt_path).await?; + let salt: [u8; LOCAL_KMS_MASTER_KEY_SALT_LEN] = salt.try_into().map_err(|_| { + KmsError::configuration_error(format!( + "Local KMS master key salt at {} must be exactly {} bytes", + salt_path.display(), + LOCAL_KMS_MASTER_KEY_SALT_LEN + )) + })?; + Aes256Gcm::new(&Self::derive_master_key(master_key, &salt)?) + } else { + Aes256Gcm::new(&legacy_key) + }; + (Some(master_cipher), Some(legacy_master_cipher)) + } else { + (None, None) + }; + + Ok(Self { + config, + master_cipher, + legacy_master_cipher, + dek_crypto: AesDekCrypto::new(), + }) + } + /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. fn derive_master_key(master_key: &str, salt: &[u8]) -> Result> { let params = Params::new( @@ -435,6 +475,20 @@ impl LocalKmsClient { Ok(key_material) } + /// Decrypt an AES-256 Local KMS key for explicit migration tooling. + /// + /// The returned buffer is zeroized on drop. Callers must treat the value as + /// plaintext key material and avoid logging or persisting it. + pub async fn decrypt_key_material_for_export(&self, key_id: &str) -> Result> { + let (stored_key, key_material) = self.decode_stored_key(key_id).await?; + if stored_key.algorithm != "AES_256" { + return Err(KmsError::unsupported_algorithm(stored_key.algorithm)); + } + let actual = key_material.len(); + let key_material = key_material.try_into().map_err(|_| KmsError::invalid_key_size(32, actual))?; + Ok(Zeroizing::new(key_material)) + } + async fn validate_existing_keys(&self) -> Result<()> { let mut entries = fs::read_dir(&self.config.key_dir).await?; while let Some(entry) = entries.next_entry().await? { @@ -1208,6 +1262,52 @@ mod tests { assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. })); } + #[tokio::test] + async fn key_export_uses_existing_local_decryption_path_without_writing_files() { + let (client, _temp_dir) = create_test_client().await; + let key_id = "export-key"; + client + .create_key(key_id, "AES_256", None) + .await + .expect("create encrypted key"); + let expected = client.get_key_material(key_id).await.expect("load expected key material"); + let salt_path = LocalKmsClient::master_key_salt_path(&client.config); + let salt_before = fs::read(&salt_path).await.expect("read existing salt"); + + let export_client = LocalKmsClient::new_for_key_export(client.config.clone()) + .await + .expect("open read-only export client"); + let exported = export_client + .decrypt_key_material_for_export(key_id) + .await + .expect("decrypt key for export"); + + assert_eq!(exported.as_ref(), expected.as_slice()); + assert_eq!(fs::read(&salt_path).await.expect("read unchanged salt"), salt_before); + } + + #[tokio::test] + async fn key_export_accepts_plaintext_dev_only_key_without_master_key() { + let (client, _temp_dir) = create_dev_mode_client().await; + let key_id = "plaintext-export-key"; + client + .create_key(key_id, "AES_256", None) + .await + .expect("create plaintext-dev-only key"); + let expected = client.get_key_material(key_id).await.expect("load expected key material"); + + let export_client = LocalKmsClient::new_for_key_export(client.config.clone()) + .await + .expect("open read-only export client"); + let exported = export_client + .decrypt_key_material_for_export(key_id) + .await + .expect("export plaintext-dev-only key"); + + assert_eq!(exported.as_ref(), expected.as_slice()); + assert!(!LocalKmsClient::master_key_salt_path(&client.config).exists()); + } + #[tokio::test] async fn test_plaintext_dev_only_storage_is_explicit_and_loadable() { let (client, _temp_dir) = create_dev_mode_client().await; diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index 7806bbc7d..db4791ec3 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result}; pub use manager::KmsManager; pub use service::{DataKey, ObjectEncryptionService}; pub use service_manager::{ - KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager, + KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager, init_global_kms_service_manager, }; pub use types::*; diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index 07b8628cf..be463c42d 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -21,12 +21,13 @@ use crate::manager::KmsManager; use crate::service::ObjectEncryptionService; use arc_swap::ArcSwap; use sha2::{Digest, Sha256}; +use std::future::Future; use std::sync::{ Arc, OnceLock, atomic::{AtomicU64, Ordering}, }; use subtle::ConstantTimeEq; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::Mutex; use tracing::{debug, error, info, warn}; const LOG_COMPONENT_KMS: &str = "kms"; @@ -93,6 +94,13 @@ pub enum KmsServiceStatus { Error(String), } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KmsStartOutcome { + Started, + Restarted, + AlreadyRunning, +} + /// Service version information for zero-downtime reconfiguration #[derive(Clone)] struct ServiceVersion { @@ -104,16 +112,17 @@ struct ServiceVersion { manager: Arc, } +#[derive(Clone)] +struct RuntimeState { + config: Option, + status: KmsServiceStatus, + current_service: Option, +} + /// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration pub struct KmsServiceManager { - /// Current service version (if running) - /// Uses ArcSwap for atomic, lock-free service switching - /// This allows instant atomic updates without blocking readers - current_service: ArcSwap>, - /// Current configuration - config: Arc>>, - /// Current status - status: Arc>, + /// Atomically published configuration, status, and current service. + state: ArcSwap, /// Version counter (monotonically increasing) version_counter: Arc, /// Mutex to protect lifecycle operations (start, stop, reconfigure) @@ -125,9 +134,11 @@ impl KmsServiceManager { /// Create a new KMS service manager (not configured) pub fn new() -> Self { Self { - current_service: ArcSwap::from_pointee(None), - config: Arc::new(RwLock::new(None)), - status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)), + state: ArcSwap::from_pointee(RuntimeState { + config: None, + status: KmsServiceStatus::NotConfigured, + current_service: None, + }), version_counter: Arc::new(AtomicU64::new(0)), lifecycle_mutex: Arc::new(Mutex::new(())), } @@ -135,44 +146,67 @@ impl KmsServiceManager { /// Get current service status pub async fn get_status(&self) -> KmsServiceStatus { - self.status.read().await.clone() + self.state.load().status.clone() } /// Get current configuration (if any) pub async fn get_config(&self) -> Option { - self.config.read().await.clone() + self.state.load().config.clone() } /// Get configuration for status and management responses without static key material. pub async fn get_redacted_config(&self) -> Option { - let mut config = self.config.read().await.clone()?; + let mut config = self.state.load().config.clone()?; + Self::redact_config(&mut config); + Some(config) + } + + /// Get status and redacted configuration from the same published snapshot. + pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option) { + let state = self.state.load(); + let mut config = state.config.clone(); + if let Some(config) = &mut config { + Self::redact_config(config); + } + (state.status.clone(), config) + } + + fn redact_config(config: &mut KmsConfig) { if let BackendConfig::Static(static_config) = &mut config.backend_config { use zeroize::Zeroize; static_config.secret_key.zeroize(); } - Some(config) } /// Configure KMS with new configuration pub async fn configure(&self, new_config: KmsConfig) -> Result<()> { - let _guard = self.lifecycle_mutex.lock().await; + self.configure_with_persistence(new_config, || async { Ok(()) }).await + } + + /// Configure KMS and publish the in-memory state only after persistence succeeds. + /// + /// The persistence callback runs under the lifecycle lock and must not call + /// another lifecycle method on this manager. + pub async fn configure_with_persistence(&self, new_config: KmsConfig, persist: Persist) -> Result<()> + where + Persist: FnOnce() -> PersistFuture, + PersistFuture: Future>, + { new_config.validate()?; - { - let config = self.config.read().await; - validate_local_transition(config.as_ref(), &new_config)?; - } - - // Update configuration - { - let mut config = self.config.write().await; - *config = Some(new_config.clone()); - } - - // Update status - { - let mut status = self.status.write().await; - *status = KmsServiceStatus::Configured; + let _guard = self.lifecycle_mutex.lock().await; + let current = self.state.load_full(); + validate_local_transition(current.config.as_ref(), &new_config)?; + if current.current_service.is_some() { + return Err(KmsError::configuration_error( + "Cannot configure KMS while it is running; use reconfigure instead", + )); } + persist().await?; + self.state.store(Arc::new(RuntimeState { + config: Some(new_config), + status: KmsServiceStatus::Configured, + current_service: None, + })); debug!( event = EVENT_KMS_SERVICE_STATE, @@ -190,19 +224,35 @@ impl KmsServiceManager { self.start_internal().await } + /// Start or restart KMS with the running-state decision serialized with the lifecycle action. + pub async fn start_or_restart(&self, force: bool) -> Result { + let _guard = self.lifecycle_mutex.lock().await; + let running = self.state.load().current_service.is_some(); + if running && !force { + return Ok(KmsStartOutcome::AlreadyRunning); + } + self.start_internal().await?; + Ok(if running { + KmsStartOutcome::Restarted + } else { + KmsStartOutcome::Started + }) + } + /// Internal start implementation (called within lifecycle mutex) async fn start_internal(&self) -> Result<()> { - let config = { - let config_guard = self.config.read().await; - match config_guard.as_ref() { - Some(config) => config.clone(), - None => { - let err_msg = "Cannot start KMS: no configuration provided"; - error!("{}", err_msg); - let mut status = self.status.write().await; - *status = KmsServiceStatus::Error(err_msg.to_string()); - return Err(KmsError::configuration_error(err_msg)); - } + let state = self.state.load_full(); + let config = match state.config.as_ref() { + Some(config) => config.clone(), + None => { + let err_msg = "Cannot start KMS: no configuration provided"; + error!("{}", err_msg); + self.state.store(Arc::new(RuntimeState { + config: None, + status: KmsServiceStatus::Error(err_msg.to_string()), + current_service: None, + })); + return Err(KmsError::configuration_error(err_msg)); } }; @@ -215,17 +265,9 @@ impl KmsServiceManager { "KMS service starting" ); - match self.create_service_version(&config).await { + match self.create_healthy_service_version(&config).await { Ok(service_version) => { - // Atomically update to new service version (lock-free, instant) - // ArcSwap::store() is a true atomic operation using CAS - self.current_service.store(Arc::new(Some(service_version))); - - // Update status - { - let mut status = self.status.write().await; - *status = KmsServiceStatus::Running; - } + self.publish_running(config, service_version); debug!( event = EVENT_KMS_SERVICE_STATE, @@ -239,13 +281,24 @@ impl KmsServiceManager { Err(e) => { let err_msg = format!("Failed to create KMS backend: {e}"); error!("{}", err_msg); - let mut status = self.status.write().await; - *status = KmsServiceStatus::Error(err_msg.clone()); + if state.current_service.is_none() { + self.state.store(Arc::new(RuntimeState { + config: state.config.clone(), + status: KmsServiceStatus::Error(err_msg.clone()), + current_service: None, + })); + } Err(KmsError::backend_error(&err_msg)) } } } + /// Replace the running service without exposing a stopped interval. + pub async fn restart(&self) -> Result<()> { + let _guard = self.lifecycle_mutex.lock().await; + self.start_internal().await + } + /// Stop KMS service /// /// Note: This stops accepting new operations, but existing operations using @@ -267,15 +320,16 @@ impl KmsServiceManager { // Atomically clear current service version (lock-free, instant) // Note: Existing Arc references will keep the service alive until operations complete - self.current_service.store(Arc::new(None)); - - // Update status (keep configuration) - { - let mut status = self.status.write().await; - if !matches!(*status, KmsServiceStatus::NotConfigured) { - *status = KmsServiceStatus::Configured; - } - } + let state = self.state.load_full(); + self.state.store(Arc::new(RuntimeState { + config: state.config.clone(), + status: if state.config.is_some() { + KmsServiceStatus::Configured + } else { + KmsServiceStatus::NotConfigured + }, + current_service: None, + })); debug!( event = EVENT_KMS_SERVICE_STATE, @@ -298,6 +352,22 @@ impl KmsServiceManager { /// This ensures zero downtime during reconfiguration, even for long-running /// operations like encrypting large files. pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> { + self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await + } + + /// Reconfigure KMS after the candidate is healthy and persistence succeeds. + /// + /// The persistence callback runs under the lifecycle lock and must not call + /// another lifecycle method on this manager. + pub async fn reconfigure_with_persistence( + &self, + new_config: KmsConfig, + persist: Persist, + ) -> Result<()> + where + Persist: FnOnce() -> PersistFuture, + PersistFuture: Future>, + { let _guard = self.lifecycle_mutex.lock().await; debug!( @@ -308,33 +378,18 @@ impl KmsServiceManager { "KMS service reconfiguring" ); new_config.validate()?; - { - let config = self.config.read().await; - validate_local_transition(config.as_ref(), &new_config)?; - } + validate_local_transition(self.state.load().config.as_ref(), &new_config)?; // Create new service version without stopping old one // This allows existing operations to continue while new operations use new service - match self.create_service_version(&new_config).await { + match self.create_healthy_service_version(&new_config).await { Ok(new_service_version) => { // Get old version for logging (lock-free read) - let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version); + let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version); - { - let mut config = self.config.write().await; - *config = Some(new_config); - } + persist().await?; - // Atomically switch to new service version (lock-free, instant CAS operation) - // This is a true atomic operation - no waiting for locks, instant switch - // Old service will be dropped when no more Arc references exist - self.current_service.store(Arc::new(Some(new_service_version.clone()))); - - // Update status - { - let mut status = self.status.write().await; - *status = KmsServiceStatus::Running; - } + self.publish_running(new_config, new_service_version.clone()); if let Some(old_ver) = old_version { info!( @@ -371,7 +426,7 @@ impl KmsServiceManager { /// Returns the manager from the current service version. /// Uses lock-free atomic load for optimal performance. pub async fn get_manager(&self) -> Option> { - self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone()) + self.state.load().current_service.as_ref().map(|sv| sv.manager.clone()) } /// Get encryption service (if running) @@ -381,7 +436,7 @@ impl KmsServiceManager { /// This ensures new operations always use the latest service version, /// while existing operations continue using their Arc references. pub async fn get_encryption_service(&self) -> Option> { - self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone()) + self.state.load().current_service.as_ref().map(|sv| sv.service.clone()) } /// Get current service version number @@ -389,14 +444,16 @@ impl KmsServiceManager { /// Useful for monitoring and debugging. /// Uses lock-free atomic load. pub async fn get_service_version(&self) -> Option { - self.current_service.load().as_ref().as_ref().map(|sv| sv.version) + self.state.load().current_service.as_ref().map(|sv| sv.version) } /// Health check for the KMS service pub async fn health_check(&self) -> Result { - let manager = self.get_manager().await; - match manager { - Some(manager) => { + let checked_state = self.state.load_full(); + match checked_state.current_service.as_ref() { + Some(service_version) => { + let manager = service_version.manager.clone(); + let checked_version = service_version.version; // Perform health check on the backend match manager.health_check().await { Ok(healthy) => { @@ -407,9 +464,8 @@ impl KmsServiceManager { } Err(e) => { error!("KMS health check error: {}", e); - // Update status to error - let mut status = self.status.write().await; - *status = KmsServiceStatus::Error(format!("Health check failed: {e}")); + let _guard = self.lifecycle_mutex.lock().await; + self.mark_health_error_if_current(checked_version, &e); Err(e) } } @@ -468,6 +524,33 @@ impl KmsServiceManager { manager: kms_manager, }) } + + async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result { + let service_version = self.create_service_version(config).await?; + if !service_version.manager.health_check().await? { + return Err(KmsError::backend_error("KMS backend health check failed")); + } + Ok(service_version) + } + + fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) { + self.state.store(Arc::new(RuntimeState { + config: Some(config), + status: KmsServiceStatus::Running, + current_service: Some(service_version), + })); + } + + fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) { + let current = self.state.load_full(); + if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) { + self.state.store(Arc::new(RuntimeState { + config: current.config.clone(), + status: KmsServiceStatus::Error(format!("Health check failed: {error}")), + current_service: current.current_service.clone(), + })); + } + } } impl Default for KmsServiceManager { @@ -500,6 +583,11 @@ pub async fn get_global_encryption_service() -> Option KmsConfig { + KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32])) + } #[tokio::test] async fn configure_rejects_insecure_development_defaults_before_state_update() { @@ -517,8 +605,6 @@ mod tests { #[tokio::test] async fn redacted_config_omits_static_key_material() { - use base64::Engine as _; - let manager = KmsServiceManager::new(); let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); manager @@ -533,6 +619,136 @@ mod tests { assert!(static_config.secret_key.is_empty()); } + #[tokio::test] + async fn configure_persistence_failure_leaves_state_unchanged() { + let manager = KmsServiceManager::new(); + + let result = manager + .configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) }) + .await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured); + assert!(manager.get_config().await.is_none()); + assert!(manager.get_encryption_service().await.is_none()); + } + + #[tokio::test] + async fn configure_rejects_running_service_without_changing_snapshot() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let version = manager.get_service_version().await; + + let result = manager.configure(static_config("key-b", 0x22)).await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, version); + assert_eq!( + manager.get_config().await.and_then(|config| config.default_key_id), + Some("key-a".to_string()) + ); + } + + #[tokio::test] + async fn reconfigure_persistence_failure_keeps_old_running_snapshot() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await; + let old_service = manager.get_encryption_service().await.expect("old service"); + + let result = manager + .reconfigure_with_persistence(static_config("key-b", 0x22), || async { + Err(KmsError::backend_error("persist failed")) + }) + .await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, old_version); + assert_eq!( + manager.get_config().await.and_then(|config| config.default_key_id), + Some("key-a".to_string()) + ); + assert!(Arc::ptr_eq( + &old_service, + &manager.get_encryption_service().await.expect("old service remains") + )); + } + + #[tokio::test] + async fn reconfigure_candidate_failure_keeps_old_running_snapshot() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await; + let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file"); + let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults(); + + let result = manager.reconfigure(invalid_config).await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, old_version); + assert_eq!( + manager.get_config().await.and_then(|config| config.default_key_id), + Some("key-a".to_string()) + ); + } + + #[tokio::test] + async fn restart_never_unpublishes_the_running_service() { + let manager = Arc::new(KmsServiceManager::new()); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await.expect("old version"); + let restarting = { + let manager = manager.clone(); + tokio::spawn(async move { manager.restart().await }) + }; + + while !restarting.is_finished() { + assert!(manager.get_encryption_service().await.is_some()); + tokio::task::yield_now().await; + } + restarting.await.expect("restart task").expect("restart"); + + assert!(manager.get_encryption_service().await.is_some()); + assert!(manager.get_service_version().await.expect("new version") > old_version); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + } + + #[tokio::test] + async fn start_or_restart_decides_under_the_lifecycle_lock() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + + assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started); + let first_version = manager.get_service_version().await.expect("first version"); + assert_eq!( + manager.start_or_restart(false).await.expect("already running"), + KmsStartOutcome::AlreadyRunning + ); + assert_eq!(manager.get_service_version().await, Some(first_version)); + assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted); + assert!(manager.get_service_version().await.expect("restarted version") > first_version); + } + + #[tokio::test] + async fn stale_health_failure_cannot_poison_new_service_status() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await.expect("old version"); + manager.restart().await.expect("restart"); + + manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure")); + + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + } + #[tokio::test] async fn forbidden_local_master_key_change_preserves_running_config_and_service() { use crate::types::{CreateKeyRequest, KeyUsage}; diff --git a/crates/rio-v2/tests/minio_fixture_lab/README.md b/crates/rio-v2/tests/minio_fixture_lab/README.md index f20abe205..81ae85ef6 100644 --- a/crates/rio-v2/tests/minio_fixture_lab/README.md +++ b/crates/rio-v2/tests/minio_fixture_lab/README.md @@ -137,7 +137,7 @@ tests read): ./capture_via_docker.sh RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \ - cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored + cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored ``` This is exactly what the nightly `minio-interop` GitHub Actions workflow runs diff --git a/crates/utils/src/http/header_compat.rs b/crates/utils/src/http/header_compat.rs index 6dda300e6..d1bd38b82 100644 --- a/crates/utils/src/http/header_compat.rs +++ b/crates/utils/src/http/header_compat.rs @@ -25,6 +25,11 @@ const RUSTFS_PREFIX: &str = "x-rustfs-"; const MINIO_PREFIX: &str = "x-minio-"; const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-"; const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-"; +const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-"; +const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart"; +const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = "x-rustfs-encryption-original-size"; +const MINIO_ENCRYPTION_ORIGINAL_SIZE: &str = "x-minio-encryption-original-size"; +const SSEC_ORIGINAL_SIZE: &str = "x-amz-server-side-encryption-customer-original-size"; // Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header. pub const SUFFIX_FORCE_DELETE: &str = "force-delete"; @@ -40,11 +45,49 @@ pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request" pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check"; pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc"; -/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or -/// x-minio-encryption-*). Case-insensitive for metadata filtering. +/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO. +/// Case-insensitive for metadata filtering. pub fn is_encryption_metadata_key(key: &str) -> bool { let lower = key.to_lowercase(); - lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX) + lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) + || lower.starts_with(MINIO_ENCRYPTION_PREFIX) + || lower.starts_with(MINIO_INTERNAL_ENCRYPTION_PREFIX) + || lower == MINIO_INTERNAL_ENCRYPTED_MULTIPART +} + +/// Returns true when a metadata key proves that object data is encrypted. +/// +/// Original-size metadata alone is not proof: older plaintext objects can +/// retain that compatibility field after metadata migration. +pub fn is_object_encryption_marker(key: &str) -> bool { + (is_encryption_metadata_key(key) + && !key.eq_ignore_ascii_case(RUSTFS_ENCRYPTION_ORIGINAL_SIZE) + && !key.eq_ignore_ascii_case(MINIO_ENCRYPTION_ORIGINAL_SIZE)) + || super::is_sse_header(key) +} + +/// Reads the logical object size recorded by encryption metadata. +pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap) -> std::io::Result> { + let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE); + let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE) + .or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE)) + .or(actual_size.as_deref()); + + let Some(size) = size.filter(|size| !size.is_empty()) else { + return Ok(None); + }; + size.parse::() + .map(Some) + .map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}"))) +} + +fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap, key: &str) -> Option<&'a str> { + metadata.get(key).map(String::as_str).or_else(|| { + metadata + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(key)) + .map(|(_, value)| value.as_str()) + }) } fn rustfs_key(suffix: &str) -> String { @@ -106,10 +149,37 @@ mod tests { assert!(is_encryption_metadata_key("x-rustfs-encryption-iv")); assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key")); assert!(is_encryption_metadata_key("x-minio-encryption-iv")); + assert!(is_encryption_metadata_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key")); + assert!(is_encryption_metadata_key("X-Minio-Internal-Encrypted-Multipart")); assert!(!is_encryption_metadata_key("x-amz-meta-custom")); assert!(!is_encryption_metadata_key("x-rustfs-internal-healing")); } + #[test] + fn object_encryption_marker_excludes_size_only_metadata() { + assert!(!is_object_encryption_marker(RUSTFS_ENCRYPTION_ORIGINAL_SIZE)); + assert!(is_object_encryption_marker("X-Minio-Internal-Server-Side-Encryption-Sealed-Key")); + assert!(is_object_encryption_marker("x-amz-server-side-encryption")); + } + + #[test] + fn object_encryption_original_size_is_case_insensitive() { + let metadata = std::collections::HashMap::from([( + "X-Amz-Server-Side-Encryption-Customer-Original-Size".to_string(), + "42".to_string(), + )]); + assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42)); + } + + #[test] + fn object_encryption_original_size_prefers_rustfs_metadata() { + let metadata = std::collections::HashMap::from([ + (SSEC_ORIGINAL_SIZE.to_string(), "21".to_string()), + (RUSTFS_ENCRYPTION_ORIGINAL_SIZE.to_string(), "42".to_string()), + ]); + assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42)); + } + #[test] fn test_get_header() { let mut headers = HeaderMap::new(); diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index 4eb3b41d5..62e95f340 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -358,7 +358,7 @@ Fixture-backed tests should run when the fixture path is present: ```bash cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture -cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture +cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture ``` ## Multi-Expert Adversarial Review Summary diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index e8fb0ee64..6a6da70d4 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -393,36 +393,27 @@ impl Operation for ConfigureKmsHandler { // Convert request to KmsConfig let kms_config = configure_request.to_kms_config(); - // Configure the service - let (success, message, status) = match service_manager.configure(kms_config.clone()).await { + let persisted_config = kms_config.clone(); + let (success, message, status) = match service_manager + .configure_with_persistence(kms_config, || async move { + save_kms_config(&persisted_config) + .await + .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}"))) + }) + .await + { Ok(()) => { - // Persist the configuration to cluster storage - if let Err(e) = save_kms_config(&kms_config).await { - let error_msg = format!("KMS configured in memory but failed to persist: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "configure", - state = "persist_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } else { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "configure", - state = "configured", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS configured successfully".to_string(), status) - } + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "configure", + state = "configured", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS configured successfully".to_string(), status) } Err(e) => { let error_msg = format!("Failed to configure KMS: {e}"); @@ -529,125 +520,61 @@ impl Operation for StartKmsHandler { ); let service_manager = kms_service_manager_from_context(); - - // Check if already running and force flag - let current_status = service_manager.get_status().await; - if matches!(current_status, KmsServiceStatus::Running) && !start_request.force.unwrap_or(false) { - warn!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "start", - state = "already_running", - "admin kms dynamic state" - ); - let response = StartKmsResponse { - success: false, - message: "KMS service is already running. Use force=true to restart.".to_string(), - status: current_status, - }; - let json_response = match serde_json::to_string(&response) { - Ok(json) => json, - Err(e) => { - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = EVENT_ADMIN_KMS_DYNAMIC_STATE, - operation = "start", - result = "response_serialize_failed", - error = %e, - "admin kms dynamic state" - ); - return Ok(S3Response::new(( - StatusCode::INTERNAL_SERVER_ERROR, - Body::from("Serialization error".to_string()), - ))); - } - }; - return Ok(S3Response::new((StatusCode::OK, Body::from(json_response)))); - } - - // Start the service (or restart if force=true) - let (success, message, status) = - if start_request.force.unwrap_or(false) && matches!(current_status, KmsServiceStatus::Running) { - // Force restart - match service_manager.stop().await { - Ok(()) => match service_manager.start().await { - Ok(()) => { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "restart", - state = "running", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS service restarted successfully".to_string(), status) - } - Err(e) => { - let error_msg = format!("Failed to restart KMS service: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "restart", - state = "start_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } - }, - Err(e) => { - let error_msg = format!("Failed to stop KMS service for restart: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "restart", - state = "stop_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } - } - } else { - // Normal start - match service_manager.start().await { - Ok(()) => { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "start", - state = "running", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS service started successfully".to_string(), status) - } - Err(e) => { - let error_msg = format!("Failed to start KMS service: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "start", - state = "start_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } - } - }; + let force = start_request.force.unwrap_or(false); + let (success, message, status) = match service_manager.start_or_restart(force).await { + Ok(rustfs_kms::KmsStartOutcome::Started) => { + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "start", + state = "running", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS service started successfully".to_string(), status) + } + Ok(rustfs_kms::KmsStartOutcome::Restarted) => { + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "restart", + state = "running", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS service restarted successfully".to_string(), status) + } + Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => { + let status = service_manager.get_status().await; + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "start", + state = "already_running", + "admin kms dynamic state" + ); + (false, "KMS service is already running. Use force=true to restart.".to_string(), status) + } + Err(e) => { + let error_msg = format!("Failed to start or restart KMS service: {e}"); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "start", + state = "start_failed", + error = %e, + "admin kms dynamic state" + ); + let status = service_manager.get_status().await; + (false, error_msg, status) + } + }; let response = StartKmsResponse { success, @@ -804,8 +731,7 @@ impl Operation for GetKmsStatusHandler { let service_manager = kms_service_manager_from_context(); - let status = service_manager.get_status().await; - let config = service_manager.get_redacted_config().await; + let (status, config) = service_manager.get_redacted_state().await; // Get backend type and health status let backend_type = config.as_ref().map(|c| c.backend.clone()); @@ -938,36 +864,27 @@ impl Operation for ReconfigureKmsHandler { // Convert request to KmsConfig let kms_config = configure_request.to_kms_config(); - // Reconfigure the service (stops, reconfigures, and starts) - let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await { + let persisted_config = kms_config.clone(); + let (success, message, status) = match service_manager + .reconfigure_with_persistence(kms_config, || async move { + save_kms_config(&persisted_config) + .await + .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}"))) + }) + .await + { Ok(()) => { - // Persist the configuration to cluster storage - if let Err(e) = save_kms_config(&kms_config).await { - let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "reconfigure", - state = "persist_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } else { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "reconfigure", - state = "reconfigured", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS reconfigured and restarted successfully".to_string(), status) - } + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "reconfigure", + state = "reconfigured", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS reconfigured and restarted successfully".to_string(), status) } Err(e) => { let error_msg = format!("Failed to reconfigure KMS: {e}"); diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index bd1baf9a6..84118caba 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -18,13 +18,12 @@ use super::handles::{ IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface, default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface, default_endpoints_interface, default_expiry_state_interface, default_federated_identity_interface, - default_internode_metrics_interface, default_kms_runtime_interface, default_local_node_name_interface, - default_lock_client_interface, default_lock_clients_interface, default_notification_system_interface, - default_notify_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface, - default_region_interface, default_replication_pool_interface, default_replication_stats_interface, - default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface, - default_server_config_interface, default_storage_class_interface, default_tier_config_interface, - default_transition_state_interface, + default_internode_metrics_interface, default_local_node_name_interface, default_lock_client_interface, + default_lock_clients_interface, default_notification_system_interface, default_notify_interface, + default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface, + default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface, + default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface, + default_storage_class_interface, default_tier_config_interface, default_transition_state_interface, }; use super::interfaces::{ ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface, @@ -80,6 +79,7 @@ pub struct AppContext { impl AppContext { pub fn new(object_store: Arc, iam: Arc, kms: Arc) -> Self { let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled(); + let kms_runtime = Arc::new(crate::app::context::handles::KmsRuntimeHandle::new(kms.handle())); // Let ecstore probe this cache inside get_object_reader, after // metadata resolution but before the erasure data read (backlog#802). crate::app::object_data_cache::register_object_data_cache_body_hook(Arc::clone(&object_data_cache)); @@ -94,7 +94,7 @@ impl AppContext { iam, federated_identity: default_federated_identity_interface(), kms, - kms_runtime: default_kms_runtime_interface(), + kms_runtime, outbound_tls_runtime: default_outbound_tls_runtime_interface(), notify: default_notify_interface(), notification_system: default_notification_system_interface(), diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 7eb4e3f29..9e376e1a5 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -128,12 +128,19 @@ impl KmsInterface for KmsHandle { } /// Default KMS runtime interface adapter. -#[derive(Default)] -pub struct KmsRuntimeHandle; +pub struct KmsRuntimeHandle { + kms: Option>, +} + +impl KmsRuntimeHandle { + pub fn new(kms: Arc) -> Self { + Self { kms: Some(kms) } + } +} impl KmsRuntimeInterface for KmsRuntimeHandle { fn service_manager(&self) -> Option> { - runtime_sources::kms_service_manager() + self.kms.clone() } } @@ -485,7 +492,9 @@ pub fn default_notification_system_interface() -> Arc Arc { - Arc::new(KmsRuntimeHandle) + Arc::new(KmsRuntimeHandle { + kms: runtime_sources::kms_service_manager(), + }) } pub fn default_outbound_tls_runtime_interface() -> Arc { @@ -607,10 +616,10 @@ pub fn default_buffer_config_interface() -> Arc { #[cfg(test)] mod tests { use super::{ - ServerConfigHandle, default_federated_identity_interface, federated_identity_interface, - publish_default_federated_identity_service, runtime_sources, + KmsRuntimeHandle, KmsServiceManager, ServerConfigHandle, default_federated_identity_interface, + federated_identity_interface, publish_default_federated_identity_service, runtime_sources, }; - use crate::app::context::interfaces::ServerConfigInterface; + use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface}; use rustfs_config::server_config::Config; use rustfs_iam::{ federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter}, @@ -733,4 +742,19 @@ mod tests { "handle B must serve its own credentials" ); } + + #[test] + fn kms_runtime_handles_keep_injected_managers_isolated() { + let manager_a = Arc::new(KmsServiceManager::new()); + let manager_b = Arc::new(KmsServiceManager::new()); + let handle_a = KmsRuntimeHandle::new(manager_a.clone()); + let handle_b = KmsRuntimeHandle::new(manager_b.clone()); + + assert!(Arc::ptr_eq(&handle_a.service_manager().expect("manager A"), &manager_a)); + assert!(Arc::ptr_eq(&handle_b.service_manager().expect("manager B"), &manager_b)); + assert!(!Arc::ptr_eq( + &handle_a.service_manager().expect("manager A"), + &handle_b.service_manager().expect("manager B") + )); + } } diff --git a/crates/ecstore/tests/minio_generated_read_test.rs b/rustfs/src/storage/minio_generated_read_test.rs similarity index 96% rename from crates/ecstore/tests/minio_generated_read_test.rs rename to rustfs/src/storage/minio_generated_read_test.rs index 875aa9be9..36285c565 100644 --- a/crates/ecstore/tests/minio_generated_read_test.rs +++ b/rustfs/src/storage/minio_generated_read_test.rs @@ -4,14 +4,13 @@ use std::fs; use std::io::Cursor; use std::path::{Path, PathBuf}; -mod storage_api; - +use super::sse::SseObjectEncryptionResolver; +use super::storage_api::ecstore_test_support::{ + DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk, +}; use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info}; use serde::Deserialize; use sha2::{Digest, Sha256}; -use storage_api::minio_generated_read::{ - DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk, -}; use temp_env::async_with_vars; use tokio::io::{AsyncReadExt, AsyncWrite}; @@ -49,7 +48,7 @@ impl AsyncWrite for VecAsyncWriter { fn fixture_root() -> PathBuf { std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT") .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated")) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated")) } fn case_dir(case_id: &str) -> PathBuf { @@ -137,12 +136,14 @@ async fn read_fixture_plaintext(encrypted: Vec, object_info: ObjectInfo, kms ("RUSTFS_SSE_S3_MASTER_KEY", None::), ], async move { - let (mut reader, offset, length) = GetObjectReader::new( + let resolver = SseObjectEncryptionResolver; + let (mut reader, offset, length) = GetObjectReader::new_with_resolver( Box::new(Cursor::new(encrypted)), None, &object_info, &ObjectOptions::default(), &http::HeaderMap::new(), + Some(&resolver), ) .await .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?; @@ -219,11 +220,12 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil readers.push(reader); } - let erasure = Erasure::new( + let erasure = Erasure::try_new( file_info.erasure.data_blocks, file_info.erasure.parity_blocks, file_info.erasure.block_size, - ); + ) + .expect("fixture erasure geometry"); let mut writer = VecAsyncWriter::default(); let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await; if let Some(err) = err { diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 9033400c4..f427a3345 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -36,6 +36,8 @@ mod ecfs_extend; mod ecfs_test; pub(crate) mod head_prefix; #[cfg(test)] +mod minio_generated_read_test; +#[cfg(test)] mod multi_factor_scheduler_integration_test; pub(crate) mod runtime_sources; #[cfg(test)] diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 0cd426d6e..e06b9dea5 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -70,6 +70,10 @@ //! ``` use super::StorageError; +use super::storage_api::ecstore_object::{ + EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial, + ReadEncryptionMode, ReadEncryptionRequest, +}; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; #[cfg(feature = "rio-v2")] use aes_gcm::aead::Payload; @@ -155,6 +159,7 @@ use rustfs_utils::http::headers::{ }; use rustfs_utils::path::path_join_buf; use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId}; +use std::borrow::Cow; // ============================================================================ // High-Level SSE Configuration @@ -652,6 +657,23 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap, } pub(crate) fn map_get_object_reader_error(err: StorageError) -> ApiError { + if let StorageError::Io(io_error) = &err + && let Some(resolution_error) = io_error + .get_ref() + .and_then(|source| source.downcast_ref::()) + { + let code = match resolution_error.kind() { + EncryptionResolutionErrorKind::InvalidRequest => S3ErrorCode::InvalidRequest, + EncryptionResolutionErrorKind::ServiceUnavailable => S3ErrorCode::ServiceUnavailable, + _ => S3ErrorCode::InternalError, + }; + return ApiError { + code, + message: resolution_error.to_string(), + source: Some(Box::new(err)), + }; + } + if let Some(message) = map_ssec_get_object_reader_error_message(&err) { return ApiError { code: S3ErrorCode::InvalidRequest, @@ -774,6 +796,117 @@ pub enum EncryptionKeyKind { Object, } +pub(crate) struct SseObjectEncryptionResolver; + +#[async_trait] +impl ObjectEncryptionResolver for SseObjectEncryptionResolver { + async fn resolve_read_material( + &self, + request: ReadEncryptionRequest<'_>, + ) -> Result, EncryptionResolutionError> { + let metadata = normalize_encryption_metadata_case(request.metadata)?; + let (customer_algorithm, customer_key, customer_key_md5) = + extract_ssec_params_from_headers(request.headers).map_err(map_encryption_resolution_error)?; + if let Some(stored_algorithm) = metadata.get("x-amz-server-side-encryption-customer-algorithm") { + let request_algorithm = customer_algorithm.as_ref().ok_or_else(|| { + map_encryption_resolution_error(ssec_invalid_request( + "The object was stored using a form of Server Side Encryption. \ + The correct parameters must be provided to retrieve the object.", + )) + })?; + if stored_algorithm != request_algorithm.as_str() { + return Err(map_encryption_resolution_error(ssec_invalid_request( + "The provided encryption parameters did not match the ones used originally to encrypt the object.", + ))); + } + } + let material = sse_decryption(DecryptionRequest { + bucket: request.bucket, + key: request.object, + metadata: &metadata, + sse_customer_key: customer_key.as_ref(), + sse_customer_key_md5: customer_key_md5.as_ref(), + }) + .await + .map_err(map_encryption_resolution_error)?; + + Ok(material.map(|material| ReadEncryptionMaterial { + key_bytes: material.key_bytes, + mode: match material.key_kind { + EncryptionKeyKind::Direct => ReadEncryptionMode::Direct { + base_nonce: material.base_nonce, + }, + EncryptionKeyKind::Object => ReadEncryptionMode::Object, + }, + })) + } +} + +fn normalize_encryption_metadata_case( + metadata: &HashMap, +) -> Result>, EncryptionResolutionError> { + const CANONICAL_KEYS: &[&str] = &[ + "x-amz-server-side-encryption", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-server-side-encryption-customer-algorithm", + "x-amz-server-side-encryption-customer-key-md5", + SSEC_ORIGINAL_SIZE_HEADER, + INTERNAL_ENCRYPTION_KEY_ID_HEADER, + INTERNAL_ENCRYPTION_KEY_HEADER, + INTERNAL_ENCRYPTION_ALGORITHM_HEADER, + INTERNAL_ENCRYPTION_IV_HEADER, + "x-rustfs-encryption-context", + "x-rustfs-encryption-tag", + INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, + MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, + MINIO_INTERNAL_ENCRYPTION_IV_HEADER, + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, + ]; + + let needs_normalization = metadata.keys().any(|key| { + CANONICAL_KEYS + .iter() + .any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical)) + }); + if !needs_normalization { + return Ok(Cow::Borrowed(metadata)); + } + + let mut normalized = metadata.clone(); + for canonical in CANONICAL_KEYS { + let mut matching_values = metadata + .iter() + .filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value)); + let Some(value) = matching_values.next() else { + continue; + }; + if matching_values.any(|candidate| candidate != value) { + return Err(EncryptionResolutionError::new( + EncryptionResolutionErrorKind::InvalidMetadata, + format!("conflicting object encryption metadata for {canonical}"), + )); + } + if !normalized.contains_key(*canonical) { + normalized.insert((*canonical).to_string(), value.clone()); + } + } + Ok(Cow::Owned(normalized)) +} + +fn map_encryption_resolution_error(error: ApiError) -> EncryptionResolutionError { + let kind = match error.code { + S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => EncryptionResolutionErrorKind::InvalidRequest, + S3ErrorCode::ServiceUnavailable => EncryptionResolutionErrorKind::ServiceUnavailable, + _ => EncryptionResolutionErrorKind::DecryptionFailed, + }; + EncryptionResolutionError::new(kind, error.message) +} + #[derive(Debug, Clone)] pub struct ManagedSealedKey { #[cfg(feature = "rio-v2")] @@ -2642,19 +2775,20 @@ fn ssec_invalid_request(message: &str) -> ApiError { mod tests { use super::{ ApiError, DataKey, DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest, - INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER, - INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, - MINIO_INTERNAL_ENCRYPTION_IV_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, - MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, - MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType, - SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material, - apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers, - extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse, - kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, md5_base64, - normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, - sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, - validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match, + EncryptionResolutionErrorKind, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, + INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + ObjectEncryptionResolver, PrepareEncryptionRequest, ReadEncryptionMode, ReadEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, + SSEType, SseDekProvider, SseObjectEncryptionResolver, SsecParams, StorageError, TestSseDekProvider, + apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata, + extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, + generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, + md5_base64, normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, + sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, + validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match, }; #[cfg(feature = "rio-v2")] use super::{ @@ -2709,11 +2843,154 @@ mod tests { use tokio::sync::Mutex; static SSE_TEST_LOCK: OnceLock> = OnceLock::new(); + static SSE_TEST_KMS_KEY_DIR: OnceLock = OnceLock::new(); async fn lock_sse_test_state() -> tokio::sync::MutexGuard<'static, ()> { SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await } + async fn configure_test_global_local_kms() -> Arc { + let key_dir = SSE_TEST_KMS_KEY_DIR.get_or_init(|| tempfile::TempDir::new().expect("create KMS key directory")); + let manager = rustfs_kms::init_global_kms_service_manager(); + manager + .reconfigure(rustfs_kms::KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) + .await + .expect("configure test KMS service"); + manager + } + + #[tokio::test] + async fn object_encryption_resolver_returns_ssec_read_material() { + let key = [0x31; 32]; + let key_b64 = BASE64_STANDARD.encode(key); + let key_md5 = md5_base64(key); + let nonce = [0x42; 12]; + let metadata = HashMap::from([ + ("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()), + ("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()), + ("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)), + ]); + let mut headers = HeaderMap::new(); + headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256")); + headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"), + ); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"), + ); + + let material = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &headers, + }) + .await + .expect("SSE-C material should resolve") + .expect("SSE-C metadata should produce material"); + + assert_eq!(material.key_bytes, key); + assert_eq!(material.mode, ReadEncryptionMode::Direct { base_nonce: nonce }); + } + + #[tokio::test] + async fn object_encryption_resolver_rejects_missing_or_invalid_ssec_algorithm() { + let key = [0x31; 32]; + let key_b64 = BASE64_STANDARD.encode(key); + let key_md5 = md5_base64(key); + let metadata = HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ("x-amz-server-side-encryption-customer-key-md5".to_string(), key_md5.clone()), + ]); + + for algorithm in [None, Some("AES128")] { + let mut headers = HeaderMap::new(); + if let Some(algorithm) = algorithm { + headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static(algorithm)); + } + headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"), + ); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"), + ); + + let result = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &headers, + }) + .await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("missing or invalid SSE-C algorithm must fail closed"), + }; + + assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest); + } + } + + #[tokio::test] + async fn object_encryption_resolver_classifies_missing_ssec_key_as_invalid_request() { + let metadata = HashMap::from([("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string())]); + let result = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &HeaderMap::new(), + }) + .await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("missing SSE-C key must fail closed"), + }; + + assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest); + } + + #[tokio::test] + async fn object_encryption_resolver_rejects_conflicting_metadata_case_variants() { + let metadata = HashMap::from([ + ("x-rustfs-encryption-key".to_string(), "first".to_string()), + ("X-Rustfs-Encryption-Key".to_string(), "second".to_string()), + ]); + let result = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &HeaderMap::new(), + }) + .await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("conflicting metadata aliases must fail closed"), + }; + + assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidMetadata); + } + + #[test] + fn normalize_encryption_metadata_case_accepts_lowercase_minio_internal_keys() { + let lowercase_key = MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase(); + let metadata = HashMap::from([(lowercase_key, "sealed-key".to_string())]); + + let normalized = super::normalize_encryption_metadata_case(&metadata).expect("metadata aliases should normalize"); + + assert_eq!( + normalized.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER), + Some(&"sealed-key".to_string()) + ); + } + struct UnavailableSseDekProvider; #[async_trait::async_trait] @@ -3433,18 +3710,11 @@ mod tests { #[cfg(feature = "rio-v2")] #[tokio::test] async fn test_sse_kms_roundtrip_persists_and_uses_minio_context() { - use rustfs_kms::config::KmsConfig; use rustfs_kms::types::{CreateKeyRequest, KeyUsage}; - use tempfile::TempDir; let _guard = lock_sse_test_state().await; reset_sse_dek_provider(); - let manager = rustfs_kms::init_global_kms_service_manager(); - let temp_dir = TempDir::new().expect("temp dir"); - manager - .reconfigure(KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("kms reconfigure should succeed"); + let manager = configure_test_global_local_kms().await; manager .get_encryption_service() .await @@ -3736,6 +4006,19 @@ mod tests { assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object); assert_eq!(decrypted.key_bytes, material.key_bytes); + + let resolved = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &HeaderMap::new(), + }) + .await + .expect("managed resolver") + .expect("managed material"); + assert_eq!(resolved.mode, ReadEncryptionMode::Object); + assert_eq!(resolved.key_bytes, material.key_bytes); }, ) .await; @@ -3796,6 +4079,29 @@ mod tests { assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object); assert_eq!(decrypted.key_bytes, material.key_bytes); + + let mut headers = HeaderMap::new(); + headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256")); + headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&customer_key).expect("customer key header"), + ); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&customer_key_md5).expect("customer key MD5 header"), + ); + let resolved = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &headers, + }) + .await + .expect("SSE-C resolver") + .expect("SSE-C material"); + assert_eq!(resolved.mode, ReadEncryptionMode::Object); + assert_eq!(resolved.key_bytes, material.key_bytes); } #[cfg(feature = "rio-v2")] @@ -4270,17 +4576,9 @@ mod tests { #[tokio::test] async fn test_managed_decryption_selects_provider_from_persisted_dek() { - use rustfs_kms::config::KmsConfig; - use tempfile::TempDir; - let _guard = lock_sse_test_state().await; reset_sse_dek_provider(); - let manager = rustfs_kms::init_global_kms_service_manager(); - let key_dir = TempDir::new().expect("create KMS key directory"); - manager - .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("start test KMS service"); + let manager = configure_test_global_local_kms().await; let local_master_key = [7u8; 32]; let local_provider = TestSseDekProvider::new_with_key(local_master_key); @@ -4348,9 +4646,6 @@ mod tests { /// the local-provider cache. #[tokio::test] async fn test_kms_envelope_never_routes_to_cached_local_provider() { - use rustfs_kms::config::KmsConfig; - use tempfile::TempDir; - let _guard = lock_sse_test_state().await; reset_sse_dek_provider(); @@ -4362,12 +4657,7 @@ mod tests { .expect("write local provider into local cache") = Some(Arc::new(TestSseDekProvider::new_with_key(local_master_key))); // 2. Start a KMS service (dynamic enable). - let manager = rustfs_kms::init_global_kms_service_manager(); - let key_dir = TempDir::new().expect("create KMS key directory"); - manager - .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("start test KMS service"); + let manager = configure_test_global_local_kms().await; // 3. Construct a KMS JSON envelope — the persisted format of a KMS-wrapped DEK. // is_data_key_envelope() will return true for this payload. @@ -4458,7 +4748,7 @@ mod tests { use rustfs_kms::config::KmsConfig; let _guard = lock_sse_test_state().await; - let manager = rustfs_kms::init_global_kms_service_manager(); + let manager = Arc::new(rustfs_kms::KmsServiceManager::new()); manager .reconfigure(KmsConfig::static_kms("first-key".to_string(), BASE64_STANDARD.encode([0x11; 32]))) @@ -4692,6 +4982,15 @@ mod tests { ); } + #[test] + fn test_map_get_object_reader_error_preserves_typed_service_unavailable() { + let resolution_error = + super::EncryptionResolutionError::new(EncryptionResolutionErrorKind::ServiceUnavailable, "KMS unavailable"); + let err = map_get_object_reader_error(StorageError::other(resolution_error)); + assert_eq!(err.code, S3ErrorCode::ServiceUnavailable); + assert_eq!(err.message, "KMS unavailable"); + } + #[test] fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() { let err = map_get_object_reader_error(StorageError::other("plain io failure")); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 5cbf1554f..de8c9d9c3 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -509,12 +509,21 @@ pub(crate) mod ecstore_object { #[cfg(test)] pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource; pub(crate) use rustfs_ecstore::api::object::{ - GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len, - lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, - unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, + ObjectEncryptionResolver, ObjectMutationHook, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, + get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; } +#[cfg(all(test, feature = "rio-v2"))] +pub(crate) mod ecstore_test_support { + pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader; + pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk}; + pub(crate) use rustfs_ecstore::api::erasure::Erasure; + pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions}; +} + pub(crate) mod ecstore_set_disk { pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class}; } @@ -945,13 +954,21 @@ pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Res /// The process-level bootstrap instance context that single-instance startup /// threads through the storage foundation (Phase 5 follow-up, backlog#1052). pub(crate) fn bootstrap_instance_ctx() -> Arc { - ecstore_runtime::bootstrap_ctx() + let context = ecstore_runtime::bootstrap_ctx(); + configure_object_encryption_resolver(&context); + context } /// Construct a fresh per-server instance context (backlog#1052 S5): a second /// embedded server owns its own erasure/region/endpoint/deployment id cells. pub(crate) fn new_instance_ctx() -> Arc { - Arc::new(InstanceContext::new()) + let context = Arc::new(InstanceContext::new()); + configure_object_encryption_resolver(&context); + context +} + +fn configure_object_encryption_resolver(context: &InstanceContext) { + let _ = context.set_object_encryption_resolver(Arc::new(super::sse::SseObjectEncryptionResolver)); } pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) { @@ -1746,7 +1763,7 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc&2 exit 1 From 30dc04c94bfd17dc081994784daff507191311a8 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 30 Jul 2026 12:57:22 +0800 Subject: [PATCH 11/52] fix(obs): label node-local metrics by server (#5465) Add stable server labels to node-local Prometheus metrics and OTLP resource attributes so dashboards can distinguish per-node CPU, memory, host network, and internode traffic series. Co-authored-by: heihutu --- Cargo.lock | 2 + crates/io-metrics/Cargo.toml | 2 + crates/io-metrics/src/internode_metrics.rs | 253 ++++++++++++++---- crates/obs/src/lib.rs | 1 + crates/obs/src/metrics/collectors/resource.rs | 24 +- .../obs/src/metrics/collectors/system_cpu.rs | 36 ++- .../src/metrics/collectors/system_drive.rs | 35 ++- .../src/metrics/collectors/system_memory.rs | 48 +++- .../src/metrics/collectors/system_network.rs | 29 +- .../metrics/collectors/system_network_host.rs | 16 +- .../src/metrics/collectors/system_process.rs | 23 +- crates/obs/src/metrics/scheduler.rs | 28 +- .../src/metrics/schema/process_resource.rs | 7 +- crates/obs/src/metrics/schema/system_cpu.rs | 35 ++- .../obs/src/metrics/schema/system_memory.rs | 57 +++- .../obs/src/metrics/schema/system_network.rs | 11 +- .../src/metrics/schema/system_network_host.rs | 16 +- .../obs/src/metrics/schema/system_process.rs | 58 ++-- crates/obs/src/metrics/stats_collector.rs | 36 ++- crates/obs/src/node_identity.rs | 34 +++ crates/obs/src/telemetry/resource.rs | 53 +++- 21 files changed, 640 insertions(+), 164 deletions(-) create mode 100644 crates/obs/src/node_identity.rs diff --git a/Cargo.lock b/Cargo.lock index 3cde4a2ad..a2c01b043 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9353,7 +9353,9 @@ dependencies = [ "metrics", "metrics-util", "num_cpus", + "rustfs-common", "rustfs-s3-ops", + "rustfs-utils", "sysinfo", "thiserror 2.0.19", "tokio", diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index eb8d6fa22..d77725ae5 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -30,7 +30,9 @@ harness = false [dependencies] metrics = { workspace = true } +rustfs-common = { workspace = true } rustfs-s3-ops = { workspace = true } +rustfs-utils = { workspace = true, features = ["ip"] } num_cpus = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] } diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index 0aa69e6fb..790536b4a 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -15,7 +15,7 @@ use metrics::{counter, gauge}; use std::collections::HashMap; use std::sync::{ - Arc, LazyLock, RwLock, + Arc, LazyLock, OnceLock, RwLock, atomic::{AtomicU64, Ordering}, }; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -40,6 +40,7 @@ pub const INTERNODE_MSGPACK_CODEC_JSON: &str = "json"; const OPERATION_LABEL: &str = "operation"; const BACKEND_LABEL: &str = "backend"; +const SERVER_LABEL: &str = "server"; const CLASSIFICATION_LABEL: &str = "classification"; const STAGE_LABEL: &str = "stage"; const DOMINANT_ERROR_LABEL: &str = "dominant_error"; @@ -77,74 +78,93 @@ pub struct InternodeOperationMetricDescriptor { pub labels: &'static [&'static str], } -const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL]; -const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]; -const OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]; -const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL]; +const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]; +const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]; +const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]; +const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]; pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[ InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_SENT_BYTES_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_RECV_BYTES_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_ERRORS_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_DURATION_MS, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL, - labels: OPERATION_BACKEND_CLASSIFICATION_LABELS, + labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_RETRIES_TOTAL, - labels: OPERATION_BACKEND_CLASSIFICATION_LABELS, + labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL, - labels: OPERATION_BACKEND_CLASSIFICATION_LABELS, + labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL, - labels: OPERATION_BACKEND_HTTP_VERSION_LABELS, + labels: SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL, - labels: QUORUM_FAILURE_LABELS, + labels: SERVER_QUORUM_FAILURE_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_PAYLOAD_BYTES, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL, - labels: OPERATION_BACKEND_LABELS, + labels: SERVER_OPERATION_BACKEND_LABELS, }, ]; +fn current_server_label() -> &'static str { + static STABLE_SERVER_LABEL: OnceLock = OnceLock::new(); + static FALLBACK_SERVER_LABEL: LazyLock = LazyLock::new(rustfs_utils::get_local_ip_with_default); + + if let Some(server) = STABLE_SERVER_LABEL.get() { + return server.as_str(); + } + + if let Some(server) = rustfs_common::try_get_global_local_node_name() { + let _ = STABLE_SERVER_LABEL.set(server); + if let Some(server) = STABLE_SERVER_LABEL.get() { + return server.as_str(); + } + } + + FALLBACK_SERVER_LABEL.as_str() +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct InternodeMetricsSnapshot { pub sent_bytes_total: u64, @@ -193,7 +213,7 @@ impl InternodeMetrics { return; } self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed); - counter!("rustfs_system_network_internode_sent_bytes_total").increment(bytes); + counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes); } pub fn record_sent_bytes_for_operation(&self, operation: &'static str, bytes: usize) { @@ -207,7 +227,13 @@ impl InternodeMetrics { if bytes == 0 { return; } - counter!(INTERNODE_OPERATION_SENT_BYTES_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(bytes); + counter!( + INTERNODE_OPERATION_SENT_BYTES_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(bytes); } pub fn record_recv_bytes(&self, bytes: usize) { @@ -216,7 +242,7 @@ impl InternodeMetrics { return; } self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed); - counter!("rustfs_system_network_internode_recv_bytes_total").increment(bytes); + counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes); } pub fn record_recv_bytes_for_operation(&self, operation: &'static str, bytes: usize) { @@ -230,12 +256,18 @@ impl InternodeMetrics { if bytes == 0 { return; } - counter!(INTERNODE_OPERATION_RECV_BYTES_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(bytes); + counter!( + INTERNODE_OPERATION_RECV_BYTES_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(bytes); } pub fn record_outgoing_request(&self) { self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed); - counter!("rustfs_system_network_internode_requests_outgoing_total").increment(1); + counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => current_server_label()).increment(1); } pub fn record_outgoing_request_for_operation(&self, operation: &'static str) { @@ -244,13 +276,18 @@ impl InternodeMetrics { pub fn record_outgoing_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { self.record_outgoing_request(); - counter!(INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend) - .increment(1); + counter!( + INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(1); } pub fn record_incoming_request(&self) { self.incoming_requests_total.fetch_add(1, Ordering::Relaxed); - counter!("rustfs_system_network_internode_requests_incoming_total").increment(1); + counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => current_server_label()).increment(1); } pub fn record_incoming_request_for_operation(&self, operation: &'static str) { @@ -259,13 +296,18 @@ impl InternodeMetrics { pub fn record_incoming_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { self.record_incoming_request(); - counter!(INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend) - .increment(1); + counter!( + INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(1); } pub fn record_error(&self) { self.errors_total.fetch_add(1, Ordering::Relaxed); - counter!("rustfs_system_network_internode_errors_total").increment(1); + counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => current_server_label()).increment(1); } pub fn record_error_for_operation(&self, operation: &'static str) { @@ -274,13 +316,24 @@ impl InternodeMetrics { pub fn record_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { self.record_error(); - counter!(INTERNODE_OPERATION_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1); + counter!( + INTERNODE_OPERATION_ERRORS_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(1); } pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) { let duration_ms = duration.as_secs_f64() * 1000.0; - metrics::histogram!(INTERNODE_OPERATION_DURATION_MS, OPERATION_LABEL => operation, BACKEND_LABEL => backend) - .record(duration_ms); + metrics::histogram!( + INTERNODE_OPERATION_DURATION_MS, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .record(duration_ms); } pub fn record_classified_error_for_operation_and_backend( @@ -291,6 +344,7 @@ impl InternodeMetrics { ) { counter!( INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL, + SERVER_LABEL => current_server_label(), OPERATION_LABEL => operation, BACKEND_LABEL => backend, CLASSIFICATION_LABEL => classification @@ -306,6 +360,7 @@ impl InternodeMetrics { ) { counter!( INTERNODE_OPERATION_RETRIES_TOTAL, + SERVER_LABEL => current_server_label(), OPERATION_LABEL => operation, BACKEND_LABEL => backend, CLASSIFICATION_LABEL => classification @@ -321,6 +376,7 @@ impl InternodeMetrics { ) { counter!( INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL, + SERVER_LABEL => current_server_label(), OPERATION_LABEL => operation, BACKEND_LABEL => backend, CLASSIFICATION_LABEL => classification @@ -337,6 +393,7 @@ impl InternodeMetrics { self.operation_http_versions_total.fetch_add(1, Ordering::Relaxed); counter!( INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL, + SERVER_LABEL => current_server_label(), OPERATION_LABEL => operation, BACKEND_LABEL => backend, HTTP_VERSION_LABEL => http_version @@ -346,13 +403,24 @@ impl InternodeMetrics { pub fn record_stall_timeout_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { self.operation_stall_timeouts_total.fetch_add(1, Ordering::Relaxed); - counter!(INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1); + counter!( + INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(1); } pub fn record_write_shutdown_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) { self.operation_write_shutdown_errors_total.fetch_add(1, Ordering::Relaxed); - counter!(INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend) - .increment(1); + counter!( + INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(1); } /// Record the payload size (bytes) of a completed internode operation into a histogram @@ -360,15 +428,26 @@ impl InternodeMetrics { /// (`ReadAll`/`ReadMultiple`/`WriteAll`) would benefit from being moved off the shared /// control-plane channel (see docs/grpc-optimization P1). pub fn record_operation_payload_bytes(&self, operation: &'static str, backend: &'static str, bytes: usize) { - metrics::histogram!(INTERNODE_OPERATION_PAYLOAD_BYTES, OPERATION_LABEL => operation, BACKEND_LABEL => backend) - .record(bytes as f64); + metrics::histogram!( + INTERNODE_OPERATION_PAYLOAD_BYTES, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .record(bytes as f64); } /// Increment the large-payload counter for an operation+backend whose payload exceeded the /// caller-configured warning threshold. Feeds alerting on large unary RPCs that contend with /// latency-sensitive control-plane traffic on the shared connection. pub fn record_large_operation_payload(&self, operation: &'static str, backend: &'static str) { - counter!(INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1); + counter!( + INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend + ) + .increment(1); } /// Count a decode that fell back to the JSON compatibility field because the msgpack `_bin` @@ -377,13 +456,20 @@ impl InternodeMetrics { /// dropped (grpc-optimization P2). `direction` is [`INTERNODE_MSGPACK_DIRECTION_REQUEST`] or /// [`INTERNODE_MSGPACK_DIRECTION_RESPONSE`]; `message` is the low-cardinality value name. pub fn record_msgpack_json_fallback(&self, direction: &'static str, message: &'static str) { - counter!(INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL, DIRECTION_LABEL => direction, MESSAGE_LABEL => message).increment(1); + counter!( + INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL, + SERVER_LABEL => current_server_label(), + DIRECTION_LABEL => direction, + MESSAGE_LABEL => message + ) + .increment(1); } pub fn record_msgpack_json_decode(&self, direction: &'static str, message: &'static str, codec: &'static str) { self.msgpack_json_decode_total.fetch_add(1, Ordering::Relaxed); counter!( INTERNODE_MSGPACK_JSON_DECODE_TOTAL, + SERVER_LABEL => current_server_label(), DIRECTION_LABEL => direction, MESSAGE_LABEL => message, CODEC_LABEL => codec @@ -395,6 +481,7 @@ impl InternodeMetrics { self.msgpack_json_decode_error_total.fetch_add(1, Ordering::Relaxed); counter!( INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL, + SERVER_LABEL => current_server_label(), DIRECTION_LABEL => direction, MESSAGE_LABEL => message, CODEC_LABEL => codec @@ -420,7 +507,7 @@ impl InternodeMetrics { /// enabled; after the strict flip the legacy fallback path is closed and the counter stays flat. pub fn record_signature_v1_fallback(&self) { self.signature_v1_fallback_total.fetch_add(1, Ordering::Relaxed); - counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1); + counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL, SERVER_LABEL => current_server_label()).increment(1); } /// Count a mutating internode disk RPC that was accepted without a signature-bound canonical @@ -431,14 +518,14 @@ impl InternodeMetrics { /// mutations are rejected and the counter stays flat. pub fn record_body_digest_fallback(&self) { self.body_digest_fallback_total.fetch_add(1, Ordering::Relaxed); - counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1); + counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL, SERVER_LABEL => current_server_label()).increment(1); } /// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is /// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`. pub fn record_replay_scope_fallback(&self) { self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed); - counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1); + counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL, SERVER_LABEL => current_server_label()).increment(1); } /// Count a body-bound internode RPC rejected because the replay-protection nonce cache was @@ -447,12 +534,13 @@ impl InternodeMetrics { /// mutation rate and writes are being refused — alert on this counter. pub fn record_replay_cache_overflow(&self) { self.replay_cache_overflow_total.fetch_add(1, Ordering::Relaxed); - counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL).increment(1); + counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL, SERVER_LABEL => current_server_label()).increment(1); } pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) { counter!( ERASURE_WRITE_QUORUM_FAILURES_TOTAL, + SERVER_LABEL => current_server_label(), STAGE_LABEL => stage, DOMINANT_ERROR_LABEL => dominant_error ) @@ -464,11 +552,12 @@ impl InternodeMetrics { self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed); let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1; let total = self.dial_total_time_nanos.load(Ordering::Relaxed); - gauge!("rustfs_system_network_internode_dial_avg_time_nanos").set(total as f64 / samples as f64); + gauge!("rustfs_system_network_internode_dial_avg_time_nanos", SERVER_LABEL => current_server_label()) + .set(total as f64 / samples as f64); if !success { self.dial_errors_total.fetch_add(1, Ordering::Relaxed); - counter!("rustfs_system_network_internode_dial_errors_total").increment(1); + counter!("rustfs_system_network_internode_dial_errors_total", SERVER_LABEL => current_server_label()).increment(1); } let now_ms = SystemTime::now() @@ -687,6 +776,9 @@ fn cluster_peer_health_keys() -> Vec { #[cfg(test)] mod tests { use super::*; + use metrics::with_local_recorder; + use metrics_util::debugging::DebuggingRecorder; + use std::collections::HashSet; #[test] fn snapshot_reports_recorded_values() { @@ -750,22 +842,22 @@ mod tests { fn operation_metric_descriptors_include_backend_and_operation_labels() { assert_eq!(INTERNODE_OPERATION_METRICS.len(), 15); for metric in &INTERNODE_OPERATION_METRICS[..6] { - assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]); + assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } for metric in &INTERNODE_OPERATION_METRICS[6..9] { - assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]); + assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]); } assert_eq!( INTERNODE_OPERATION_METRICS[9].labels, - &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL] + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL] ); for metric in &INTERNODE_OPERATION_METRICS[10..12] { - assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]); + assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } - assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]); // Payload histogram + large-payload counter carry operation+backend labels. - assert_eq!(INTERNODE_OPERATION_METRICS[13].labels, &[OPERATION_LABEL, BACKEND_LABEL]); - assert_eq!(INTERNODE_OPERATION_METRICS[14].labels, &[OPERATION_LABEL, BACKEND_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[13].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[14].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } #[test] @@ -843,6 +935,59 @@ mod tests { ); } + #[test] + fn direct_internode_metrics_emit_stable_server_label() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let metrics = InternodeMetrics::default(); + + with_local_recorder(&recorder, || { + metrics.record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + 128, + ); + metrics.record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_PUT_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + 256, + ); + metrics.record_dial_result(Duration::from_millis(3), false); + }); + + let observed: Vec<(String, HashSet, Option)> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(composite, _, _, _)| { + matches!( + composite.key().name(), + "rustfs_system_network_internode_sent_bytes_total" + | "rustfs_system_network_internode_recv_bytes_total" + | INTERNODE_OPERATION_SENT_BYTES_TOTAL + | INTERNODE_OPERATION_RECV_BYTES_TOTAL + | "rustfs_system_network_internode_dial_avg_time_nanos" + | "rustfs_system_network_internode_dial_errors_total" + ) + }) + .map(|(composite, _, _, _)| { + let labels = composite.key().labels(); + let keys = labels.clone().map(|label| label.key().to_string()).collect(); + let server = labels + .filter(|label| label.key() == SERVER_LABEL) + .map(|label| label.value().to_string()) + .next(); + (composite.key().name().to_string(), keys, server) + }) + .collect(); + + assert_eq!(observed.len(), 6); + for (name, keys, server) in observed { + assert!(keys.contains(SERVER_LABEL), "{name} must carry the server label"); + assert!(server.is_some_and(|value| !value.is_empty()), "{name} server label must not be empty"); + } + } + #[test] fn msgpack_json_fallback_counter_records_without_panicking() { // Smoke test: the counter accepts both directions and a static message label. diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index b2919d9b2..55b079e6b 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -64,6 +64,7 @@ mod error; mod global; mod logging; pub mod metrics; +mod node_identity; mod telemetry; pub use cleaner::*; diff --git a/crates/obs/src/metrics/collectors/resource.rs b/crates/obs/src/metrics/collectors/resource.rs index 7d7e0ee68..011859ce6 100644 --- a/crates/obs/src/metrics/collectors/resource.rs +++ b/crates/obs/src/metrics/collectors/resource.rs @@ -22,6 +22,7 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::process_resource::*; +use crate::node_identity::SERVER_LABEL; /// Resource statistics for metrics collection. /// @@ -30,6 +31,8 @@ use crate::metrics::schema::process_resource::*; /// this struct from their available data sources. #[derive(Debug, Clone, Default)] pub struct ResourceStats { + /// Stable local node identity for labeling node-local process resource metrics + pub server: String, /// CPU usage as a percentage (can exceed 100% on multi-core systems) pub cpu_percent: f64, /// Resident memory usage in bytes @@ -43,10 +46,14 @@ pub struct ResourceStats { /// Uses the metric descriptors from `metrics_type::process_resource` module. /// Returns a vector of Prometheus metrics for resource statistics. pub fn collect_resource_metrics(stats: &ResourceStats) -> Vec { + let server_label = stats.server.as_str(); vec![ - PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent), - PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64), - PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64), + PrometheusMetric::from_descriptor(&PROCESS_CPU_PERCENT_MD, stats.cpu_percent) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&PROCESS_MEMORY_BYTES_MD, stats.memory_bytes as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&PROCESS_UPTIME_SECONDS_MD, stats.uptime_seconds as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), ] } @@ -58,6 +65,7 @@ mod tests { #[test] fn test_collect_resource_metrics() { let stats = ResourceStats { + server: "node1:9000".to_string(), cpu_percent: 45.5, memory_bytes: 1024 * 1024 * 256, uptime_seconds: 7200, @@ -67,6 +75,11 @@ mod tests { report_metrics(&metrics); assert_eq!(metrics.len(), 3); + assert!( + metrics + .iter() + .all(|m| m.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")) + ); // Verify CPU metric let cpu_metric_name = PROCESS_CPU_PERCENT_MD.get_full_metric_name(); @@ -97,13 +110,16 @@ mod tests { for metric in &metrics { assert_eq!(metric.value, 0.0); - assert!(metric.labels.is_empty()); + assert_eq!(metric.labels.len(), 1); + assert_eq!(metric.labels[0].0, SERVER_LABEL); + assert!(metric.labels[0].1.is_empty()); } } #[test] fn test_collect_resource_metrics_high_cpu() { let stats = ResourceStats { + server: "node1:9000".to_string(), cpu_percent: 150.0, // Can exceed 100% on multi-core systems memory_bytes: 0, uptime_seconds: 0, diff --git a/crates/obs/src/metrics/collectors/system_cpu.rs b/crates/obs/src/metrics/collectors/system_cpu.rs index 302e9fbe0..5e829a8b4 100644 --- a/crates/obs/src/metrics/collectors/system_cpu.rs +++ b/crates/obs/src/metrics/collectors/system_cpu.rs @@ -25,11 +25,14 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::system_cpu::*; use crate::metrics::schema::system_process::{PROCESS_CPU_USAGE_MD, PROCESS_CPU_UTILIZATION_MD}; +use crate::node_identity::SERVER_LABEL; use std::borrow::Cow; /// System CPU statistics. #[derive(Debug, Clone, Default)] pub struct CpuStats { + /// Stable local node identity for labeling node-local CPU metrics + pub server: String, /// Average CPU idle time (percentage, 0-100) pub avg_idle: f64, /// CPU load average over 1 minute @@ -56,11 +59,16 @@ pub struct ProcessCpuStats { /// Uses the metric descriptors from `metrics_type::system_cpu` module. /// Returns a vector of Prometheus metrics for CPU statistics. pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec { + let server_label = stats.server.as_str(); vec![ - PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle), - PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg), - PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc), - PrometheusMetric::from_descriptor(&SYS_CPU_USAGE_PERC_MD, stats.usage_perc), + PrometheusMetric::from_descriptor(&SYS_CPU_AVG_IDLE_MD, stats.avg_idle) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_MD, stats.load_avg) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&SYS_CPU_LOAD_PERC_MD, stats.load_avg_perc) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&SYS_CPU_USAGE_PERC_MD, stats.usage_perc) + .with_label_owned(SERVER_LABEL, server_label.to_string()), ] } @@ -91,10 +99,12 @@ pub fn collect_process_cpu_metrics( mod tests { use super::*; use crate::metrics::report::report_metrics; + use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; #[test] fn test_collect_cpu_metrics() { let stats = CpuStats { + server: "node1:9000".to_string(), avg_idle: 75.5, load_avg: 1.5, load_avg_perc: 37.5, @@ -108,6 +118,11 @@ mod tests { // Verify that metric names are properly generated from descriptors assert!(metrics.iter().all(|m| m.name.starts_with("rustfs_system_cpu_"))); + assert!( + metrics + .iter() + .all(|m| m.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")) + ); } #[test] @@ -118,13 +133,16 @@ mod tests { assert_eq!(metrics.len(), 4); for metric in &metrics { assert_eq!(metric.value, 0.0); - assert!(metric.labels.is_empty()); + assert_eq!(metric.labels.len(), 1); + assert_eq!(metric.labels[0].0, SERVER_LABEL); + assert!(metric.labels[0].1.is_empty()); } } #[test] fn system_cpu_metrics_export_total_usage_under_honest_name() { let stats = CpuStats { + server: "node1:9000".to_string(), avg_idle: 40.0, load_avg: 1.0, load_avg_perc: 25.0, @@ -171,8 +189,9 @@ mod tests { }; let labels = vec![ - ("process_pid", Cow::Borrowed("12345")), - ("process_executable_name", Cow::Borrowed("rustfs")), + (SERVER_LABEL, Cow::Borrowed("node1:9000")), + (PROCESS_PID_LABEL, Cow::Borrowed("12345")), + (PROCESS_EXECUTABLE_NAME_LABEL, Cow::Borrowed("rustfs")), ]; let metrics = collect_process_cpu_metrics(&stats, Some(&labels)); @@ -180,7 +199,8 @@ mod tests { // All metrics should have the labels for metric in &metrics { - assert_eq!(metric.labels.len(), 2); + assert_eq!(metric.labels.len(), 3); + assert!(metric.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")); } } } diff --git a/crates/obs/src/metrics/collectors/system_drive.rs b/crates/obs/src/metrics/collectors/system_drive.rs index 521ee3f59..7cda25a07 100644 --- a/crates/obs/src/metrics/collectors/system_drive.rs +++ b/crates/obs/src/metrics/collectors/system_drive.rs @@ -24,7 +24,7 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::system_drive::*; -use crate::metrics::schema::system_process::PROCESS_DISK_IO_MD; +use crate::metrics::schema::system_process::{DIRECTION_LABEL, PROCESS_DISK_IO_MD}; use std::borrow::Cow; /// Detailed drive statistics for a single drive. @@ -223,8 +223,8 @@ pub fn collect_process_disk_metrics( let mut read_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.read_bytes as f64); let mut write_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.written_bytes as f64); - read_metric.labels.push(("direction", Cow::Borrowed("read"))); - write_metric.labels.push(("direction", Cow::Borrowed("write"))); + read_metric.labels.push((DIRECTION_LABEL, Cow::Borrowed("read"))); + write_metric.labels.push((DIRECTION_LABEL, Cow::Borrowed("write"))); if let Some(l) = labels { read_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); @@ -238,6 +238,7 @@ pub fn collect_process_disk_metrics( mod tests { use super::*; use crate::metrics::report::report_metrics; + use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; use std::collections::BTreeSet; fn assert_metric_label_keys( @@ -371,4 +372,32 @@ mod tests { assert!(offline.is_some()); assert_eq!(offline.map(|m| m.value), Some(2.0)); } + + #[test] + fn test_collect_process_disk_metrics_with_node_and_process_labels() { + let stats = ProcessDiskStats { + read_bytes: 1024, + written_bytes: 2048, + }; + let labels = vec![ + (SERVER_LABEL, Cow::Borrowed("node1:9000")), + (PROCESS_PID_LABEL, Cow::Borrowed("12345")), + (PROCESS_EXECUTABLE_NAME_LABEL, Cow::Borrowed("rustfs")), + ]; + + let metrics = collect_process_disk_metrics(&stats, Some(&labels)); + + assert_eq!(metrics.len(), 2); + assert_metric_label_keys( + &metrics, + &PROCESS_DISK_IO_MD, + 1024.0, + &[ + DIRECTION_LABEL, + SERVER_LABEL, + PROCESS_PID_LABEL, + PROCESS_EXECUTABLE_NAME_LABEL, + ], + ); + } } diff --git a/crates/obs/src/metrics/collectors/system_memory.rs b/crates/obs/src/metrics/collectors/system_memory.rs index 78a4d625e..c9e319365 100644 --- a/crates/obs/src/metrics/collectors/system_memory.rs +++ b/crates/obs/src/metrics/collectors/system_memory.rs @@ -25,11 +25,14 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::system_memory::*; use crate::metrics::schema::system_process::{PROCESS_RESIDENT_MEMORY_BYTES_MD, PROCESS_VIRTUAL_MEMORY_BYTES_MD}; +use crate::node_identity::SERVER_LABEL; use std::borrow::Cow; /// System memory statistics. #[derive(Debug, Clone, Default)] pub struct MemoryStats { + /// Stable local node identity for labeling node-local memory metrics + pub server: String, /// Total memory in bytes pub total: u64, /// Used memory in bytes @@ -64,15 +67,24 @@ pub struct ProcessMemoryStats { /// Uses the metric descriptors from `metrics_type::system_memory` module. /// Returns a vector of Prometheus metrics for memory statistics. pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec { + let server_label = stats.server.as_str(); vec![ - PrometheusMetric::from_descriptor(&MEM_TOTAL_MD, stats.total as f64), - PrometheusMetric::from_descriptor(&MEM_USED_MD, stats.used as f64), - PrometheusMetric::from_descriptor(&MEM_USED_PERC_MD, stats.used_perc), - PrometheusMetric::from_descriptor(&MEM_FREE_MD, stats.free as f64), - PrometheusMetric::from_descriptor(&MEM_BUFFERS_MD, stats.buffers as f64), - PrometheusMetric::from_descriptor(&MEM_CACHE_MD, stats.cache as f64), - PrometheusMetric::from_descriptor(&MEM_SHARED_MD, stats.shared as f64), - PrometheusMetric::from_descriptor(&MEM_AVAILABLE_MD, stats.available as f64), + PrometheusMetric::from_descriptor(&MEM_TOTAL_MD, stats.total as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_USED_MD, stats.used as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_USED_PERC_MD, stats.used_perc) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_FREE_MD, stats.free as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_BUFFERS_MD, stats.buffers as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_CACHE_MD, stats.cache as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_SHARED_MD, stats.shared as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&MEM_AVAILABLE_MD, stats.available as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), ] } @@ -104,10 +116,12 @@ pub fn collect_process_memory_metrics( mod tests { use super::*; use crate::metrics::report::report_metrics; + use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; #[test] fn test_collect_memory_metrics() { let stats = MemoryStats { + server: "node1:9000".to_string(), total: 16 * 1024 * 1024 * 1024, // 16 GB used: 8 * 1024 * 1024 * 1024, // 8 GB used_perc: 50.0, @@ -123,6 +137,11 @@ mod tests { assert_eq!(metrics.len(), 8); assert!(metrics.iter().all(|m| m.name.starts_with("rustfs_system_memory_"))); + assert!( + metrics + .iter() + .all(|m| m.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")) + ); } #[test] @@ -133,7 +152,9 @@ mod tests { assert_eq!(metrics.len(), 8); for metric in &metrics { assert_eq!(metric.value, 0.0); - assert!(metric.labels.is_empty()); + assert_eq!(metric.labels.len(), 1); + assert_eq!(metric.labels[0].0, SERVER_LABEL); + assert!(metric.labels[0].1.is_empty()); } } @@ -157,13 +178,18 @@ mod tests { virtual_mem: 1024 * 1024 * 1024, }; - let labels = vec![("process_pid", Cow::Borrowed("12345"))]; + let labels = vec![ + (SERVER_LABEL, Cow::Borrowed("node1:9000")), + (PROCESS_PID_LABEL, Cow::Borrowed("12345")), + (PROCESS_EXECUTABLE_NAME_LABEL, Cow::Borrowed("rustfs")), + ]; let metrics = collect_process_memory_metrics(&stats, Some(&labels)); assert_eq!(metrics.len(), 2); for metric in &metrics { - assert_eq!(metric.labels.len(), 1); + assert_eq!(metric.labels.len(), 3); + assert!(metric.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")); } } } diff --git a/crates/obs/src/metrics/collectors/system_network.rs b/crates/obs/src/metrics/collectors/system_network.rs index 4392e398c..8bb660de0 100644 --- a/crates/obs/src/metrics/collectors/system_network.rs +++ b/crates/obs/src/metrics/collectors/system_network.rs @@ -23,10 +23,13 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::system_network::*; +use crate::node_identity::SERVER_LABEL; /// Network statistics for internode communication. #[derive(Debug, Clone, Default)] pub struct NetworkStats { + /// Stable local node identity for labeling node-local internode metrics + pub server: String, /// Total number of failed internode calls pub internode_errors_total: u64, /// Total number of TCP dial timeouts and errors @@ -44,12 +47,18 @@ pub struct NetworkStats { /// Uses the metric descriptors from `metrics_type::system_network` module. /// Returns a vector of Prometheus metrics for network statistics. pub fn collect_network_metrics(stats: &NetworkStats) -> Vec { + let server_label = stats.server.as_str(); vec![ - PrometheusMetric::from_descriptor(&INTERNODE_ERRORS_TOTAL_MD, stats.internode_errors_total as f64), - PrometheusMetric::from_descriptor(&INTERNODE_DIAL_ERRORS_TOTAL_MD, stats.internode_dial_errors_total as f64), - PrometheusMetric::from_descriptor(&INTERNODE_DIAL_AVG_TIME_NANOS_MD, stats.internode_dial_avg_time_nanos as f64), - PrometheusMetric::from_descriptor(&INTERNODE_SENT_BYTES_TOTAL_MD, stats.internode_sent_bytes_total as f64), - PrometheusMetric::from_descriptor(&INTERNODE_RECV_BYTES_TOTAL_MD, stats.internode_recv_bytes_total as f64), + PrometheusMetric::from_descriptor(&INTERNODE_ERRORS_TOTAL_MD, stats.internode_errors_total as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&INTERNODE_DIAL_ERRORS_TOTAL_MD, stats.internode_dial_errors_total as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&INTERNODE_DIAL_AVG_TIME_NANOS_MD, stats.internode_dial_avg_time_nanos as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&INTERNODE_SENT_BYTES_TOTAL_MD, stats.internode_sent_bytes_total as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), + PrometheusMetric::from_descriptor(&INTERNODE_RECV_BYTES_TOTAL_MD, stats.internode_recv_bytes_total as f64) + .with_label_owned(SERVER_LABEL, server_label.to_string()), ] } @@ -61,6 +70,7 @@ mod tests { #[test] fn test_collect_network_metrics() { let stats = NetworkStats { + server: "node1:9000".to_string(), internode_errors_total: 10, internode_dial_errors_total: 5, internode_dial_avg_time_nanos: 1_500_000, // 1.5ms @@ -73,6 +83,11 @@ mod tests { assert_eq!(metrics.len(), 5); assert!(metrics.iter().all(|m| m.name.contains("internode"))); + assert!( + metrics + .iter() + .all(|m| m.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")) + ); } #[test] @@ -83,7 +98,9 @@ mod tests { assert_eq!(metrics.len(), 5); for metric in &metrics { assert_eq!(metric.value, 0.0); - assert!(metric.labels.is_empty()); + assert_eq!(metric.labels.len(), 1); + assert_eq!(metric.labels[0].0, SERVER_LABEL); + assert!(metric.labels[0].1.is_empty()); } } } diff --git a/crates/obs/src/metrics/collectors/system_network_host.rs b/crates/obs/src/metrics/collectors/system_network_host.rs index 9dccd9d20..083ad8d69 100644 --- a/crates/obs/src/metrics/collectors/system_network_host.rs +++ b/crates/obs/src/metrics/collectors/system_network_host.rs @@ -18,6 +18,7 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::system_network_host::{ DIRECTION_LABEL, HOST_NETWORK_IO_MD, HOST_NETWORK_IO_PER_INTERFACE_MD, INTERFACE_LABEL, }; +use crate::node_identity::SERVER_LABEL; use std::borrow::Cow; /// Network I/O statistics. @@ -25,6 +26,8 @@ use std::borrow::Cow; /// Contains host-wide network I/O totals and per-interface counters. #[derive(Debug, Clone, Default)] pub struct HostNetworkStats { + /// Stable local node identity for labeling host-wide network metrics. + pub server: String, /// Total bytes received across observed host interfaces. pub total_received: u64, /// Total bytes transmitted across observed host interfaces. @@ -47,7 +50,11 @@ pub fn collect_host_network_metrics( let mut received_metric = PrometheusMetric::from_descriptor(&HOST_NETWORK_IO_MD, stats.total_received as f64); let mut transmitted_metric = PrometheusMetric::from_descriptor(&HOST_NETWORK_IO_MD, stats.total_transmitted as f64); + received_metric.labels.push((SERVER_LABEL, Cow::Owned(stats.server.clone()))); received_metric.labels.push((DIRECTION_LABEL, Cow::Borrowed("received"))); + transmitted_metric + .labels + .push((SERVER_LABEL, Cow::Owned(stats.server.clone()))); transmitted_metric .labels .push((DIRECTION_LABEL, Cow::Borrowed("transmitted"))); @@ -64,9 +71,13 @@ pub fn collect_host_network_metrics( let mut iface_received = PrometheusMetric::from_descriptor(&HOST_NETWORK_IO_PER_INTERFACE_MD, *received as f64); let mut iface_transmitted = PrometheusMetric::from_descriptor(&HOST_NETWORK_IO_PER_INTERFACE_MD, *transmitted as f64); + iface_received.labels.push((SERVER_LABEL, Cow::Owned(stats.server.clone()))); iface_received.labels.push((INTERFACE_LABEL, Cow::Owned(interface.clone()))); iface_received.labels.push((DIRECTION_LABEL, Cow::Borrowed("received"))); + iface_transmitted + .labels + .push((SERVER_LABEL, Cow::Owned(stats.server.clone()))); iface_transmitted .labels .push((INTERFACE_LABEL, Cow::Owned(interface.clone()))); @@ -92,6 +103,7 @@ mod tests { #[test] fn host_network_metrics_use_dedicated_network_host_prefix() { let stats = HostNetworkStats { + server: "node1:9000".to_string(), total_received: 1024, total_transmitted: 2048, per_interface: vec![("eth0".to_string(), 512, 256)], @@ -107,9 +119,9 @@ mod tests { ); let total_keys: BTreeSet<&str> = metrics[0].labels.iter().map(|(key, _)| *key).collect(); - assert_eq!(total_keys, BTreeSet::from([DIRECTION_LABEL])); + assert_eq!(total_keys, BTreeSet::from([SERVER_LABEL, DIRECTION_LABEL])); let per_interface_keys: BTreeSet<&str> = metrics[2].labels.iter().map(|(key, _)| *key).collect(); - assert_eq!(per_interface_keys, BTreeSet::from([DIRECTION_LABEL, INTERFACE_LABEL])); + assert_eq!(per_interface_keys, BTreeSet::from([SERVER_LABEL, DIRECTION_LABEL, INTERFACE_LABEL])); } } diff --git a/crates/obs/src/metrics/collectors/system_process.rs b/crates/obs/src/metrics/collectors/system_process.rs index ebdcb2516..708c1e975 100644 --- a/crates/obs/src/metrics/collectors/system_process.rs +++ b/crates/obs/src/metrics/collectors/system_process.rs @@ -24,6 +24,7 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::system_process::*; +use crate::node_identity::SERVER_LABEL; use std::borrow::Cow; use sysinfo::{Pid, ProcessStatus, System}; @@ -146,6 +147,8 @@ impl From for ProcessStatusType { /// Process statistics for the RustFS server process. #[derive(Debug, Clone, Default)] pub struct ProcessStats { + /// Stable local node identity for labeling node-local process metrics + pub server: String, /// Total read locks held pub locks_read_total: u64, /// Total write locks held @@ -190,6 +193,7 @@ pub struct ProcessStats { /// /// Returns a vector of Prometheus metrics for process statistics. pub fn collect_process_metrics(stats: &ProcessStats) -> Vec { + let server_label = stats.server.as_str(); let mut metrics = vec![ PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64), PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64), @@ -209,12 +213,18 @@ pub fn collect_process_metrics(stats: &ProcessStats) -> Vec { PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64), PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64), ]; + for metric in &mut metrics { + metric.labels.push((SERVER_LABEL, Cow::Owned(server_label.to_string()))); + } // Add process status metric let mut status_metric = PrometheusMetric::from_descriptor(&PROCESS_STATUS_MD, stats.status_value as f64); status_metric .labels - .push(("status", Cow::Owned(format!("{:?}", stats.status)))); + .push((SERVER_LABEL, Cow::Owned(server_label.to_string()))); + status_metric + .labels + .push((STATUS_LABEL, Cow::Owned(format!("{:?}", stats.status)))); metrics.push(status_metric); metrics @@ -235,6 +245,7 @@ mod tests { #[test] fn test_collect_process_metrics() { let stats = ProcessStats { + server: "node1:9000".to_string(), locks_read_total: 100, locks_write_total: 50, cpu_total_seconds: 1234.56, @@ -261,6 +272,11 @@ mod tests { // 17 original metrics + 1 status metric = 18 assert_eq!(metrics.len(), 18); + assert!( + metrics + .iter() + .all(|m| m.labels.iter().any(|(k, v)| *k == SERVER_LABEL && v == "node1:9000")) + ); // Verify uptime let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name(); @@ -288,6 +304,7 @@ mod tests { // 17 original metrics + 1 status metric = 18 assert_eq!(metrics.len(), 18); + assert!(metrics.iter().all(|m| m.labels.iter().any(|(k, _)| *k == SERVER_LABEL))); } #[test] @@ -296,7 +313,7 @@ mod tests { let result = collect_process_attributes(); assert!(result.is_ok()); - let attrs = result.unwrap(); + let attrs = result.expect("current process attributes should be collectable"); assert!(attrs.pid > 0); assert!(!attrs.executable_name.is_empty()); } @@ -319,7 +336,7 @@ mod tests { let labels = attrs.to_labels(); assert_eq!(labels.len(), 4); - assert_eq!(labels[0].0, "process_pid"); + assert_eq!(labels[0].0, PROCESS_PID_LABEL); assert_eq!(labels[0].1, "12345"); } } diff --git a/crates/obs/src/metrics/scheduler.rs b/crates/obs/src/metrics/scheduler.rs index e5a002aa0..189c0c04d 100644 --- a/crates/obs/src/metrics/scheduler.rs +++ b/crates/obs/src/metrics/scheduler.rs @@ -92,6 +92,7 @@ use crate::metrics::schema::notification_target::{ NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, TARGET_ID as NOTIFICATION_TARGET_ID_LABEL, TARGET_TYPE as NOTIFICATION_TARGET_TYPE_LABEL, }; +use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; use crate::metrics::stats_collector::{ ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats, collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats, @@ -100,6 +101,7 @@ use crate::metrics::stats_collector::{ collect_process_metric_bundle_with, collect_replication_stats, collect_scanner_metric_stats, collect_system_cpu_and_memory_stats_with, }; +use crate::node_identity::{SERVER_LABEL, current_local_node_identity}; use crate::telemetry::retire_metric_series; use futures_util::FutureExt; use rustfs_audit::audit_target_metrics; @@ -1509,7 +1511,7 @@ pub fn init_metrics_runtime(token: CancellationToken) { let token_clone = token.clone(); tokio::spawn(async move { - let labels = current_process_metric_labels(); + let process_attribute_labels = current_process_attribute_labels(); let mut host_system = System::new_all(); let mut host_networks = Networks::new(); let mut process_sampler = ProcessSampler::new(); @@ -1560,6 +1562,7 @@ pub fn init_metrics_runtime(token: CancellationToken) { } if now >= next_system_run { + let labels = current_process_metric_labels(&process_attribute_labels); #[cfg(feature = "gpu")] let mut metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system, &mut host_networks); @@ -1686,24 +1689,33 @@ fn advance_deadline(deadline: &mut Instant, interval: Duration, now: Instant) { } } -fn current_process_metric_labels() -> Vec<(&'static str, Cow<'static, str>)> { +fn current_process_attribute_labels() -> Vec<(&'static str, Cow<'static, str>)> { match collect_process_attributes() { Ok(attrs) => vec![ - ("process_pid", Cow::Owned(attrs.pid.to_string())), - ("process_executable_name", Cow::Owned(attrs.executable_name)), + (PROCESS_PID_LABEL, Cow::Owned(attrs.pid.to_string())), + (PROCESS_EXECUTABLE_NAME_LABEL, Cow::Owned(attrs.executable_name)), ], - Err(err) => fallback_process_metric_labels(err), + Err(err) => fallback_process_attribute_labels(err), } } -fn fallback_process_metric_labels(err: ProcessAttributeError) -> Vec<(&'static str, Cow<'static, str>)> { +fn fallback_process_attribute_labels(err: ProcessAttributeError) -> Vec<(&'static str, Cow<'static, str>)> { warn!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "process_metric_labels", result = "collect_failed", error = %err, "metrics runtime state changed"); vec![ - ("process_pid", Cow::Owned(std::process::id().to_string())), - ("process_executable_name", Cow::Borrowed("unknown")), + (PROCESS_PID_LABEL, Cow::Owned(std::process::id().to_string())), + (PROCESS_EXECUTABLE_NAME_LABEL, Cow::Borrowed("unknown")), ] } +fn current_process_metric_labels( + process_attribute_labels: &[(&'static str, Cow<'static, str>)], +) -> Vec<(&'static str, Cow<'static, str>)> { + let mut labels = Vec::with_capacity(process_attribute_labels.len() + 1); + labels.push((SERVER_LABEL, Cow::Owned(current_local_node_identity()))); + labels.extend(process_attribute_labels.iter().map(|(key, value)| (*key, value.clone()))); + labels +} + fn collect_system_monitoring_metrics( bundle: &ProcessMetricBundle, labels: &[(&'static str, Cow<'static, str>)], diff --git a/crates/obs/src/metrics/schema/process_resource.rs b/crates/obs/src/metrics/schema/process_resource.rs index 9c0c03e34..5bac84f5a 100644 --- a/crates/obs/src/metrics/schema/process_resource.rs +++ b/crates/obs/src/metrics/schema/process_resource.rs @@ -14,6 +14,7 @@ #![allow(dead_code)] +use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, MetricSubsystem, new_gauge_md}; use std::sync::LazyLock; @@ -22,7 +23,7 @@ pub static PROCESS_CPU_PERCENT_MD: LazyLock = LazyLock::new(|| new_gauge_md( MetricName::Custom("cpu_percent".to_string()), "CPU usage of the RustFS process as a percentage", - &[], + &[SERVER_LABEL], MetricSubsystem::new("/process"), ) }); @@ -32,7 +33,7 @@ pub static PROCESS_MEMORY_BYTES_MD: LazyLock = LazyLock::new(| new_gauge_md( MetricName::Custom("memory_bytes".to_string()), "Resident memory usage of the RustFS process in bytes", - &[], + &[SERVER_LABEL], MetricSubsystem::new("/process"), ) }); @@ -42,7 +43,7 @@ pub static PROCESS_UPTIME_SECONDS_MD: LazyLock = LazyLock::new new_gauge_md( MetricName::Custom("uptime_seconds".to_string()), "Uptime of the RustFS process in seconds", - &[], + &[SERVER_LABEL], MetricSubsystem::new("/process"), ) }); diff --git a/crates/obs/src/metrics/schema/system_cpu.rs b/crates/obs/src/metrics/schema/system_cpu.rs index 350b8310d..f0a3ea784 100644 --- a/crates/obs/src/metrics/schema/system_cpu.rs +++ b/crates/obs/src/metrics/schema/system_cpu.rs @@ -14,24 +14,37 @@ #![allow(dead_code)] +use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; /// CPU system-related metric descriptors use std::sync::LazyLock; -pub static SYS_CPU_AVG_IDLE_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIdle, "Average CPU idle time", &[], subsystems::SYSTEM_CPU)); +pub static SYS_CPU_AVG_IDLE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::SysCPUAvgIdle, + "Average CPU idle time", + &[SERVER_LABEL], + subsystems::SYSTEM_CPU, + ) +}); -pub static SYS_CPU_AVG_IOWAIT_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPUAvgIOWait, "Average CPU IOWait time", &[], subsystems::SYSTEM_CPU)); +pub static SYS_CPU_AVG_IOWAIT_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::SysCPUAvgIOWait, + "Average CPU IOWait time", + &[SERVER_LABEL], + subsystems::SYSTEM_CPU, + ) +}); pub static SYS_CPU_LOAD_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPULoad, "CPU load average 1min", &[], subsystems::SYSTEM_CPU)); + LazyLock::new(|| new_gauge_md(MetricName::SysCPULoad, "CPU load average 1min", &[SERVER_LABEL], subsystems::SYSTEM_CPU)); pub static SYS_CPU_LOAD_PERC_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::SysCPULoadPerc, "CPU load average 1min (percentage)", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_CPU, ) }); @@ -40,19 +53,19 @@ pub static SYS_CPU_USAGE_PERC_MD: LazyLock = LazyLock::new(|| new_gauge_md( MetricName::Custom("usage_perc".to_string()), "Total CPU usage percentage across all measured CPU time categories", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_CPU, ) }); pub static SYS_CPU_NICE_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPUNice, "CPU nice time", &[], subsystems::SYSTEM_CPU)); + LazyLock::new(|| new_gauge_md(MetricName::SysCPUNice, "CPU nice time", &[SERVER_LABEL], subsystems::SYSTEM_CPU)); pub static SYS_CPU_STEAL_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPUSteal, "CPU steal time", &[], subsystems::SYSTEM_CPU)); + LazyLock::new(|| new_gauge_md(MetricName::SysCPUSteal, "CPU steal time", &[SERVER_LABEL], subsystems::SYSTEM_CPU)); pub static SYS_CPU_SYSTEM_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPUSystem, "CPU system time", &[], subsystems::SYSTEM_CPU)); + LazyLock::new(|| new_gauge_md(MetricName::SysCPUSystem, "CPU system time", &[SERVER_LABEL], subsystems::SYSTEM_CPU)); pub static SYS_CPU_USER_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::SysCPUUser, "CPU user time", &[], subsystems::SYSTEM_CPU)); + LazyLock::new(|| new_gauge_md(MetricName::SysCPUUser, "CPU user time", &[SERVER_LABEL], subsystems::SYSTEM_CPU)); diff --git a/crates/obs/src/metrics/schema/system_memory.rs b/crates/obs/src/metrics/schema/system_memory.rs index de45de6fd..0ffa9f193 100644 --- a/crates/obs/src/metrics/schema/system_memory.rs +++ b/crates/obs/src/metrics/schema/system_memory.rs @@ -14,43 +14,74 @@ #![allow(dead_code)] +use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; use std::sync::LazyLock; /// Total memory available on the node -pub static MEM_TOTAL_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemTotal, "Total memory on the node", &[], subsystems::SYSTEM_MEMORY)); +pub static MEM_TOTAL_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::MemTotal, + "Total memory on the node", + &[SERVER_LABEL], + subsystems::SYSTEM_MEMORY, + ) +}); /// Memory currently in use on the node pub static MEM_USED_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemUsed, "Used memory on the node", &[], subsystems::SYSTEM_MEMORY)); + LazyLock::new(|| new_gauge_md(MetricName::MemUsed, "Used memory on the node", &[SERVER_LABEL], subsystems::SYSTEM_MEMORY)); /// Percentage of total memory currently in use pub static MEM_USED_PERC_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::MemUsedPerc, "Used memory percentage on the node", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_MEMORY, ) }); /// Memory not currently in use and available for allocation pub static MEM_FREE_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemFree, "Free memory on the node", &[], subsystems::SYSTEM_MEMORY)); + LazyLock::new(|| new_gauge_md(MetricName::MemFree, "Free memory on the node", &[SERVER_LABEL], subsystems::SYSTEM_MEMORY)); /// Memory used for file buffers by the kernel -pub static MEM_BUFFERS_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemBuffers, "Buffers memory on the node", &[], subsystems::SYSTEM_MEMORY)); +pub static MEM_BUFFERS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::MemBuffers, + "Buffers memory on the node", + &[SERVER_LABEL], + subsystems::SYSTEM_MEMORY, + ) +}); /// Memory used for caching file data by the kernel -pub static MEM_CACHE_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemCache, "Cache memory on the node", &[], subsystems::SYSTEM_MEMORY)); +pub static MEM_CACHE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::MemCache, + "Cache memory on the node", + &[SERVER_LABEL], + subsystems::SYSTEM_MEMORY, + ) +}); /// Memory shared between multiple processes -pub static MEM_SHARED_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemShared, "Shared memory on the node", &[], subsystems::SYSTEM_MEMORY)); +pub static MEM_SHARED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::MemShared, + "Shared memory on the node", + &[SERVER_LABEL], + subsystems::SYSTEM_MEMORY, + ) +}); /// Estimate of memory available for new applications without swapping -pub static MEM_AVAILABLE_MD: LazyLock = - LazyLock::new(|| new_gauge_md(MetricName::MemAvailable, "Available memory on the node", &[], subsystems::SYSTEM_MEMORY)); +pub static MEM_AVAILABLE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::MemAvailable, + "Available memory on the node", + &[SERVER_LABEL], + subsystems::SYSTEM_MEMORY, + ) +}); diff --git a/crates/obs/src/metrics/schema/system_network.rs b/crates/obs/src/metrics/schema/system_network.rs index 6ac2bc75d..5571951fd 100644 --- a/crates/obs/src/metrics/schema/system_network.rs +++ b/crates/obs/src/metrics/schema/system_network.rs @@ -14,6 +14,7 @@ #![allow(dead_code)] +use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; @@ -22,7 +23,7 @@ pub static INTERNODE_ERRORS_TOTAL_MD: LazyLock = LazyLock::new new_counter_md( MetricName::InternodeErrorsTotal, "Total number of failed internode calls", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_NETWORK_INTERNODE, ) }); @@ -32,7 +33,7 @@ pub static INTERNODE_DIAL_ERRORS_TOTAL_MD: LazyLock = LazyLock new_counter_md( MetricName::InternodeDialErrorsTotal, "Total number of internode TCP dial timeouts and errors", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_NETWORK_INTERNODE, ) }); @@ -42,7 +43,7 @@ pub static INTERNODE_DIAL_AVG_TIME_NANOS_MD: LazyLock = LazyLo new_gauge_md( MetricName::InternodeDialAvgTimeNanos, "Average dial time of internode TCP calls in nanoseconds", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_NETWORK_INTERNODE, ) }); @@ -52,7 +53,7 @@ pub static INTERNODE_SENT_BYTES_TOTAL_MD: LazyLock = LazyLock: new_counter_md( MetricName::InternodeSentBytesTotal, "Total number of bytes sent to other peer nodes", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_NETWORK_INTERNODE, ) }); @@ -62,7 +63,7 @@ pub static INTERNODE_RECV_BYTES_TOTAL_MD: LazyLock = LazyLock: new_counter_md( MetricName::InternodeRecvBytesTotal, "Total number of bytes received from other peer nodes", - &[], + &[SERVER_LABEL], subsystems::SYSTEM_NETWORK_INTERNODE, ) }); diff --git a/crates/obs/src/metrics/schema/system_network_host.rs b/crates/obs/src/metrics/schema/system_network_host.rs index b7d26572c..45cc89ffe 100644 --- a/crates/obs/src/metrics/schema/system_network_host.rs +++ b/crates/obs/src/metrics/schema/system_network_host.rs @@ -14,6 +14,7 @@ #![allow(dead_code)] +use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_counter_md, subsystems}; use std::sync::LazyLock; @@ -25,7 +26,7 @@ pub static HOST_NETWORK_IO_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::HostNetworkIO, "Network bytes transferred across system network interfaces", - &[DIRECTION_LABEL], + &[SERVER_LABEL, DIRECTION_LABEL], subsystems::SYSTEM_NETWORK_HOST, ) }); @@ -35,7 +36,7 @@ pub static HOST_NETWORK_IO_PER_INTERFACE_MD: LazyLock = LazyLo new_counter_md( MetricName::HostNetworkIOPerInterface, "Network bytes transferred across system network interfaces (per interface)", - &[INTERFACE_LABEL, DIRECTION_LABEL], + &[SERVER_LABEL, INTERFACE_LABEL, DIRECTION_LABEL], subsystems::SYSTEM_NETWORK_HOST, ) }); @@ -48,12 +49,19 @@ mod tests { #[test] fn host_network_descriptors_export_counter_labels() { assert_eq!(HOST_NETWORK_IO_MD.metric_type, MetricType::Counter); - assert_eq!(HOST_NETWORK_IO_MD.variable_labels, vec![DIRECTION_LABEL.to_string()]); + assert_eq!( + HOST_NETWORK_IO_MD.variable_labels, + vec![SERVER_LABEL.to_string(), DIRECTION_LABEL.to_string()] + ); assert_eq!(HOST_NETWORK_IO_PER_INTERFACE_MD.metric_type, MetricType::Counter); assert_eq!( HOST_NETWORK_IO_PER_INTERFACE_MD.variable_labels, - vec![INTERFACE_LABEL.to_string(), DIRECTION_LABEL.to_string()] + vec![ + SERVER_LABEL.to_string(), + INTERFACE_LABEL.to_string(), + DIRECTION_LABEL.to_string() + ] ); } } diff --git a/crates/obs/src/metrics/schema/system_process.rs b/crates/obs/src/metrics/schema/system_process.rs index 34b46157c..ecddbc9c3 100644 --- a/crates/obs/src/metrics/schema/system_process.rs +++ b/crates/obs/src/metrics/schema/system_process.rs @@ -14,15 +14,31 @@ #![allow(dead_code)] +use crate::node_identity::SERVER_LABEL; use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems}; use std::sync::LazyLock; +pub const PROCESS_PID_LABEL: &str = "process_pid"; +pub const PROCESS_EXECUTABLE_NAME_LABEL: &str = "process_executable_name"; +pub const DIRECTION_LABEL: &str = "direction"; +pub const STATUS_LABEL: &str = "status"; + +const PROCESS_LABELS: &[&str] = &[SERVER_LABEL]; +const PROCESS_WITH_ATTRIBUTES_LABELS: &[&str] = &[SERVER_LABEL, PROCESS_PID_LABEL, PROCESS_EXECUTABLE_NAME_LABEL]; +const PROCESS_DISK_IO_LABELS: &[&str] = &[ + DIRECTION_LABEL, + SERVER_LABEL, + PROCESS_PID_LABEL, + PROCESS_EXECUTABLE_NAME_LABEL, +]; +const PROCESS_STATUS_LABELS: &[&str] = &[SERVER_LABEL, STATUS_LABEL]; + /// Number of current READ locks on this peer pub static PROCESS_LOCKS_READ_TOTAL_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ProcessLocksReadTotal, "Number of current READ locks on this peer", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -32,7 +48,7 @@ pub static PROCESS_LOCKS_WRITE_TOTAL_MD: LazyLock = LazyLock:: new_gauge_md( MetricName::ProcessLocksWriteTotal, "Number of current WRITE locks on this peer", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -42,7 +58,7 @@ pub static PROCESS_CPU_TOTAL_SECONDS_MD: LazyLock = LazyLock:: new_counter_md( MetricName::ProcessCPUTotalSeconds, "Total user and system CPU time spent in seconds", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -52,7 +68,7 @@ pub static PROCESS_GO_ROUTINE_TOTAL_MD: LazyLock = LazyLock::n new_gauge_md( MetricName::ProcessGoRoutineTotal, "Total number of go routines running", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -62,7 +78,7 @@ pub static PROCESS_IO_RCHAR_BYTES_MD: LazyLock = LazyLock::new new_counter_md( MetricName::ProcessIORCharBytes, "Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -72,7 +88,7 @@ pub static PROCESS_IO_READ_BYTES_MD: LazyLock = LazyLock::new( new_counter_md( MetricName::ProcessIOReadBytes, "Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -82,7 +98,7 @@ pub static PROCESS_IO_WCHAR_BYTES_MD: LazyLock = LazyLock::new new_counter_md( MetricName::ProcessIOWCharBytes, "Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -92,7 +108,7 @@ pub static PROCESS_IO_WRITE_BYTES_MD: LazyLock = LazyLock::new new_counter_md( MetricName::ProcessIOWriteBytes, "Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -102,7 +118,7 @@ pub static PROCESS_START_TIME_SECONDS_MD: LazyLock = LazyLock: new_gauge_md( MetricName::ProcessStartTimeSeconds, "Start time for RustFS process in seconds since Unix epoch", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -112,7 +128,7 @@ pub static PROCESS_UPTIME_SECONDS_MD: LazyLock = LazyLock::new new_gauge_md( MetricName::ProcessUptimeSeconds, "Uptime for RustFS process in seconds", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -122,7 +138,7 @@ pub static PROCESS_FILE_DESCRIPTOR_LIMIT_TOTAL_MD: LazyLock = new_gauge_md( MetricName::ProcessFileDescriptorLimitTotal, "Limit on total number of open file descriptors for the RustFS Server process", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -132,7 +148,7 @@ pub static PROCESS_FILE_DESCRIPTOR_OPEN_TOTAL_MD: LazyLock = L new_gauge_md( MetricName::ProcessFileDescriptorOpenTotal, "Total number of open file descriptors by the RustFS Server process", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -142,7 +158,7 @@ pub static PROCESS_SYSCALL_READ_TOTAL_MD: LazyLock = LazyLock: new_counter_md( MetricName::ProcessSyscallReadTotal, "Total read SysCalls to the kernel. /proc/[pid]/io syscr", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -152,7 +168,7 @@ pub static PROCESS_SYSCALL_WRITE_TOTAL_MD: LazyLock = LazyLock new_counter_md( MetricName::ProcessSyscallWriteTotal, "Total write SysCalls to the kernel. /proc/[pid]/io syscw", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -162,7 +178,7 @@ pub static PROCESS_RESIDENT_MEMORY_BYTES_MD: LazyLock = LazyLo new_gauge_md( MetricName::ProcessResidentMemoryBytes, "Resident memory size in bytes", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -172,7 +188,7 @@ pub static PROCESS_VIRTUAL_MEMORY_BYTES_MD: LazyLock = LazyLoc new_gauge_md( MetricName::ProcessVirtualMemoryBytes, "Virtual memory size in bytes", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -182,7 +198,7 @@ pub static PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: LazyLock = Laz new_gauge_md( MetricName::ProcessVirtualMemoryMaxBytes, "Maximum virtual memory size in bytes", - &[], + PROCESS_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -196,7 +212,7 @@ pub static PROCESS_CPU_USAGE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ProcessCPUUsage, "The percentage of CPU in use by the process", - &[], + PROCESS_WITH_ATTRIBUTES_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -206,7 +222,7 @@ pub static PROCESS_CPU_UTILIZATION_MD: LazyLock = LazyLock::ne new_gauge_md( MetricName::ProcessCPUUtilization, "The amount of CPU in use by the process (considering multiple cores)", - &[], + PROCESS_WITH_ATTRIBUTES_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -216,7 +232,7 @@ pub static PROCESS_DISK_IO_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ProcessDiskIO, "Disk bytes transferred by the process", - &[], + PROCESS_DISK_IO_LABELS, subsystems::SYSTEM_PROCESS, ) }); @@ -226,7 +242,7 @@ pub static PROCESS_STATUS_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ProcessStatus, "Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)", - &[], + PROCESS_STATUS_LABELS, subsystems::SYSTEM_PROCESS, ) }); diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index 743722921..7f14aa599 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -33,6 +33,7 @@ use crate::metrics::{ obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, }; +use crate::node_identity::current_local_node_identity; use chrono::Utc; use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{ScannerMetricsReport, global_metrics}; @@ -539,12 +540,13 @@ pub async fn collect_disk_stats() -> Vec { disk_stats } -fn build_system_cpu_stats(system: &System) -> CpuStats { +fn build_system_cpu_stats(system: &System, server: &str) -> CpuStats { let cpu_usage = system.global_cpu_usage() as f64; let cpu_count = system.cpus().len().max(1) as f64; let load_avg = System::load_average().one; CpuStats { + server: server.to_string(), avg_idle: (100.0 - cpu_usage).max(0.0), load_avg, load_avg_perc: (load_avg / cpu_count) * 100.0, @@ -552,11 +554,12 @@ fn build_system_cpu_stats(system: &System) -> CpuStats { } } -fn build_system_memory_stats(system: &System) -> MemoryStats { +fn build_system_memory_stats(system: &System, server: &str) -> MemoryStats { let total = system.total_memory(); let used = system.used_memory(); MemoryStats { + server: server.to_string(), total, used, used_perc: if total > 0 { @@ -582,7 +585,8 @@ pub fn collect_system_cpu_and_memory_stats() -> (CpuStats, MemoryStats) { pub fn collect_system_cpu_and_memory_stats_with(system: &mut System) -> (CpuStats, MemoryStats) { system.refresh_cpu_all(); system.refresh_memory(); - (build_system_cpu_stats(system), build_system_memory_stats(system)) + let server = current_local_node_identity(); + (build_system_cpu_stats(system, &server), build_system_memory_stats(system, &server)) } /// Collect system CPU statistics from the current host. @@ -692,6 +696,7 @@ fn process_metric_bundle_from_snapshots( resource_snapshot: ProcessResourceSnapshot, process_snapshot: ProcessSystemSnapshot, ) -> ProcessMetricBundle { + let server = current_local_node_identity(); let status = match process_snapshot.status { ProcessStatusSnapshot::Running => ProcessStatusType::Running, ProcessStatusSnapshot::Sleeping => ProcessStatusType::Sleeping, @@ -700,11 +705,13 @@ fn process_metric_bundle_from_snapshots( }; let resource_stats = ResourceStats { + server: server.clone(), cpu_percent: resource_snapshot.cpu_percent, memory_bytes: resource_snapshot.memory_bytes, uptime_seconds: resource_snapshot.uptime_seconds, }; let process_stats = ProcessStats { + server, locks_read_total: process_snapshot.locks_read_total, locks_write_total: process_snapshot.locks_write_total, cpu_total_seconds: process_snapshot.cpu_total_seconds, @@ -770,6 +777,7 @@ pub fn collect_host_network_stats_with(networks: &Networks) -> HostNetworkStats } HostNetworkStats { + server: current_local_node_identity(), total_received, total_transmitted, per_interface, @@ -794,6 +802,7 @@ pub fn collect_internode_network_stats() -> Option { let snapshot = global_internode_metrics().snapshot(); Some(NetworkStats { + server: current_local_node_identity(), internode_errors_total: snapshot.errors_total, internode_dial_errors_total: snapshot.dial_errors_total, internode_dial_avg_time_nanos: snapshot.dial_avg_time_nanos, @@ -1307,6 +1316,27 @@ mod tests { assert!(cluster_config_stats_from_backend_parities(Some(1), Some(overflow)).is_none()); } + #[tokio::test] + async fn node_local_resource_stats_use_stable_local_node_identity() { + let _guard = crate::node_identity::local_node_identity_test_guard().await; + let previous = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name("node1:9000").await; + + let mut system = System::new_all(); + let (cpu, memory) = collect_system_cpu_and_memory_stats_with(&mut system); + let host_network = collect_host_network_stats_with(&Networks::new()); + let process_bundle = + process_metric_bundle_from_snapshots(ProcessResourceSnapshot::default(), ProcessSystemSnapshot::default()); + + assert_eq!(cpu.server, "node1:9000"); + assert_eq!(memory.server, "node1:9000"); + assert_eq!(host_network.server, "node1:9000"); + assert_eq!(process_bundle.resource.server, "node1:9000"); + assert_eq!(process_bundle.process.server, "node1:9000"); + + rustfs_common::set_global_local_node_name(&previous).await; + } + #[test] fn erasure_set_stats_skip_unknown_backend_layout() { let storage_info = storage_info_with_one_online_disk(); diff --git a/crates/obs/src/node_identity.rs b/crates/obs/src/node_identity.rs new file mode 100644 index 000000000..2128db816 --- /dev/null +++ b/crates/obs/src/node_identity.rs @@ -0,0 +1,34 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) const RUSTFS_NODE_ATTRIBUTE: &str = "rustfs.node"; +pub(crate) const SERVER_LABEL: &str = "server"; + +#[cfg(test)] +static LOCAL_NODE_IDENTITY_TEST_LOCK: std::sync::LazyLock> = + std::sync::LazyLock::new(|| tokio::sync::Mutex::new(())); + +pub(crate) fn local_node_identity(local_ip: &str) -> String { + rustfs_common::try_get_global_local_node_name().unwrap_or_else(|| local_ip.to_string()) +} + +pub(crate) fn current_local_node_identity() -> String { + let local_ip = rustfs_utils::get_local_ip_with_default(); + local_node_identity(&local_ip) +} + +#[cfg(test)] +pub(crate) async fn local_node_identity_test_guard() -> tokio::sync::MutexGuard<'static, ()> { + LOCAL_NODE_IDENTITY_TEST_LOCK.lock().await +} diff --git a/crates/obs/src/telemetry/resource.rs b/crates/obs/src/telemetry/resource.rs index a323047f9..37bda8a5f 100644 --- a/crates/obs/src/telemetry/resource.rs +++ b/crates/obs/src/telemetry/resource.rs @@ -16,15 +16,20 @@ //! //! A `Resource` describes the entity producing telemetry data. The resource //! built here includes the service name, service version, deployment -//! environment, and the local machine IP address so that data can be -//! correlated across services in a distributed system. +//! environment, the stable RustFS node identity, and the local machine IP +//! address so that data can be correlated across services in a distributed +//! system. use crate::config::OtelConfig; +use crate::node_identity::{RUSTFS_NODE_ATTRIBUTE, local_node_identity}; use opentelemetry::KeyValue; use opentelemetry_sdk::Resource; use opentelemetry_semantic_conventions::{ SCHEMA_URL, - attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION}, + attribute::{ + DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_INSTANCE_ID as OTEL_SERVICE_INSTANCE_ID, + SERVICE_VERSION as OTEL_SERVICE_VERSION, + }, }; use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION}; use rustfs_utils::get_local_ip_with_default; @@ -38,12 +43,18 @@ use std::borrow::Cow; /// [`SERVICE_VERSION`]. /// - `deployment.environment` — from `config.environment`, defaulting to /// [`ENVIRONMENT`]. +/// - `rustfs.node` / `service.instance.id` — the stable RustFS local node name +/// when available, falling back to the local IP during early startup. /// - `network.local.address` — the primary local IP of the current host, -/// useful for identifying individual nodes in a cluster. +/// useful as an operational fallback when the stable node name is not yet +/// initialized. /// /// All attributes are attached to the resource using the semantic conventions /// schema URL to ensure compatibility with standard OTLP backends. pub(super) fn build_resource(config: &OtelConfig) -> Resource { + let local_ip = get_local_ip_with_default(); + let node_identity = local_node_identity(&local_ip); + Resource::builder() .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(APP_NAME)).to_string()) .with_schema_url( @@ -56,9 +67,41 @@ pub(super) fn build_resource(config: &OtelConfig) -> Resource { DEPLOYMENT_ENVIRONMENT_NAME, Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(), ), - KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()), + KeyValue::new(RUSTFS_NODE_ATTRIBUTE, node_identity.clone()), + KeyValue::new(OTEL_SERVICE_INSTANCE_ID, node_identity), + KeyValue::new(NETWORK_LOCAL_ADDRESS, local_ip), ], SCHEMA_URL, ) .build() } + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::Key; + + #[tokio::test] + async fn build_resource_uses_stable_local_node_identity() { + let _guard = crate::node_identity::local_node_identity_test_guard().await; + let previous = rustfs_common::get_global_local_node_name().await; + rustfs_common::set_global_local_node_name("node1:9000").await; + + let resource = build_resource(&OtelConfig::default()); + + assert_eq!( + resource + .get(&Key::from_static_str(RUSTFS_NODE_ATTRIBUTE)) + .map(|value| value.to_string()), + Some("node1:9000".to_string()) + ); + assert_eq!( + resource + .get(&Key::from_static_str(OTEL_SERVICE_INSTANCE_ID)) + .map(|value| value.to_string()), + Some("node1:9000".to_string()) + ); + + rustfs_common::set_global_local_node_name(&previous).await; + } +} From 145d38133b1cdefa4365760d763939989034d812 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 30 Jul 2026 14:09:13 +0800 Subject: [PATCH 12/52] fix(ci): generate release notes with the correct baseline (#5467) fix(ci): generate release notes with correct baseline --- .../skills/rustfs-release-publish/SKILL.md | 4 + .github/workflows/audit.yml | 2 + .github/workflows/build.yml | 55 +--- scripts/release/create_or_update_release.sh | 224 +++++++++++++++++ .../check_preview_release_workflow.sh | 235 ++++++++++++++++-- 5 files changed, 458 insertions(+), 62 deletions(-) create mode 100755 scripts/release/create_or_update_release.sh diff --git a/.agents/skills/rustfs-release-publish/SKILL.md b/.agents/skills/rustfs-release-publish/SKILL.md index 9c246398d..9bf8ab297 100644 --- a/.agents/skills/rustfs-release-publish/SKILL.md +++ b/.agents/skills/rustfs-release-publish/SKILL.md @@ -57,6 +57,8 @@ Rules: - Preview Release assets are versioned and intentionally visible on the Releases page. Do not label them Latest or use them to update any latest distribution channel. - Tags have no `v` prefix. Always annotated: `git tag -a -m "Release "`. - The final tag MUST point at exactly `PREVIEW_HASH` — the commit the validated preview tag points at. Never tag current `main` HEAD (commits merged after validation are unvalidated), and never create an extra version-bump commit between preview and final. +- When a previous deliverable exists, GitHub Release notes for the preview and final tags MUST use it as their shared comparison baseline: the most recently published non-preview Release before the target. Internal `-preview.N` Releases are explicitly excluded from that selection, even when they point at the same commit as the final tag. If no previous deliverable exists, omit `previous_tag_name` and record that GitHub's default baseline fallback was used. +- Generated Release notes carry a workflow-management marker so retries can repair them. Before manually curating a generated body, remove that marker; unmarked non-placeholder notes are preserved by later workflow runs. - Phases run in order; a failure in any phase blocks everything after it. After the fix lands on main, restart from Phase 2 with the next preview iteration against the new `origin/main` hash — do not resume mid-pipeline against a stale hash. - If the release is abandoned after Phase 1 merged, main's version files claim a version that was never tagged. Either revert the bump PR or leave it to be overwritten by the next release — but tell the user explicitly and record the decision. - User-facing status updates in Chinese; commits, PR titles/bodies, and tag messages in English. No hard-wrapping in commit messages, PR bodies, or documentation prose — one logical line per sentence/paragraph, let soft wrap handle display. @@ -153,6 +155,7 @@ On a restart (N+1), refresh `PREVIEW_HASH=$(git rev-parse origin/main)` first - Find and watch the tag build: `gh run list --workflow build.yml --branch "" --limit 1` then `gh run watch `. Every build matrix target must succeed (linux x86_64/aarch64 × musl/gnu, macos-aarch64, windows-x86_64). - Confirm the Release publication jobs (`create-release`, `upload-release-assets`, and `publish-release`) succeed while `update-latest-version` is skipped. - Verify `gh release view "" --json isPrerelease,assets,url`: `isPrerelease` must be `true`, and the Release must contain all 6 versioned platform zips, checksums, SBOM, and provenance with no `-latest` assets. Confirm `gh api repos/{owner}/{repo}/releases/latest --jq .tag_name` does not return ``. +- Record `PREVIOUS_DELIVERABLE`, selected from published Releases by `publishedAt` after excluding the current tag and every `-preview.N` tag. Verify `gh release view "" --json body --jq .body` contains `## What's Changed` and, when `PREVIOUS_DELIVERABLE` exists, `**Full Changelog**: https://github.com/rustfs/rustfs/compare/...`. For a repository with no previous deliverable, verify a Full Changelog link exists and record the GitHub baseline fallback. - Confirm preview-triggered Docker and Helm jobs are skipped. Preview validation covers the built RustFS binaries, embedded console, and rc compatibility; Docker image construction and Helm publication are deferred to the final tag because the Dockerfiles consume GitHub Release assets. ## Phase 4 — Run the artifact locally, verify the console @@ -217,6 +220,7 @@ git push origin "" - CI rebuilds from the same source; the only changed input is the tag name, so the binary now self-reports ``. - Verify the final tag's complete publication path: all matrix and release jobs green; `gh release view ""` shows the full versioned and `-latest` asset set plus checksums, SBOM, and provenance; Docker and Helm workflows succeed; `latest.json` points to ``. A stable target must have `isPrerelease=false` and `isLatest=true`. An alpha/beta/rc target must have `isPrerelease=true`; GitHub does not permit prereleases to be Latest, but the project `latest.json` still advances to the final non-preview target. +- Verify the final Release body contains `## What's Changed` and a Full Changelog link. When `PREVIOUS_DELIVERABLE` exists, the link MUST be `https://github.com/rustfs/rustfs/compare/...` and the baseline MUST equal the preview Release baseline; for example, both `1.0.0-beta.12-preview.1` and `1.0.0-beta.12` compare from `1.0.0-beta.11`. - Optionally spot-check `./rustfs --version` from a final-tag artifact — it must report ``. ## Output contract diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index f33240860..a7a5c914f 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -23,6 +23,7 @@ on: - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' + - 'scripts/release/create_or_update_release.sh' - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_workflow_pins.sh' pull_request: @@ -34,6 +35,7 @@ on: - 'deny.toml' - '.github/actions/**' - '.github/workflows/**' + - 'scripts/release/create_or_update_release.sh' - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_workflow_pins.sh' schedule: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2274763d0..cfb6cd337 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -781,6 +781,7 @@ jobs: VERSION="${{ needs.build-check.outputs.version }}" IS_PRERELEASE="${{ needs.build-check.outputs.is_prerelease }}" BUILD_TYPE="${{ needs.build-check.outputs.build_type }}" + TARGET_COMMITISH=$(git rev-parse --verify "refs/tags/${TAG}^{commit}") # Determine release type for title if [[ "$BUILD_TYPE" == "preview" ]]; then @@ -799,49 +800,18 @@ jobs: RELEASE_TYPE="release" fi - # Check if release already exists - if gh release view "$TAG" >/dev/null 2>&1; then - echo "Release $TAG already exists" - RELEASE_ID=$(gh release view "$TAG" --json databaseId --jq '.databaseId') - RELEASE_URL=$(gh release view "$TAG" --json url --jq '.url') + # Create release title + if [[ "$IS_PRERELEASE" == "true" ]]; then + TITLE="RustFS $VERSION (${RELEASE_TYPE})" else - # Get release notes from tag message - RELEASE_NOTES=$(git tag -l --format='%(contents)' "${TAG}") - if [[ -z "$RELEASE_NOTES" || "$RELEASE_NOTES" =~ ^[[:space:]]*$ ]]; then - if [[ "$IS_PRERELEASE" == "true" ]]; then - RELEASE_NOTES="Pre-release ${VERSION} (${RELEASE_TYPE})" - else - RELEASE_NOTES="Release ${VERSION}" - fi - fi - - # Create release title - if [[ "$IS_PRERELEASE" == "true" ]]; then - TITLE="RustFS $VERSION (${RELEASE_TYPE})" - else - TITLE="RustFS $VERSION" - fi - - # Create the release - PRERELEASE_FLAG="" - if [[ "$IS_PRERELEASE" == "true" ]]; then - PRERELEASE_FLAG="--prerelease" - fi - - gh release create "$TAG" \ - --title "$TITLE" \ - --notes "$RELEASE_NOTES" \ - $PRERELEASE_FLAG \ - --latest=false \ - --draft - - RELEASE_ID=$(gh release view "$TAG" --json databaseId --jq '.databaseId') - RELEASE_URL=$(gh release view "$TAG" --json url --jq '.url') + TITLE="RustFS $VERSION" fi - echo "release_id=$RELEASE_ID" >> "$GITHUB_OUTPUT" - echo "release_url=$RELEASE_URL" >> "$GITHUB_OUTPUT" - echo "Created release: $RELEASE_URL" + ./scripts/release/create_or_update_release.sh \ + "$TAG" \ + "$TARGET_COMMITISH" \ + "$TITLE" \ + "$IS_PRERELEASE" # Prepare and upload release assets upload-release-assets: @@ -1002,10 +972,7 @@ jobs: permissions: contents: write steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - - name: Update release notes and publish + - name: Publish release env: GH_TOKEN: ${{ github.token }} shell: bash diff --git a/scripts/release/create_or_update_release.sh b/scripts/release/create_or_update_release.sh new file mode 100755 index 000000000..506ba9953 --- /dev/null +++ b/scripts/release/create_or_update_release.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +set -euo pipefail + +MANAGED_NOTES_MARKER='' + +select_previous_release_tag() { + local current_tag="$1" + local current_release_at="${2:-}" + + jq -r --arg current_tag "$current_tag" --arg current_release_at "$current_release_at" ' + (add // []) + | ($current_tag | sub("-preview\\.[0-9]+$"; "")) as $deliverable_tag + | map( + select( + .draft == false + and .published_at != null + and .tag_name != $current_tag + and .tag_name != $deliverable_tag + and (.tag_name | test("-preview\\.[0-9]+$") | not) + and ($current_release_at == "" or .published_at < $current_release_at) + ) + ) + | sort_by(.published_at) + | last + | .tag_name // empty + ' +} + +release_notes_action() { + local existing_release_json="$1" + local tag="$2" + local body + + if [[ ! -s "$existing_release_json" ]]; then + printf '%s\n' create + return + fi + + body=$(jq -r '.body // ""' "$existing_release_json") + if [[ -z "${body//[[:space:]]/}" ]] || [[ "$body" == "Release $tag" ]] || + [[ "$body" == "Pre-release ${tag} ("*")" ]] || grep -Fq "$MANAGED_NOTES_MARKER" <<<"$body"; then + printf '%s\n' update + else + printf '%s\n' preserve + fi +} + +validate_release_notes() { + local notes_file="$1" + local repository="$2" + local tag="$3" + local previous_tag="$4" + + if ! grep -Fxq "## What's Changed" "$notes_file"; then + echo "release notes are missing the What's Changed section" >&2 + return 1 + fi + + if [[ -n "$previous_tag" ]]; then + local expected_changelog="**Full Changelog**: https://github.com/${repository}/compare/${previous_tag}...${tag}" + if ! grep -Fqx "$expected_changelog" "$notes_file"; then + echo "release notes use an unexpected changelog baseline" >&2 + return 1 + fi + elif ! grep -Fq '**Full Changelog**:' "$notes_file"; then + echo "release notes are missing the Full Changelog link" >&2 + return 1 + fi +} + +write_generated_release_notes() { + local response_json="$1" + local notes_file="$2" + local repository="$3" + local tag="$4" + local previous_tag="$5" + + printf '%s\n' "$MANAGED_NOTES_MARKER" > "$notes_file" + jq -er '.body | strings | select(length > 0)' "$response_json" >> "$notes_file" + validate_release_notes "$notes_file" "$repository" "$tag" "$previous_tag" +} + +write_generate_notes_request() { + local tag="$1" + local target_commitish="$2" + local previous_tag="$3" + local request_json="$4" + + jq -n \ + --arg tag_name "$tag" \ + --arg target_commitish "$target_commitish" \ + --arg previous_tag_name "$previous_tag" \ + '{tag_name: $tag_name, target_commitish: $target_commitish} + + (if $previous_tag_name == "" then {} else {previous_tag_name: $previous_tag_name} end)' \ + > "$request_json" +} + +generate_release_notes() { + local tag="$1" + local target_commitish="$2" + local notes_file="$3" + local work_dir="$4" + local previous_tag="$5" + local request_json="${work_dir}/generate-notes-request.json" + local response_json="${work_dir}/generate-notes-response.json" + + write_generate_notes_request "$tag" "$target_commitish" "$previous_tag" "$request_json" + + if [[ -n "$previous_tag" ]]; then + echo "Generating release notes from previous deliverable $previous_tag" + else + echo "No previous deliverable release found; using GitHub's default baseline" + fi + + gh api --method POST "repos/${GITHUB_REPOSITORY}/releases/generate-notes" \ + --input "$request_json" > "$response_json" + write_generated_release_notes "$response_json" "$notes_file" "$GITHUB_REPOSITORY" "$tag" "$previous_tag" +} + +emit_release_outputs() { + local release_json="$1" + local release_id + local release_url + + release_id=$(jq -er '.databaseId | tostring' "$release_json") + release_url=$(jq -er '.url | strings | select(length > 0)' "$release_json") + if [[ "$release_id" == *$'\n'* || "$release_url" == *$'\n'* ]]; then + echo "release metadata contains an unexpected newline" >&2 + return 1 + fi + + { + printf 'release_id=%s\n' "$release_id" + printf 'release_url=%s\n' "$release_url" + } >> "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}" +} + +create_or_update_release() { + local tag="$1" + local target_commitish="$2" + local title="$3" + local is_prerelease="$4" + local work_dir + local existing_release_json + local release_json + local notes_file + local view_error + local action + local current_release_at + local previous_tag + + if [[ "$is_prerelease" != "true" && "$is_prerelease" != "false" ]]; then + echo "is_prerelease must be true or false" >&2 + return 1 + fi + + work_dir=$(mktemp -d "${RUNNER_TEMP:-/tmp}/rustfs-release-notes.XXXXXX") + existing_release_json="${work_dir}/existing-release.json" + release_json="${work_dir}/release.json" + notes_file="${work_dir}/release-notes.md" + view_error="${work_dir}/release-view-error.txt" + + if ! gh release view "$tag" --json databaseId,url,body,createdAt,publishedAt > "$existing_release_json" 2> "$view_error"; then + if grep -Fqx "release not found" "$view_error"; then + : > "$existing_release_json" + else + echo "Unable to determine whether release $tag exists:" >&2 + sed 's/^/ /' "$view_error" >&2 + rm -rf -- "$work_dir" + return 1 + fi + fi + + action=$(release_notes_action "$existing_release_json" "$tag") + current_release_at="" + if [[ -s "$existing_release_json" ]]; then + current_release_at=$(jq -r '.publishedAt // .createdAt // empty' "$existing_release_json") + fi + gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/releases?per_page=100" > "${work_dir}/releases.json" + previous_tag=$(select_previous_release_tag "$tag" "$current_release_at" < "${work_dir}/releases.json") + if [[ "$action" == "preserve" ]]; then + jq -er '.body | strings | select(length > 0)' "$existing_release_json" > "$notes_file" + if ! validate_release_notes "$notes_file" "$GITHUB_REPOSITORY" "$tag" "$previous_tag"; then + echo "Refusing to overwrite or publish non-managed release notes for $tag" >&2 + rm -rf -- "$work_dir" + return 1 + fi + echo "Release $tag has non-managed notes; preserving them" + emit_release_outputs "$existing_release_json" + rm -rf -- "$work_dir" + return + fi + + generate_release_notes "$tag" "$target_commitish" "$notes_file" "$work_dir" "$previous_tag" + + if [[ "$action" == "update" ]]; then + gh release edit "$tag" --notes-file "$notes_file" >/dev/null + echo "Updated managed release notes for $tag" + else + local create_args=(release create "$tag" --title "$title" --notes-file "$notes_file" --latest=false --draft) + if [[ "$is_prerelease" == "true" ]]; then + create_args+=(--prerelease) + fi + gh "${create_args[@]}" >/dev/null + echo "Created draft release $tag with generated notes" + fi + + gh release view "$tag" --json databaseId,url > "$release_json" + emit_release_outputs "$release_json" + rm -rf -- "$work_dir" +} + +main() { + if [[ "$#" -ne 4 ]]; then + echo "usage: $0 <is-prerelease>" >&2 + exit 2 + fi + + create_or_update_release "$1" "$2" "$3" "$4" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/security/check_preview_release_workflow.sh b/scripts/security/check_preview_release_workflow.sh index 1ad081643..638d68b7f 100755 --- a/scripts/security/check_preview_release_workflow.sh +++ b/scripts/security/check_preview_release_workflow.sh @@ -4,6 +4,10 @@ set -euo pipefail build_workflow=".github/workflows/build.yml" docker_workflow=".github/workflows/docker.yml" helm_workflow=".github/workflows/helm-package.yml" +release_script="scripts/release/create_or_update_release.sh" + +# shellcheck source=scripts/release/create_or_update_release.sh +source "$release_script" require_line() { local file="$1" @@ -16,6 +20,17 @@ require_line() { fi } +require_absent() { + local file="$1" + local text="$2" + local description="$3" + + if grep -Fq -- "$text" "$file"; then + echo "invalid preview release workflow contract: $description" >&2 + exit 1 + fi +} + extract_job_if() { local file="$1" local job="$2" @@ -125,24 +140,12 @@ done latest_guard="startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')" require_job_if "$build_workflow" "update-latest-version" " if: $latest_guard" require_line "$build_workflow" " needs: [ build-check, publish-release ]" "latest update must follow release publication" -require_line "$build_workflow" " --latest=false \\" "new releases must start outside the latest channel" -prerelease_flag_block=$(awk ' - $0 == " PRERELEASE_FLAG=\"\"" { in_block = 1 } - in_block { print } - in_block && $0 == " fi" { exit } -' "$build_workflow") -IFS= read -r -d '' expected_prerelease_flag_block <<'EOF' || true - PRERELEASE_FLAG="" - if [[ "$IS_PRERELEASE" == "true" ]]; then - PRERELEASE_FLAG="--prerelease" - fi -EOF -expected_prerelease_flag_block=${expected_prerelease_flag_block%$'\n'} -if [[ "$prerelease_flag_block" != "$expected_prerelease_flag_block" ]]; then - echo "missing preview release workflow contract: prerelease flag propagation" >&2 - exit 1 -fi -require_line "$build_workflow" " \$PRERELEASE_FLAG \\" "GitHub prerelease creation flag" +require_line "$build_workflow" " TARGET_COMMITISH=\$(git rev-parse --verify \"refs/tags/\${TAG}^{commit}\")" "release target commit resolution" +require_line "$build_workflow" " ./scripts/release/create_or_update_release.sh \\" "managed release creation" +require_absent "$build_workflow" "git tag -l --format='%(contents)'" "annotated tag messages must not become release notes" +require_absent "$build_workflow" "--generate-notes" "GitHub must not choose the release notes baseline implicitly" +require_line "$release_script" " gh api --method POST \"repos/\${GITHUB_REPOSITORY}/releases/generate-notes\" \\" "Generate Release Notes API" +require_line "$release_script" " local create_args=(release create \"\$tag\" --title \"\$title\" --notes-file \"\$notes_file\" --latest=false --draft)" "draft release creation with a notes file" release_channel_block=$(awk ' $0 == " if [[ \"\$BUILD_TYPE\" == \"release\" ]]; then" { in_block = 1 } @@ -168,6 +171,20 @@ if [[ "$release_channel_block" != "$expected_release_channel_block" ]]; then exit 1 fi +publish_release_block=$(awk ' + $0 == " publish-release:" { in_job = 1 } + in_job { print } + in_job && /^ [A-Za-z0-9_-]+:$/ && $0 != " publish-release:" { exit } +' "$build_workflow") +if [[ "$publish_release_block" != *" - name: Publish release"* ]]; then + echo "missing preview release workflow contract: publish job must only publish the prepared release" >&2 + exit 1 +fi +if grep -Eq -- '--notes|generate-notes|create_or_update_release' <<<"$publish_release_block"; then + echo "invalid preview release workflow contract: publish job must not replace release notes" >&2 + exit 1 +fi + IFS= read -r -d '' expected_docker_automatic_guard <<'EOF' || true if: >- github.event_name == 'workflow_dispatch' || @@ -202,4 +219,186 @@ EOF expected_helm_guard=${expected_helm_guard%$'\n'} require_job_if "$helm_workflow" "build-helm-package" "$expected_helm_guard" +assert_equal() { + local expected="$1" + local actual="$2" + local description="$3" + + if [[ "$actual" != "$expected" ]]; then + echo "invalid preview release workflow contract: $description (expected '$expected', got '$actual')" >&2 + exit 1 + fi +} + +IFS= read -r -d '' release_fixture <<'EOF' || true +[ + [ + {"tag_name":"1.0.0-beta.12-preview.1","created_at":"2026-07-30T02:30:00Z","published_at":"2026-07-30T02:36:43Z","draft":false}, + {"tag_name":"1.0.0-beta.12","created_at":"2026-07-30T04:00:00Z","published_at":"2026-07-30T04:13:04Z","draft":false}, + {"tag_name":"1.0.0-beta.12-preview.0","created_at":"2026-07-29T10:00:00Z","published_at":"2026-07-29T10:05:00Z","draft":false} + ], + [ + {"tag_name":"1.0.0-beta.10","created_at":"2026-07-17T05:50:00Z","published_at":"2026-07-17T05:58:19Z","draft":false}, + {"tag_name":"1.0.0-beta.11","created_at":"2026-07-23T04:00:00Z","published_at":"2026-07-23T04:07:31Z","draft":false}, + {"tag_name":"9.0.0","created_at":"2025-01-01T00:00:00Z","published_at":"2025-01-01T00:00:00Z","draft":false}, + {"tag_name":"1.0.0-beta.13","created_at":"2026-08-01T00:00:00Z","published_at":null,"draft":true} + ] +] +EOF + +preview_previous=$(select_previous_release_tag "1.0.0-beta.12-preview.1" "2026-07-30T02:36:43Z" <<<"$release_fixture") +assert_equal "1.0.0-beta.11" "$preview_previous" "preview notes baseline must exclude preview releases and its final deliverable" + +final_previous=$(select_previous_release_tag "1.0.0-beta.12" "2026-07-30T04:13:04Z" <<<"$release_fixture") +assert_equal "1.0.0-beta.11" "$final_previous" "final notes baseline must exclude the same-commit preview release" + +no_previous=$(select_previous_release_tag "1.0.0-beta.1-preview.1" "2026-01-01T00:00:00Z" <<'EOF' +[[{"tag_name":"1.0.0-beta.1-preview.1","created_at":"2026-01-01T00:00:00Z","published_at":"2026-01-01T00:00:00Z","draft":false}]] +EOF +) +assert_equal "" "$no_previous" "repositories without a previous deliverable must use GitHub's fallback baseline" + +delayed_draft_previous=$(select_previous_release_tag "1.0.0-beta.12" "2026-07-30T04:00:00Z" <<'EOF' +[[ + {"tag_name":"1.0.0-beta.12","created_at":"2026-07-30T04:00:00Z","published_at":null,"draft":true}, + {"tag_name":"1.0.0-beta.13","created_at":"2026-08-01T00:00:00Z","published_at":"2026-08-01T00:05:00Z","draft":false}, + {"tag_name":"1.0.0-beta.11","created_at":"2026-07-23T04:00:00Z","published_at":"2026-07-23T04:07:31Z","draft":false} +]] +EOF +) +assert_equal "1.0.0-beta.11" "$delayed_draft_previous" "draft rerun must exclude deliverables published after the draft was created" + +release_test_dir=$(mktemp -d "${TMPDIR:-/tmp}/rustfs-release-workflow-test.XXXXXX") +trap 'rm -rf -- "$release_test_dir"' EXIT +request_json="${release_test_dir}/request.json" +write_generate_notes_request "1.0.0-beta.12" "0123456789abcdef" "1.0.0-beta.11" "$request_json" +assert_equal "1.0.0-beta.11" "$(jq -r .previous_tag_name "$request_json")" "explicit previous_tag_name request field" +write_generate_notes_request "1.0.0-beta.1" "0123456789abcdef" "" "$request_json" +assert_equal "false" "$(jq 'has("previous_tag_name")' "$request_json")" "fallback request must omit previous_tag_name" + +existing_release_json="${release_test_dir}/existing-release.json" +jq -n --arg body "Pre-release 1.0.0-beta.12 (beta)" '{body: $body}' > "$existing_release_json" +assert_equal "update" "$(release_notes_action "$existing_release_json" "1.0.0-beta.12")" "rerun must repair legacy prerelease placeholder notes" +jq -n --arg body "${MANAGED_NOTES_MARKER} +## What's Changed" '{body: $body}' > "$existing_release_json" +assert_equal "update" "$(release_notes_action "$existing_release_json" "1.0.0-beta.12")" "rerun must refresh managed notes" + +generated_body="## What's Changed + +* Fix release notes by @alice in #123 +* First contribution by @bob in #124 + +## New Contributors +* @bob made their first contribution in #124 + +**Full Changelog**: https://github.com/rustfs/rustfs/compare/1.0.0-beta.11...1.0.0-beta.12" +response_json="${release_test_dir}/response.json" +notes_file="${release_test_dir}/notes.md" +jq -n --arg body "$generated_body" '{body: $body}' > "$response_json" +write_generated_release_notes "$response_json" "$notes_file" "rustfs/rustfs" "1.0.0-beta.12" "1.0.0-beta.11" +expected_notes="${MANAGED_NOTES_MARKER} +${generated_body}" +assert_equal "$expected_notes" "$(<"$notes_file")" "generated multiline Markdown must remain intact" + +mock_log="${release_test_dir}/gh.log" +mock_notes="${release_test_dir}/captured-notes.md" +mock_generate_calls="${release_test_dir}/generate-calls" +MOCK_RELEASES_JSON="$release_fixture" +MOCK_GENERATED_BODY="$generated_body" +MOCK_RELEASE_EXISTS=true +MOCK_RELEASE_BODY="Release 1.0.0-beta.12" +MOCK_RELEASE_VIEW_ERROR="" + +gh() { + local input_file="" + local notes_path="" + + if [[ "$1" == "api" && "$*" == *"--paginate --slurp"* ]]; then + printf '%s\n' "$MOCK_RELEASES_JSON" + return + fi + + if [[ "$1" == "api" && "$*" == *"releases/generate-notes"* ]]; then + while [[ "$#" -gt 0 ]]; do + if [[ "$1" == "--input" ]]; then + input_file="$2" + break + fi + shift + done + jq -e '.tag_name == "1.0.0-beta.12" and .target_commitish == "0123456789abcdef" and .previous_tag_name == "1.0.0-beta.11"' "$input_file" >/dev/null + printf '%s\n' called >> "$mock_generate_calls" + jq -n --arg body "$MOCK_GENERATED_BODY" '{body: $body}' + return + fi + + if [[ "$1" == "release" && "$2" == "view" ]]; then + if [[ -n "$MOCK_RELEASE_VIEW_ERROR" ]]; then + printf '%s\n' "$MOCK_RELEASE_VIEW_ERROR" >&2 + return 1 + fi + if [[ "$MOCK_RELEASE_EXISTS" != "true" ]]; then + echo "release not found" >&2 + return 1 + fi + jq -n --arg body "$MOCK_RELEASE_BODY" '{databaseId: 123, url: "https://github.com/rustfs/rustfs/releases/tag/test", body: $body, createdAt: "2026-07-30T04:00:00Z", publishedAt: "2026-07-30T04:13:04Z"}' + return + fi + + if [[ "$1" == "release" && ("$2" == "edit" || "$2" == "create") ]]; then + local action="$2" + shift 2 + while [[ "$#" -gt 0 ]]; do + if [[ "$1" == "--notes-file" ]]; then + notes_path="$2" + break + fi + shift + done + cp "$notes_path" "$mock_notes" + MOCK_RELEASE_BODY=$(<"$mock_notes") + MOCK_RELEASE_EXISTS=true + printf '%s\n' "$action" >> "$mock_log" + return + fi + + echo "unexpected mock gh invocation: $*" >&2 + return 1 +} + +GITHUB_REPOSITORY="rustfs/rustfs" +GITHUB_OUTPUT="${release_test_dir}/github-output" +RUNNER_TEMP="$release_test_dir" +create_or_update_release "1.0.0-beta.12" "0123456789abcdef" "RustFS 1.0.0-beta.12 (beta)" true +assert_equal "edit" "$(<"$mock_log")" "existing placeholder release must be updated on rerun" +assert_equal "$expected_notes" "$(<"$mock_notes")" "rerun must update notes through --notes-file" + +: > "$mock_log" +MOCK_RELEASE_EXISTS=false +MOCK_RELEASE_BODY="" +create_or_update_release "1.0.0-beta.12" "0123456789abcdef" "RustFS 1.0.0-beta.12 (beta)" true +assert_equal "create" "$(<"$mock_log")" "first run must create the draft release" + +: > "$mock_log" +: > "$mock_generate_calls" +MOCK_RELEASE_EXISTS=true +MOCK_RELEASE_BODY="$generated_body" +create_or_update_release "1.0.0-beta.12" "0123456789abcdef" "RustFS 1.0.0-beta.12 (beta)" true +assert_equal "" "$(<"$mock_log")" "manual notes must not trigger a release edit" +assert_equal "" "$(<"$mock_generate_calls")" "manual notes must be preserved without regeneration" + +MOCK_RELEASE_BODY="Manually curated release guidance" +if create_or_update_release "1.0.0-beta.12" "0123456789abcdef" "RustFS 1.0.0-beta.12 (beta)" true; then + echo "invalid preview release workflow contract: incomplete manual notes must block publication" >&2 + exit 1 +fi +assert_equal "" "$(<"$mock_log")" "incomplete manual notes must not be overwritten" + +MOCK_RELEASE_VIEW_ERROR="gh: service unavailable (HTTP 503)" +if create_or_update_release "1.0.0-beta.12" "0123456789abcdef" "RustFS 1.0.0-beta.12 (beta)" true; then + echo "invalid preview release workflow contract: transient release lookup failures must fail closed" >&2 + exit 1 +fi +assert_equal "" "$(<"$mock_log")" "release lookup failures must not attempt create or edit" + echo "Preview release workflow contract ok." From e08847d2b638b5e6abc51e72d9c4ca84931d03a0 Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Thu, 30 Jul 2026 17:51:34 +0800 Subject: [PATCH 13/52] docs(obs): add Grafana server-label dashboard (#5468) Co-authored-by: heihutu <heihutu@gmail.com> --- deploy/observability/README.md | 11 + .../grafana/rustfs-node-observability.json | 885 ++++++++++++++++++ 2 files changed, 896 insertions(+) create mode 100644 deploy/observability/README.md create mode 100644 deploy/observability/grafana/rustfs-node-observability.json diff --git a/deploy/observability/README.md b/deploy/observability/README.md new file mode 100644 index 000000000..d06deb7b3 --- /dev/null +++ b/deploy/observability/README.md @@ -0,0 +1,11 @@ +# RustFS Observability Dashboards + +This directory contains optional dashboards and observability assets for operating RustFS deployments. + +## Grafana + +Import `grafana/rustfs-node-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. + +The dashboard uses the RustFS `server` metric label introduced with the node-local observability updates. `server` represents the RustFS node identity and is preferred for RustFS node comparisons. Prometheus `instance` still identifies the scrape target and remains useful for scrape/debugging views, but dashboards that compare RustFS nodes should group and filter by `server`. + +During a rolling upgrade, older nodes may still emit metrics without the `server` label. Complete the rollout before using this dashboard for node-by-node comparisons. diff --git a/deploy/observability/grafana/rustfs-node-observability.json b/deploy/observability/grafana/rustfs-node-observability.json new file mode 100644 index 000000000..d72afcaf4 --- /dev/null +++ b/deploy/observability/grafana/rustfs-node-observability.json @@ -0,0 +1,885 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server) (rate(rustfs_system_network_internode_sent_bytes_total{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} sent", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server) (rate(rustfs_system_network_internode_recv_bytes_total{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} received", + "range": true, + "refId": "B" + } + ], + "title": "Internode Traffic by Server", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server, operation, backend) (rate(rustfs_system_network_internode_operation_sent_bytes_total{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} {{backend}} {{operation}} sent", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server, operation, backend) (rate(rustfs_system_network_internode_operation_recv_bytes_total{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} {{backend}} {{operation}} received", + "range": true, + "refId": "B" + } + ], + "title": "Internode Operation Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server, direction) (rate(rustfs_system_network_host_network_io{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} {{direction}}", + "range": true, + "refId": "A" + } + ], + "title": "Host Network I/O by Server", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server, interface, direction) (rate(rustfs_system_network_host_network_io_per_interface{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} {{interface}} {{direction}}", + "range": true, + "refId": "A" + } + ], + "title": "Host Network I/O by Interface", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (server) (rustfs_system_cpu_usage_perc{server=~\"$server\"})", + "legendFormat": "{{server}} CPU", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage by Server", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (server) (rustfs_system_memory_used_perc{server=~\"$server\"})", + "legendFormat": "{{server}} memory", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage by Server", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (server) (rustfs_system_process_resident_memory_bytes{server=~\"$server\"})", + "legendFormat": "{{server}} resident", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (server) (rustfs_system_process_virtual_memory_bytes{server=~\"$server\"})", + "legendFormat": "{{server}} virtual", + "range": true, + "refId": "B" + } + ], + "title": "Process Memory by Server", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server) (rate(rustfs_system_network_internode_requests_outgoing_total{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} outgoing", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (server) (rate(rustfs_system_network_internode_requests_incoming_total{server=~\"$server\"}[$__rate_interval]))", + "legendFormat": "{{server}} incoming", + "range": true, + "refId": "B" + } + ], + "title": "Internode Request Rate by Server", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "rustfs", + "observability", + "server-label" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_system_cpu_usage_perc, server)", + "hide": 0, + "includeAll": true, + "label": "RustFS server", + "multi": true, + "name": "server", + "options": [], + "query": "label_values(rustfs_system_cpu_usage_perc, server)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RustFS Node Observability", + "uid": "rustfs-node-observability", + "version": 1, + "weekStart": "" +} From e86d4cb5790f933867d6362154b364d7a36a957f Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Thu, 30 Jul 2026 19:13:37 +0800 Subject: [PATCH 14/52] fix(kms): make local backend persistence crash-durable (#5471) --- crates/kms/src/backends/local.rs | 872 +++++++++++++++++++++++++++++-- 1 file changed, 819 insertions(+), 53 deletions(-) diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 60081963f..064afff16 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -33,6 +33,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::fs; use tracing::{debug, warn}; @@ -74,6 +75,313 @@ const LOCAL_KMS_ARGON2_M_COST_KIB: u32 = 19 * 1024; const LOCAL_KMS_ARGON2_T_COST: u32 = 2; const LOCAL_KMS_ARGON2_P_COST: u32 = 1; +/// Strict matcher for leftover commit temp files (`<prefix>.tmp-<uuid>`). +/// +/// Both temp shapes ever produced by this backend are covered: key temps +/// `<stem>.tmp-<uuid>` (`with_extension` replaced the `.key` suffix) and salt +/// temps `.master-key.salt.tmp-<uuid>` (suffix appended to the full name). +/// Published key files always end in `.key` — even a key literally named +/// `foo.tmp-<uuid>` is stored as `foo.tmp-<uuid>.key` — so the `.key` guard +/// plus the exact hyphenated-UUID check makes it impossible to match an +/// authoritative file. +fn is_orphan_commit_temp_name(file_name: &str) -> bool { + if file_name.ends_with(".key") { + return false; + } + let Some((prefix, suffix)) = file_name.rsplit_once(".tmp-") else { + return false; + }; + !prefix.is_empty() && suffix.len() == 36 && uuid::Uuid::try_parse(suffix).is_ok() +} + +/// Durable single-file commit protocol for the key directory. +/// +/// Key material and metadata are unrecoverable state, so every mutation of the +/// key directory must survive a crash or power loss at any point. All writers +/// share one protocol: exclusively create a temp file in the destination +/// directory, write and fsync the content, publish it atomically (`rename` to +/// replace, `hard_link` to create without clobbering) and fsync the parent +/// directory so the new directory entry itself is durable. Deletion mirrors +/// the tail of the protocol (`remove_file` + parent directory fsync) so a +/// removed key cannot resurface after power loss. +/// +/// This intentionally mirrors ecstore's fsync helpers without depending on the +/// ecstore crate: the KMS backend stays decoupled from storage internals. +mod durable_file { + use std::io::{self, Write}; + use std::path::{Path, PathBuf}; + + /// How the fully written temp file becomes visible under its final name. + pub(super) enum Publish { + /// Atomically replace whatever is at the destination via `rename`. + Replace, + /// Publish via `hard_link`, failing with [`CommitError::AlreadyExists`] + /// when the destination exists so concurrent creates stay linearized. + NoClobber, + } + + #[derive(Debug)] + pub(super) enum CommitError { + AlreadyExists, + Io(io::Error), + /// Test-only simulated crash: the protocol stops after the given step + /// with no cleanup, exactly as a power loss would. + #[cfg(test)] + InjectedCrash(CommitStep), + } + + impl From<io::Error> for CommitError { + fn from(error: io::Error) -> Self { + CommitError::Io(error) + } + } + + impl From<CommitError> for crate::error::KmsError { + fn from(error: CommitError) -> Self { + match error { + // Callers publishing with `NoClobber` are expected to map + // `AlreadyExists` to their own domain error before this. + CommitError::AlreadyExists => crate::error::KmsError::internal_error("durable commit destination already exists"), + CommitError::Io(error) => error.into(), + #[cfg(test)] + CommitError::InjectedCrash(step) => { + crate::error::KmsError::internal_error(format!("injected crash after {step:?}")) + } + } + } + } + + /// Protocol steps in execution order. Tests arm a failpoint after any step + /// to prove that every interrupted prefix recovers to either the complete + /// old state or the complete new state. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum CommitStep { + TempWritten, + FileSynced, + Published, + DirSynced, + } + + pub(super) async fn commit( + temp_path: PathBuf, + final_path: PathBuf, + content: Vec<u8>, + permissions: Option<u32>, + publish: Publish, + ) -> Result<(), CommitError> { + tokio::task::spawn_blocking(move || commit_blocking(&temp_path, &final_path, &content, permissions, &publish)) + .await + .map_err(|join_error| CommitError::Io(io::Error::other(join_error)))? + } + + /// Remove a published file durably: without the parent directory fsync a + /// deleted key could resurface after power loss. + pub(super) async fn remove_durably(path: PathBuf) -> io::Result<()> { + tokio::task::spawn_blocking(move || { + std::fs::remove_file(&path)?; + let parent = path + .parent() + .ok_or_else(|| io::Error::other("path has no parent directory"))?; + fsync_dir(parent) + }) + .await + .map_err(io::Error::other)? + } + + fn commit_blocking( + temp_path: &Path, + final_path: &Path, + content: &[u8], + permissions: Option<u32>, + publish: &Publish, + ) -> Result<(), CommitError> { + let file = open_temp_exclusive(temp_path, permissions)?; + match run_protocol(file, temp_path, final_path, content, permissions, publish) { + // A simulated crash must leave the directory exactly as a real one + // would: no cleanup. + #[cfg(test)] + Err(CommitError::InjectedCrash(step)) => Err(CommitError::InjectedCrash(step)), + Err(error) => { + let _ = std::fs::remove_file(temp_path); + Err(error) + } + Ok(()) => Ok(()), + } + } + + fn open_temp_exclusive(temp_path: &Path, permissions: Option<u32>) -> io::Result<std::fs::File> { + let mut options = std::fs::OpenOptions::new(); + // `create_new` refuses to follow anything already at the temp path, so + // the temp file is always a fresh regular file owned by this process. + options.write(true).create_new(true); + #[cfg(unix)] + if let Some(mode) = permissions { + use std::os::unix::fs::OpenOptionsExt; + options.mode(mode & 0o7777); + } + #[cfg(not(unix))] + let _ = permissions; + options.open(temp_path) + } + + fn run_protocol( + mut file: std::fs::File, + temp_path: &Path, + final_path: &Path, + content: &[u8], + permissions: Option<u32>, + publish: &Publish, + ) -> Result<(), CommitError> { + file.write_all(content)?; + crash_if_armed(final_path, CommitStep::TempWritten)?; + + // The umask can only narrow the creation mode, so apply and verify the + // exact requested permissions before the content becomes durable. + #[cfg(unix)] + if let Some(mode) = permissions { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(mode))?; + let actual = file.metadata()?.permissions().mode() & 0o7777; + if actual != mode & 0o7777 { + return Err(CommitError::Io(io::Error::other(format!( + "temp file permissions {actual:o} do not match requested {mode:o}" + )))); + } + } + #[cfg(not(unix))] + let _ = permissions; + + file.sync_all()?; + #[cfg(test)] + fsync_recorder::record_file(final_path); + crash_if_armed(final_path, CommitStep::FileSynced)?; + drop(file); + + match publish { + Publish::Replace => std::fs::rename(temp_path, final_path)?, + Publish::NoClobber => { + if let Err(error) = std::fs::hard_link(temp_path, final_path) { + if error.kind() == io::ErrorKind::AlreadyExists { + return Err(CommitError::AlreadyExists); + } + return Err(error.into()); + } + } + } + crash_if_armed(final_path, CommitStep::Published)?; + + let parent = final_path + .parent() + .ok_or_else(|| io::Error::other("destination has no parent directory"))?; + fsync_dir(parent)?; + crash_if_armed(final_path, CommitStep::DirSynced)?; + + // The published name is durable at this point; the extra temp link left + // by `hard_link` is only cleanup. A crash here leaves an orphan that + // startup recovery removes. + if matches!(publish, Publish::NoClobber) { + let _ = std::fs::remove_file(temp_path); + } + Ok(()) + } + + /// Fsync a directory so recently created, renamed or removed entries + /// survive power loss. No-op on non-Unix platforms where directories + /// cannot be opened for syncing. + fn fsync_dir(dir: &Path) -> io::Result<()> { + #[cfg(test)] + fsync_recorder::record_dir(dir); + #[cfg(unix)] + { + std::fs::File::open(dir)?.sync_all()?; + } + #[cfg(not(unix))] + let _ = dir; + Ok(()) + } + + fn crash_if_armed(_final_path: &Path, _step: CommitStep) -> Result<(), CommitError> { + #[cfg(test)] + if failpoint::is_armed(_final_path, _step) { + return Err(CommitError::InjectedCrash(_step)); + } + Ok(()) + } + + /// Test-only recorder mirroring ecstore's `fsync_dir_recorder`: durability + /// regressions are invisible to ordinary behavior tests (the data is on + /// disk either way), so tests assert directly on which paths were synced. + /// File syncs are recorded under the commit's destination path because the + /// temp name is randomized. Records are global; tests must match paths + /// under their own unique tempdir to stay robust against parallel tests. + #[cfg(test)] + pub(super) mod fsync_recorder { + use std::path::{Path, PathBuf}; + use std::sync::Mutex; + + static FILE_SYNCS: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new()); + static DIR_SYNCS: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new()); + + pub(super) fn record_file(path: &Path) { + FILE_SYNCS.lock().expect("fsync recorder poisoned").push(path.to_path_buf()); + } + + pub(super) fn record_dir(dir: &Path) { + DIR_SYNCS.lock().expect("fsync recorder poisoned").push(dir.to_path_buf()); + } + + pub(crate) fn file_sync_count(path: &Path) -> usize { + FILE_SYNCS + .lock() + .expect("fsync recorder poisoned") + .iter() + .filter(|recorded| recorded.as_path() == path) + .count() + } + + pub(crate) fn dir_sync_count(dir: &Path) -> usize { + DIR_SYNCS + .lock() + .expect("fsync recorder poisoned") + .iter() + .filter(|recorded| recorded.as_path() == dir) + .count() + } + } + + /// Test-only failpoints simulating a crash after a given commit step. + /// Armed per directory so parallel tests never affect each other. + #[cfg(test)] + pub(super) mod failpoint { + use super::CommitStep; + use std::path::{Path, PathBuf}; + use std::sync::Mutex; + + static ARMED: Mutex<Vec<(PathBuf, CommitStep)>> = Mutex::new(Vec::new()); + + pub(crate) fn arm(dir: &Path, step: CommitStep) { + let mut armed = ARMED.lock().expect("commit failpoint poisoned"); + armed.retain(|(armed_dir, _)| armed_dir != dir); + armed.push((dir.to_path_buf(), step)); + } + + pub(crate) fn disarm(dir: &Path) { + ARMED + .lock() + .expect("commit failpoint poisoned") + .retain(|(armed_dir, _)| armed_dir != dir); + } + + pub(super) fn is_armed(final_path: &Path, step: CommitStep) -> bool { + ARMED + .lock() + .expect("commit failpoint poisoned") + .iter() + .any(|(dir, armed_step)| *armed_step == step && final_path.starts_with(dir)) + } + } +} + /// Local KMS client that stores keys in local files pub struct LocalKmsClient { config: LocalConfig, @@ -83,6 +391,9 @@ pub struct LocalKmsClient { legacy_master_cipher: Option<Aes256Gcm>, /// DEK encryption implementation dek_crypto: AesDekCrypto, + /// Per-key write locks serializing read-modify-write updates within this + /// process (see [`Self::lock_key_for_write`]). + key_write_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -142,6 +453,7 @@ impl LocalKmsClient { master_cipher, legacy_master_cipher, dek_crypto: AesDekCrypto::new(), + key_write_locks: Mutex::new(HashMap::new()), }; client.validate_existing_keys().await?; Ok(client) @@ -183,9 +495,25 @@ impl LocalKmsClient { master_cipher, legacy_master_cipher, dek_crypto: AesDekCrypto::new(), + key_write_locks: Mutex::new(HashMap::new()), }) } + /// Serialize writers of one key within this process. + /// + /// Status updates are read-modify-write cycles over the key file, so two + /// concurrent writers would silently drop one update or interleave a + /// delete with a rewrite. Cross-process writers sharing a key directory + /// remain unsupported. Entries live for the client's lifetime; the table + /// is bounded by the number of distinct key ids this process touches. + async fn lock_key_for_write(&self, key_id: &str) -> tokio::sync::OwnedMutexGuard<()> { + let lock = { + let mut locks = self.key_write_locks.lock().expect("Local KMS key write lock table poisoned"); + Arc::clone(locks.entry(key_id.to_string()).or_default()) + }; + lock.lock_owned().await + } + /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. fn derive_master_key(master_key: &str, salt: &[u8]) -> Result<Key<Aes256Gcm>> { let params = Params::new( @@ -229,23 +557,27 @@ impl LocalKmsClient { }); } + Self::ensure_missing_salt_can_be_generated(config).await?; + let mut salt = [0u8; LOCAL_KMS_MASTER_KEY_SALT_LEN]; rand::rng().fill(&mut salt[..]); let temp_path = config .key_dir .join(format!("{LOCAL_KMS_MASTER_KEY_SALT_FILE}.tmp-{}", uuid::Uuid::new_v4())); - fs::write(&temp_path, salt).await?; - Self::set_file_permissions(&temp_path, config.file_permissions).await?; - match fs::hard_link(&temp_path, &salt_path).await { + match durable_file::commit( + temp_path, + salt_path.clone(), + salt.to_vec(), + config.file_permissions, + durable_file::Publish::NoClobber, + ) + .await + { Ok(()) => { - if let Err(error) = fs::remove_file(&temp_path).await { - warn!(path = ?temp_path, %error, "Failed to remove Local KMS salt temporary file"); - } debug!(path = ?salt_path, "Local KMS master key salt created"); Ok(salt) } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - let _ = fs::remove_file(&temp_path).await; + Err(durable_file::CommitError::AlreadyExists) => { let bytes = fs::read(&salt_path).await?; bytes.try_into().map_err(|_| { KmsError::configuration_error(format!( @@ -255,27 +587,49 @@ impl LocalKmsClient { )) }) } - Err(error) => { - let _ = fs::remove_file(&temp_path).await; - Err(error.into()) + Err(error) => Err(error.into()), + } + } + + /// Refuse to generate a fresh salt when the directory already holds keys + /// explicitly marked `encrypted-master-key`: their KDF output depends on + /// the missing salt, so a replacement salt could never decrypt them. + /// Failing closed with a salt-specific error points the operator at the + /// real problem (restore the salt file or the whole directory) instead of + /// a generic decrypt failure. + /// + /// Files that do not parse are ignored here — startup key validation + /// reports them with their own errors right after. Legacy pre-marker files + /// are also ignored: pre-beta.9 directories legitimately have no salt file + /// yet, and an empty directory must keep initializing as before. + async fn ensure_missing_salt_can_be_generated(config: &LocalConfig) -> Result<()> { + #[derive(Deserialize)] + struct ProtectionProbe { + #[serde(default)] + at_rest_protection: StoredKeyProtection, + } + + let mut entries = fs::read_dir(&config.key_dir).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if path.extension().is_none_or(|extension| extension != "key") { + continue; + } + let Ok(content) = fs::read(&path).await else { + continue; + }; + let Ok(probe) = serde_json::from_slice::<ProtectionProbe>(&content) else { + continue; + }; + if probe.at_rest_protection == StoredKeyProtection::EncryptedMasterKey { + return Err(KmsError::configuration_error(format!( + "Local KMS master key salt at {} is missing but {} is marked encrypted-master-key; \ + restore the salt file from backup instead of generating a new one", + Self::master_key_salt_path(config).display(), + path.display() + ))); } } - } - - #[cfg(unix)] - async fn set_file_permissions(path: &std::path::Path, permissions: Option<u32>) -> Result<()> { - if let Some(mode) = permissions { - use std::os::unix::fs::PermissionsExt; - - let perms = std::fs::Permissions::from_mode(mode); - fs::set_permissions(path, perms).await?; - } - - Ok(()) - } - - #[cfg(not(unix))] - async fn set_file_permissions(_path: &std::path::Path, _permissions: Option<u32>) -> Result<()> { Ok(()) } @@ -392,14 +746,19 @@ impl LocalKmsClient { }) } - /// Save a master key to disk + /// Save a master key to disk, durably replacing any existing file async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> { let key_path = self.master_key_path(&master_key.key_id)?; let content = self.encode_master_key(master_key, key_material)?; let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); - fs::write(&temp_path, &content).await?; - Self::set_file_permissions(&temp_path, self.config.file_permissions).await?; - fs::rename(&temp_path, &key_path).await?; + durable_file::commit( + temp_path, + key_path.clone(), + content, + self.config.file_permissions, + durable_file::Publish::Replace, + ) + .await?; debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved"); Ok(()) @@ -409,25 +768,21 @@ impl LocalKmsClient { let key_path = self.master_key_path(&master_key.key_id)?; let content = self.encode_master_key(master_key, key_material)?; let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); - fs::write(&temp_path, &content).await?; - Self::set_file_permissions(&temp_path, self.config.file_permissions).await?; - - match fs::hard_link(&temp_path, &key_path).await { + match durable_file::commit( + temp_path, + key_path.clone(), + content, + self.config.file_permissions, + durable_file::Publish::NoClobber, + ) + .await + { Ok(()) => { - if let Err(error) = fs::remove_file(&temp_path).await { - warn!(path = ?temp_path, %error, "Failed to remove Local KMS key temporary file"); - } debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key created"); Ok(()) } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { - let _ = fs::remove_file(&temp_path).await; - Err(KmsError::key_already_exists(&master_key.key_id)) - } - Err(error) => { - let _ = fs::remove_file(&temp_path).await; - Err(error.into()) - } + Err(durable_file::CommitError::AlreadyExists) => Err(KmsError::key_already_exists(&master_key.key_id)), + Err(error) => Err(error.into()), } } @@ -489,19 +844,45 @@ impl LocalKmsClient { Ok(Zeroizing::new(key_material)) } + /// Startup recovery and validation for the key directory. + /// + /// Leftover commit temp files are removed first: publishing is atomic + /// (`rename`/`hard_link`), so a strictly matching temp name can only be an + /// unpublished remnant of an interrupted commit, never the authoritative + /// copy. Every published `.key` file must then decode. async fn validate_existing_keys(&self) -> Result<()> { + let mut key_ids = Vec::new(); + let mut orphan_temps = Vec::new(); let mut entries = fs::read_dir(&self.config.key_dir).await?; while let Some(entry) = entries.next_entry().await? { let path = entry.path(); - if path.extension().is_none_or(|extension| extension != "key") { + if path.extension().is_some_and(|extension| extension == "key") { + let key_id = path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?; + key_ids.push(key_id.to_string()); continue; } + if entry.file_type().await?.is_file() + && let Some(file_name) = path.file_name().and_then(|name| name.to_str()) + && is_orphan_commit_temp_name(file_name) + { + orphan_temps.push(path); + } + } - let key_id = path - .file_stem() - .and_then(|stem| stem.to_str()) - .ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))?; - self.decode_stored_key(key_id).await?; + for temp_path in orphan_temps { + // Best effort: a temp file that cannot be removed is inert, so + // startup proceeds and retries on the next initialization. + match durable_file::remove_durably(temp_path.clone()).await { + Ok(()) => warn!(path = ?temp_path, "Removed orphaned Local KMS commit temp file"), + Err(error) => warn!(path = ?temp_path, %error, "Failed to remove orphaned Local KMS commit temp file"), + } + } + + for key_id in key_ids { + self.decode_stored_key(&key_id).await?; } Ok(()) } @@ -698,6 +1079,7 @@ impl KmsClient for LocalKmsClient { async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); + let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; master_key.status = KeyStatus::Active; @@ -714,6 +1096,7 @@ impl KmsClient for LocalKmsClient { async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); + let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; master_key.status = KeyStatus::Disabled; @@ -734,6 +1117,7 @@ impl KmsClient for LocalKmsClient { ) -> Result<()> { debug!("Scheduling deletion for key: {}", key_id); + let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; master_key.status = KeyStatus::PendingDeletion; @@ -750,6 +1134,7 @@ impl KmsClient for LocalKmsClient { async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Canceling deletion for key: {}", key_id); + let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; master_key.status = KeyStatus::Active; @@ -945,6 +1330,10 @@ impl KmsBackend for LocalKmsBackend { // unless a pending window is specified let key_id = &request.key_id; + // Deletion is a read-modify-write (or read-then-remove) cycle, so hold + // the per-key write lock across it. + let _write_guard = self.client.lock_key_for_write(key_id).await; + // First, load the key from disk to get the master key let mut master_key = self .client @@ -955,7 +1344,7 @@ impl KmsBackend for LocalKmsBackend { let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) { // For immediate deletion, actually delete the key from filesystem let key_path = self.client.master_key_path(key_id)?; - tokio::fs::remove_file(&key_path) + durable_file::remove_durably(key_path) .await .map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?; @@ -1025,6 +1414,9 @@ impl KmsBackend for LocalKmsBackend { async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> { let key_id = &request.key_id; + // Cancelling is a read-modify-write cycle, so hold the per-key write lock. + let _write_guard = self.client.lock_key_for_write(key_id).await; + // Load the key from disk to get the master key let mut master_key = self .client @@ -1652,4 +2044,378 @@ mod tests { assert!(matches!(error, KmsError::KeyAlreadyExists { .. })); assert_eq!(first.client.get_key_material("concurrent-key").await.expect("load key").len(), 32); } + + fn test_config(dir: &std::path::Path) -> LocalConfig { + LocalConfig { + key_dir: dir.to_path_buf(), + master_key: Some("test-master-key".to_string()), + file_permissions: Some(0o600), + } + } + + async fn sorted_dir_file_names(dir: &std::path::Path) -> Vec<String> { + let mut names = Vec::new(); + let mut entries = fs::read_dir(dir).await.expect("read key directory"); + while let Some(entry) = entries.next_entry().await.expect("read directory entry") { + names.push(entry.file_name().to_str().expect("UTF-8 file name").to_string()); + } + names.sort(); + names + } + + const ALL_COMMIT_STEPS: [durable_file::CommitStep; 4] = [ + durable_file::CommitStep::TempWritten, + durable_file::CommitStep::FileSynced, + durable_file::CommitStep::Published, + durable_file::CommitStep::DirSynced, + ]; + + #[test] + fn orphan_commit_temp_matcher_is_strict() { + let uuid = uuid::Uuid::new_v4(); + // The two shapes the backend actually produces. + assert!(is_orphan_commit_temp_name(&format!("mykey.tmp-{uuid}"))); + assert!(is_orphan_commit_temp_name(&format!(".master-key.salt.tmp-{uuid}"))); + // Authoritative files must never match, even with temp-looking names. + assert!(!is_orphan_commit_temp_name("mykey.key")); + assert!(!is_orphan_commit_temp_name(&format!("decoy.tmp-{uuid}.key"))); + assert!(!is_orphan_commit_temp_name(".master-key.salt")); + // Near misses stay untouched. + assert!(!is_orphan_commit_temp_name("mykey.tmp-not-a-uuid")); + assert!(!is_orphan_commit_temp_name(&format!("mykey.tmp-{}", uuid.simple()))); + assert!(!is_orphan_commit_temp_name(&format!(".tmp-{uuid}"))); + assert!(!is_orphan_commit_temp_name("mykey.tmp-")); + } + + #[tokio::test] + async fn durable_commit_fsyncs_every_write_path() { + use durable_file::fsync_recorder; + + let (client, temp_dir) = create_test_client().await; + let dir = temp_dir.path(); + + // Salt creation during construction is itself a durable commit. + let salt_path = LocalKmsClient::master_key_salt_path(&client.config); + assert!(fsync_recorder::file_sync_count(&salt_path) >= 1, "salt file must be fsynced"); + assert!(fsync_recorder::dir_sync_count(dir) >= 1, "salt publish must fsync the key directory"); + + let key_path = client.master_key_path("durable-key").expect("valid key id"); + let files_before = fsync_recorder::file_sync_count(&key_path); + let dirs_before = fsync_recorder::dir_sync_count(dir); + client.create_key("durable-key", "AES_256", None).await.expect("create key"); + assert!( + fsync_recorder::file_sync_count(&key_path) > files_before, + "create must fsync the key file" + ); + assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "create must fsync the key directory"); + + let files_before = fsync_recorder::file_sync_count(&key_path); + let dirs_before = fsync_recorder::dir_sync_count(dir); + client.disable_key("durable-key", None).await.expect("disable key"); + assert!( + fsync_recorder::file_sync_count(&key_path) > files_before, + "update must fsync the key file" + ); + assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "update must fsync the key directory"); + + let backend = LocalKmsBackend { client }; + let dirs_before = fsync_recorder::dir_sync_count(dir); + backend + .delete_key(DeleteKeyRequest { + key_id: "durable-key".to_string(), + pending_window_in_days: None, + force_immediate: Some(true), + }) + .await + .expect("delete key"); + assert!(!key_path.exists(), "immediate delete must remove the key file"); + assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory"); + } + + #[tokio::test] + async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() { + use durable_file::{CommitStep, failpoint}; + + for step in ALL_COMMIT_STEPS { + let (client, temp_dir) = create_test_client().await; + let key_id = "crash-update-key"; + client.create_key(key_id, "AES_256", None).await.expect("create key"); + let original_material = client.get_key_material(key_id).await.expect("original material"); + + failpoint::arm(temp_dir.path(), step); + let error = client + .disable_key(key_id, None) + .await + .expect_err("armed commit must simulate a crash"); + failpoint::disarm(temp_dir.path()); + assert!(error.to_string().contains("injected crash"), "unexpected error: {error}"); + drop(client); + + // Restart on the same directory: recovery must observe either the + // complete old state or the complete new state, with temps cleaned. + let recovered = LocalKmsClient::new(test_config(temp_dir.path())) + .await + .expect("recovery after an interrupted update must succeed"); + let status = recovered + .describe_key(key_id, None) + .await + .expect("key must survive an interrupted update") + .status; + let expected = if matches!(step, CommitStep::TempWritten | CommitStep::FileSynced) { + // Crash before publish: the old state is authoritative. + KeyStatus::Active + } else { + // Crash after publish: the new state is authoritative. + KeyStatus::Disabled + }; + assert_eq!(status, expected, "step {step:?} must recover to a complete state"); + assert_eq!( + recovered.get_key_material(key_id).await.expect("material must survive"), + original_material, + "step {step:?} must preserve key material" + ); + assert_eq!( + sorted_dir_file_names(temp_dir.path()).await, + vec![".master-key.salt".to_string(), format!("{key_id}.key")], + "step {step:?} must leave no commit temps behind" + ); + } + } + + #[tokio::test] + async fn interrupted_create_commit_recovers_to_absent_or_complete_key() { + use durable_file::{CommitStep, failpoint}; + + for step in ALL_COMMIT_STEPS { + let (client, temp_dir) = create_test_client().await; + let key_id = "crash-create-key"; + + failpoint::arm(temp_dir.path(), step); + client + .create_key(key_id, "AES_256", None) + .await + .expect_err("armed commit must simulate a crash"); + failpoint::disarm(temp_dir.path()); + drop(client); + + let recovered = LocalKmsClient::new(test_config(temp_dir.path())) + .await + .expect("recovery after an interrupted create must succeed"); + if matches!(step, CommitStep::TempWritten | CommitStep::FileSynced) { + // Crash before publish: the key was never created. + let error = recovered + .describe_key(key_id, None) + .await + .expect_err("unpublished key must not exist after recovery"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "step {step:?}: {error:?}"); + assert_eq!( + sorted_dir_file_names(temp_dir.path()).await, + vec![".master-key.salt".to_string()], + "step {step:?} must remove the unpublished temp" + ); + } else { + // Crash after publish: the key is complete and usable. + let material = recovered + .get_key_material(key_id) + .await + .expect("published key must survive recovery"); + assert_eq!(material.len(), 32, "step {step:?} must keep complete key material"); + assert_eq!( + sorted_dir_file_names(temp_dir.path()).await, + vec![".master-key.salt".to_string(), format!("{key_id}.key")], + "step {step:?} must leave only the published key" + ); + } + } + } + + #[tokio::test] + async fn interrupted_salt_commit_recovers_cleanly() { + use durable_file::{CommitStep, failpoint}; + + for step in ALL_COMMIT_STEPS { + let temp_dir = TempDir::new().expect("create temp dir"); + let config = test_config(temp_dir.path()); + let salt_path = LocalKmsClient::master_key_salt_path(&config); + + failpoint::arm(temp_dir.path(), step); + let error = match LocalKmsClient::new(config.clone()).await { + Ok(_) => panic!("armed salt commit must fail initialization"), + Err(error) => error, + }; + failpoint::disarm(temp_dir.path()); + assert!(error.to_string().contains("injected crash"), "unexpected error: {error}"); + + // If the crash hit after publish, the salt is durable and must be + // reused on restart; before publish, a fresh one may be generated. + let published_salt = if matches!(step, CommitStep::Published | CommitStep::DirSynced) { + Some(fs::read(&salt_path).await.expect("published salt must exist")) + } else { + assert!(!salt_path.exists(), "step {step:?} must not publish a salt"); + None + }; + + let client = LocalKmsClient::new(config).await.expect("recovery must succeed"); + let salt_now = fs::read(&salt_path).await.expect("salt must exist after recovery"); + if let Some(published) = published_salt { + assert_eq!(salt_now, published, "step {step:?}: a published salt must be reused"); + } + assert_eq!( + sorted_dir_file_names(temp_dir.path()).await, + vec![".master-key.salt".to_string()], + "step {step:?} must leave exactly one salt file" + ); + client + .create_key("post-recovery-key", "AES_256", None) + .await + .expect("create key"); + } + } + + #[tokio::test] + async fn startup_removes_only_strictly_matching_commit_temps() { + let (client, temp_dir) = create_test_client().await; + client.create_key("real-key", "AES_256", None).await.expect("create key"); + // A key whose name itself looks like a temp is stored with `.key` and + // must survive cleanup. + let decoy_id = format!("decoy.tmp-{}", uuid::Uuid::new_v4()); + client.create_key(&decoy_id, "AES_256", None).await.expect("create decoy key"); + drop(client); + + let key_temp = temp_dir.path().join(format!("real-key.tmp-{}", uuid::Uuid::new_v4())); + let salt_temp = temp_dir.path().join(format!(".master-key.salt.tmp-{}", uuid::Uuid::new_v4())); + let not_a_uuid = temp_dir.path().join("real-key.tmp-not-a-uuid"); + let stray = temp_dir.path().join("operator-notes.txt"); + for path in [&key_temp, &salt_temp, ¬_a_uuid, &stray] { + fs::write(path, b"leftover").await.expect("seed leftover file"); + } + + let client = LocalKmsClient::new(test_config(temp_dir.path())) + .await + .expect("restart with leftover temps must succeed"); + + assert!(!key_temp.exists(), "key commit temp must be removed"); + assert!(!salt_temp.exists(), "salt commit temp must be removed"); + assert!(not_a_uuid.exists(), "non-UUID suffixes must not match the temp pattern"); + assert!(stray.exists(), "unrelated files must be left alone"); + client + .describe_key("real-key", None) + .await + .expect("real key must survive cleanup"); + client + .describe_key(&decoy_id, None) + .await + .expect("temp-looking key name must survive cleanup"); + } + + #[tokio::test] + async fn missing_salt_with_encrypted_keys_fails_closed_without_generating_a_salt() { + let (client, temp_dir) = create_test_client().await; + client + .create_key("sealed-key", "AES_256", None) + .await + .expect("create encrypted key"); + let config = client.config.clone(); + drop(client); + + let salt_path = LocalKmsClient::master_key_salt_path(&config); + fs::remove_file(&salt_path).await.expect("remove salt file"); + + let error = match LocalKmsClient::new(config).await { + Ok(_) => panic!("missing salt with encrypted keys must fail initialization"), + Err(error) => error, + }; + assert!( + matches!(error, KmsError::ConfigurationError { .. }), + "expected a salt-specific configuration error, got {error:?}" + ); + assert!(error.to_string().contains("salt"), "error must point at the missing salt: {error}"); + assert!(!salt_path.exists(), "a replacement salt must never be generated"); + assert_eq!( + sorted_dir_file_names(temp_dir.path()).await, + vec!["sealed-key.key".to_string()], + "the failed startup must not modify the key directory" + ); + } + + #[tokio::test] + async fn missing_salt_with_only_plaintext_dev_keys_still_initializes() { + let (dev_client, temp_dir) = create_dev_mode_client().await; + dev_client + .create_key("plain-key", "AES_256", None) + .await + .expect("create plaintext-dev-only key"); + drop(dev_client); + + // Enabling a master key over a directory of plaintext-dev-only keys is + // a legitimate first-time salt creation, not a lost salt. + let client = LocalKmsClient::new(test_config(temp_dir.path())) + .await + .expect("salt creation must proceed for plaintext-dev-only directories"); + assert!(LocalKmsClient::master_key_salt_path(&client.config).exists()); + } + + #[tokio::test] + async fn per_key_write_lock_blocks_concurrent_status_updates() { + let (client, _temp_dir) = create_test_client().await; + let client = Arc::new(client); + client.create_key("locked-key", "AES_256", None).await.expect("create key"); + let key_path = client.master_key_path("locked-key").expect("valid key id"); + + let guard = client.lock_key_for_write("locked-key").await; + let contender = { + let client = Arc::clone(&client); + tokio::spawn(async move { client.disable_key("locked-key", None).await }) + }; + // Drive the runtime through enough polls and blocking-pool round trips + // that the contender would have finished if it did not honor the lock. + for _ in 0..8 { + tokio::task::yield_now().await; + let _ = fs::metadata(&key_path).await; + } + let status = client.describe_key("locked-key", None).await.expect("describe key").status; + assert_eq!( + status, + KeyStatus::Active, + "a status update must not proceed while the per-key write lock is held" + ); + + drop(guard); + contender + .await + .expect("join contender") + .expect("disable must succeed once the lock is released"); + let status = client.describe_key("locked-key", None).await.expect("describe key").status; + assert_eq!(status, KeyStatus::Disabled); + } + + #[tokio::test] + async fn concurrent_status_updates_preserve_material_and_a_complete_state() { + let (client, _temp_dir) = create_test_client().await; + let key_id = "contended-key"; + client.create_key(key_id, "AES_256", None).await.expect("create key"); + let original_material = client.get_key_material(key_id).await.expect("original material"); + + let (disable, schedule, enable) = tokio::join!( + client.disable_key(key_id, None), + client.schedule_key_deletion(key_id, 7, None), + client.enable_key(key_id, None), + ); + disable.expect("disable"); + schedule.expect("schedule deletion"); + enable.expect("enable"); + + // Whatever the serialization order, the file must be one writer's + // complete output with the original material intact. + let info = client.describe_key(key_id, None).await.expect("key file must stay decodable"); + assert!(matches!( + info.status, + KeyStatus::Active | KeyStatus::Disabled | KeyStatus::PendingDeletion + )); + assert_eq!( + client.get_key_material(key_id).await.expect("material must stay readable"), + original_material, + "concurrent status updates must never lose or regenerate key material" + ); + } } From 6e5f330ff56f6e31976ee28f065aaea8711ddb25 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Thu, 30 Jul 2026 19:20:17 +0800 Subject: [PATCH 15/52] feat(kms): add operation timeout and typed retry policy engine (#5472) --- Cargo.lock | 3 + Cargo.toml | 1 + crates/kms/Cargo.toml | 6 + crates/kms/src/backends/vault.rs | 19 +- crates/kms/src/backends/vault_transit.rs | 20 +- crates/kms/src/config.rs | 71 ++- crates/kms/src/error.rs | 18 + crates/kms/src/lib.rs | 4 + crates/kms/src/policy.rs | 605 +++++++++++++++++++++++ 9 files changed, 735 insertions(+), 12 deletions(-) create mode 100644 crates/kms/src/policy.rs diff --git a/Cargo.lock b/Cargo.lock index a2c01b043..0acca68ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9441,6 +9441,7 @@ name = "rustfs-kms" version = "1.0.0-beta.12" dependencies = [ "aes-gcm", + "anyhow", "arc-swap", "argon2", "async-trait", @@ -9455,6 +9456,7 @@ dependencies = [ "reqwest", "rustfs-security-governance", "rustfs-utils", + "rustify", "serde", "serde_json", "sha2 0.11.0", @@ -9463,6 +9465,7 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tokio", + "tokio-util", "tracing", "url", "uuid", diff --git a/Cargo.toml b/Cargo.toml index d8512078f..34a4b6327 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -285,6 +285,7 @@ reed-solomon-simd = "3.1.0" regex = { version = "1.13.1" } rumqttc = { package = "rumqttc-next", version = "0.33.3" } redis = { version = "1.5.0" } +rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 370f1e90d..e492f0e83 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -65,11 +65,17 @@ rustfs-security-governance = { workspace = true } # HTTP client for Vault reqwest = { workspace = true } vaultrs = { workspace = true } +# vaultrs surfaces transport-level failures as wrapped rustify errors; the +# operation policy needs the concrete type to classify them for retry decisions. +rustify = { workspace = true } +tokio-util = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } +tokio = { workspace = true, features = ["test-util"] } [features] default = [] diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 0ba44b23b..b9f8e6364 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -68,10 +68,17 @@ struct VaultKeyData { impl VaultKmsClient { /// Create a new Vault KMS client - pub async fn new(config: VaultConfig) -> Result<Self> { + /// + /// `attempt_timeout` caps every HTTP request issued through this client. + pub async fn new(config: VaultConfig, attempt_timeout: Duration) -> Result<Self> { // Create client settings let mut settings_builder = VaultClientSettingsBuilder::default(); settings_builder.address(&config.address); + // Defense in depth against stalled connections: vaultrs leaves the + // underlying reqwest client without any timeout by default, so a hung + // request would otherwise wait forever regardless of the + // operation-level retry policy. + settings_builder.timeout(Some(attempt_timeout)); // Set authentication token based on method let token = match &config.auth_method { @@ -607,7 +614,7 @@ impl VaultKmsBackend { } }; - let client = VaultKmsClient::new(vault_config).await?; + let client = VaultKmsClient::new(vault_config, config.effective_timeout()).await?; Ok(Self { client }) } @@ -851,7 +858,9 @@ mod tests { tls: None, }; - let client = VaultKmsClient::new(config).await.expect("Failed to create Vault client"); + let client = VaultKmsClient::new(config, Duration::from_secs(30)) + .await + .expect("Failed to create Vault client"); // Test key operations let key_id = "test-key-vault"; @@ -906,7 +915,9 @@ mod tests { // Regression: get_key_material previously "self-healed" a decrypt/length failure by // minting a fresh random master key and overwriting the stored value — destroying the // original key and making every DEK wrapped by it permanently undecryptable. - let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"); let key_id = format!("corrupt-{}", uuid::Uuid::new_v4()); client.create_key(&key_id, "AES_256", None).await.expect("create"); diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 7be6487ba..9b95f826f 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -138,9 +138,17 @@ pub struct VaultTransitKmsClient { } impl VaultTransitKmsClient { - pub async fn new(config: VaultTransitConfig) -> Result<Self> { + /// Create a new Vault Transit KMS client + /// + /// `attempt_timeout` caps every HTTP request issued through this client. + pub async fn new(config: VaultTransitConfig, attempt_timeout: Duration) -> Result<Self> { let mut settings_builder = VaultClientSettingsBuilder::default(); settings_builder.address(&config.address); + // Defense in depth against stalled connections: vaultrs leaves the + // underlying reqwest client without any timeout by default, so a hung + // request would otherwise wait forever regardless of the + // operation-level retry policy. + settings_builder.timeout(Some(attempt_timeout)); let token = match &config.auth_method { crate::config::VaultAuthMethod::Token { token } => token.clone(), @@ -607,7 +615,7 @@ impl VaultTransitKmsBackend { } }; - let client = VaultTransitKmsClient::new(vault_config).await?; + let client = VaultTransitKmsClient::new(vault_config, config.effective_timeout()).await?; Ok(Self { client }) } } @@ -788,7 +796,7 @@ mod tests { let config = test_vault_transit_config(); // --- First "process": create a key and disable it --- - let client1 = VaultTransitKmsClient::new(config.clone()) + let client1 = VaultTransitKmsClient::new(config.clone(), Duration::from_secs(30)) .await .expect("Failed to create VaultTransit client"); @@ -811,7 +819,7 @@ mod tests { assert_eq!(info_after_disable.status, KeyStatus::Disabled, "key must be Disabled after disable_key"); // --- Simulate restart: create a brand new client with empty cache --- - let client2 = VaultTransitKmsClient::new(config) + let client2 = VaultTransitKmsClient::new(config, Duration::from_secs(30)) .await .expect("Failed to create second VaultTransit client (restart simulation)"); @@ -841,7 +849,7 @@ mod tests { async fn test_transit_pending_deletion_survives_restart_simulation() { let config = test_vault_transit_config(); - let client1 = VaultTransitKmsClient::new(config.clone()) + let client1 = VaultTransitKmsClient::new(config.clone(), Duration::from_secs(30)) .await .expect("Failed to create VaultTransit client"); @@ -865,7 +873,7 @@ mod tests { "key must be PendingDeletion after schedule_key_deletion" ); - let client2 = VaultTransitKmsClient::new(config) + let client2 = VaultTransitKmsClient::new(config, Duration::from_secs(30)) .await .expect("Failed to create second VaultTransit client (restart simulation)"); diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 3d2a469a5..04aab91d6 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -32,6 +32,15 @@ pub const ENV_KMS_STATIC_SECRET_KEY_FILE: &str = "RUSTFS_KMS_STATIC_SECRET_KEY_F pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata"; +/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior. +/// +/// Out-of-range values are clamped at use rather than rejected so existing +/// deployments with oversized settings keep starting after an upgrade. +pub(crate) const MAX_OPERATION_TIMEOUT: Duration = Duration::from_secs(300); + +/// Upper bound applied to `KmsConfig::retry_attempts` when deriving backend behavior. +pub(crate) const MAX_RETRY_ATTEMPTS: u32 = 10; + fn default_vault_transit_metadata_kv_mount() -> String { DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT.to_string() } @@ -117,9 +126,15 @@ pub struct KmsConfig { /// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault. #[serde(default)] pub allow_insecure_dev_defaults: bool, - /// Operation timeout + /// Timeout for a single backend attempt. + /// + /// This bounds one outbound request, not the whole operation: the operation + /// policy owns the total deadline across retries. Values above 300 seconds + /// are clamped at use (see `KmsConfig::effective_timeout`). pub timeout: Duration, - /// Number of retry attempts + /// Number of retry attempts. + /// + /// Values above 10 are clamped at use (see `KmsConfig::effective_retry_attempts`). pub retry_attempts: u32, /// Enable caching pub enable_cache: bool, @@ -536,6 +551,16 @@ impl KmsConfig { self } + /// Per-attempt timeout with the configured value clamped to the supported maximum. + pub(crate) fn effective_timeout(&self) -> Duration { + self.timeout.min(MAX_OPERATION_TIMEOUT) + } + + /// Retry attempts with the configured value clamped to the supported maximum. + pub(crate) fn effective_retry_attempts(&self) -> u32 { + self.retry_attempts.min(MAX_RETRY_ATTEMPTS) + } + /// Validate the configuration pub fn validate(&self) -> Result<()> { // Validate timeout @@ -548,6 +573,23 @@ impl KmsConfig { return Err(KmsError::configuration_error("Retry attempts must be greater than 0")); } + // Oversized values are clamped at use (not rejected) so pre-existing + // configurations cannot keep the service from starting after upgrade. + if self.timeout > MAX_OPERATION_TIMEOUT { + tracing::warn!( + configured_secs = self.timeout.as_secs(), + max_secs = MAX_OPERATION_TIMEOUT.as_secs(), + "KMS timeout exceeds the supported maximum; backend operations clamp it to the maximum" + ); + } + if self.retry_attempts > MAX_RETRY_ATTEMPTS { + tracing::warn!( + configured = self.retry_attempts, + max = MAX_RETRY_ATTEMPTS, + "KMS retry_attempts exceeds the supported maximum; backend operations clamp it to the maximum" + ); + } + // Validate backend-specific configuration match &self.backend_config { BackendConfig::Local(config) => { @@ -855,6 +897,31 @@ mod tests { assert_eq!(local_config.key_dir, temp_dir.path()); } + #[test] + fn test_oversized_timeout_and_retries_clamped_not_rejected() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let config = KmsConfig { + timeout: Duration::from_secs(3_600), + retry_attempts: 50, + ..KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults() + }; + + // Out-of-range values must not keep the service from starting. + assert!(config.validate().is_ok()); + assert_eq!(config.effective_timeout(), MAX_OPERATION_TIMEOUT); + assert_eq!(config.effective_retry_attempts(), MAX_RETRY_ATTEMPTS); + + // In-range values pass through unchanged. + let config = KmsConfig { + timeout: Duration::from_secs(45), + retry_attempts: 5, + ..config + }; + assert!(config.validate().is_ok()); + assert_eq!(config.effective_timeout(), Duration::from_secs(45)); + assert_eq!(config.effective_retry_attempts(), 5); + } + #[test] fn test_local_development_defaults_require_opt_in() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); diff --git a/crates/kms/src/error.rs b/crates/kms/src/error.rs index f6999aa51..d1031a32f 100644 --- a/crates/kms/src/error.rs +++ b/crates/kms/src/error.rs @@ -90,6 +90,14 @@ pub enum KmsError { /// Encryption context mismatch #[error("Encryption context mismatch: {message}")] ContextMismatch { message: String }, + + /// Backend operation exceeded its per-attempt timeout or total deadline + #[error("Operation timed out: {message}")] + OperationTimedOut { message: String }, + + /// Backend operation aborted by cancellation or shutdown + #[error("Operation cancelled: {message}")] + OperationCancelled { message: String }, } impl KmsError { @@ -187,6 +195,16 @@ impl KmsError { pub fn context_mismatch<S: Into<String>>(message: S) -> Self { Self::ContextMismatch { message: message.into() } } + + /// Create an operation timed out error + pub fn operation_timed_out<S: Into<String>>(message: S) -> Self { + Self::OperationTimedOut { message: message.into() } + } + + /// Create an operation cancelled error + pub fn operation_cancelled<S: Into<String>>(message: S) -> Self { + Self::OperationCancelled { message: message.into() } + } } /// Convert from standard library errors diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index db4791ec3..d15718ef9 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -71,6 +71,10 @@ pub mod config; mod encryption; mod error; pub mod manager; +// The executor is wired into the Vault backends in a follow-up change; until +// then the module is only exercised by its own tests. +#[allow(dead_code)] +mod policy; pub mod service; pub mod service_manager; mod time_serde; diff --git a/crates/kms/src/policy.rs b/crates/kms/src/policy.rs new file mode 100644 index 000000000..c2014b0e0 --- /dev/null +++ b/crates/kms/src/policy.rs @@ -0,0 +1,605 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Operation execution policy for external KMS backend calls. +//! +//! Vault-backed operations leave the process boundary, so every call needs a +//! per-attempt timeout, a total operation deadline, and classification-driven +//! bounded retries. This module provides the engine only; the Vault backends +//! wire their call sites through [`execute`] in a follow-up change. +//! +//! Retry safety is driven by two orthogonal classifications: +//! - [`OpClass`] states whether replaying the operation is safe at all. +//! - [`ErrorClass`] states whether the observed failure is worth replaying. +//! +//! Mutating operations without an idempotency key or CAS precondition are never +//! retried automatically: a response lost after the server applied the write +//! would otherwise be replayed into duplicate side effects (extra key versions, +//! repeated deletes). + +use std::future::Future; +use std::time::Duration; + +use rand::{RngExt, SeedableRng, rngs::StdRng}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; + +use crate::config::KmsConfig; +use crate::error::{KmsError, Result}; + +/// Default backoff cap before the first retry; doubles per retry. +const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100); + +/// Default upper bound for a single backoff sleep. +const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(2); + +/// Replay safety of a backend operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OpClass { + /// No persistent side effects (encrypt/decrypt/generate/describe/list/health); + /// safe to retry on any retryable failure. + ReadIdempotent, + /// External write without an idempotency key or CAS precondition + /// (create/rotate/delete/configure); executed at most once. + MutatingNonIdempotent, + /// Authentication exchange (login, token renewal); safe to resend. + Auth, +} + +impl OpClass { + fn retryable(self) -> bool { + !matches!(self, OpClass::MutatingNonIdempotent) + } +} + +/// Retry-relevant classification of a failed attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ErrorClass { + /// Connection-level failure (connect/send/response read); the server may or + /// may not have observed the request. + RetryableConn, + /// Retryable HTTP status: 429 throttling or a recoverable 5xx. + RetryableStatus, + /// Deterministic failure (auth, validation, not-found, malformed data); + /// retrying cannot help and may mask the real problem. + Fatal, +} + +/// Classify a `vaultrs` client error for retry purposes. +/// +/// Status codes are inspected both on `ClientError::APIError` (JSON error body) +/// and on a wrapped rustify `ServerResponseError` (non-JSON body, e.g. an HTML +/// page from an intermediate load balancer). Everything that is not throttling, +/// a recoverable 5xx, or a connection-level failure is fatal; in particular +/// 400/401/403/404 must never be retried. +pub(crate) fn classify_vaultrs(error: &vaultrs::error::ClientError) -> ErrorClass { + use rustify::errors::ClientError as RestError; + use vaultrs::error::ClientError; + + match error { + ClientError::APIError { code, .. } => classify_status(*code), + ClientError::RestClientError { source } => match source { + RestError::ServerResponseError { code, .. } => classify_status(*code), + RestError::RequestError { .. } | RestError::ResponseError { .. } => ErrorClass::RetryableConn, + _ => ErrorClass::Fatal, + }, + _ => ErrorClass::Fatal, + } +} + +fn classify_status(code: u16) -> ErrorClass { + match code { + 429 | 500 | 502 | 503 | 504 => ErrorClass::RetryableStatus, + _ => ErrorClass::Fatal, + } +} + +/// Failure of a single attempt, carrying its retry classification. +#[derive(Debug)] +pub(crate) struct AttemptError { + pub(crate) class: ErrorClass, + pub(crate) error: KmsError, +} + +/// Budgets applied by [`execute`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RetryPolicy { + /// Upper bound for one backend attempt. + pub(crate) attempt_timeout: Duration, + /// Upper bound for the whole operation, including retries and backoff. + pub(crate) op_deadline: Duration, + /// Maximum attempts for retryable operation classes; non-idempotent + /// mutations always run exactly once regardless of this value. + pub(crate) max_attempts: u32, + /// Backoff cap before the first retry; doubles per retry. + pub(crate) base_backoff: Duration, + /// Upper bound for a single backoff sleep. + pub(crate) max_backoff: Duration, +} + +impl RetryPolicy { + /// Derive the policy from the KMS configuration. + /// + /// `timeout` and `retry_attempts` are taken with the config-level clamps + /// applied. The operation deadline covers the worst-case budget of all + /// attempts plus backoff, so it bounds runaway loops without cutting any + /// attempt short; an independently configurable deadline is left to the + /// admin-API follow-up. + pub(crate) fn from_config(config: &KmsConfig) -> Self { + let mut policy = Self { + attempt_timeout: config.effective_timeout(), + op_deadline: Duration::ZERO, + max_attempts: config.effective_retry_attempts(), + base_backoff: DEFAULT_BASE_BACKOFF, + max_backoff: DEFAULT_MAX_BACKOFF, + }; + policy.op_deadline = policy.worst_case_budget(); + policy + } + + /// Total worst-case duration: every attempt hits `attempt_timeout` and + /// every backoff sleeps its full cap. + fn worst_case_budget(&self) -> Duration { + let attempts = self.max_attempts.max(1); + let mut budget = self.attempt_timeout.saturating_mul(attempts); + for completed in 1..attempts { + budget = budget.saturating_add(backoff_cap(self, completed)); + } + budget + } +} + +/// Exponential backoff cap after `completed_attempts` failed attempts. +fn backoff_cap(policy: &RetryPolicy, completed_attempts: u32) -> Duration { + let doublings = completed_attempts.saturating_sub(1).min(31); + policy.base_backoff.saturating_mul(1u32 << doublings).min(policy.max_backoff) +} + +/// Equal jitter: sleep within `[cap / 2, cap]`, keeping at least half the cap +/// so backoff still backs off while decorrelating retry bursts. +fn equal_jitter(rng: &mut impl RngExt, cap: Duration) -> Duration { + let half = cap / 2; + let spread = u64::try_from(half.as_nanos()).unwrap_or(u64::MAX); + half + Duration::from_nanos(rng.random_range(0..=spread)) +} + +/// Run `attempt` under the policy. +/// +/// Each attempt is bounded by `attempt_timeout` (further capped by whatever is +/// left of `op_deadline`), and retryable failures are replayed with exponential +/// backoff and jitter when the operation class allows it. Cancellation aborts +/// both in-flight attempts and backoff sleeps. +/// +/// A timed-out attempt counts as a connection-class failure: the server may +/// have processed the request, which is exactly why non-idempotent mutations +/// are never replayed. +pub(crate) async fn execute<T, F, Fut>( + operation: &'static str, + class: OpClass, + policy: &RetryPolicy, + cancel: &CancellationToken, + attempt: F, +) -> Result<T> +where + F: FnMut() -> Fut, + Fut: Future<Output = std::result::Result<T, AttemptError>>, +{ + // Seed an owned RNG up front: the thread-local RNG is not Send and must + // not be held across await points. + let mut rng = StdRng::from_rng(&mut rand::rng()); + execute_with_jitter(operation, class, policy, cancel, move |cap| equal_jitter(&mut rng, cap), attempt).await +} + +/// [`execute`] with an injectable jitter source so tests can pin deterministic +/// backoff durations instead of asserting around random sleeps. +pub(crate) async fn execute_with_jitter<T, F, Fut, J>( + operation: &'static str, + class: OpClass, + policy: &RetryPolicy, + cancel: &CancellationToken, + mut jitter: J, + mut attempt: F, +) -> Result<T> +where + F: FnMut() -> Fut, + Fut: Future<Output = std::result::Result<T, AttemptError>>, + J: FnMut(Duration) -> Duration, +{ + let deadline = Instant::now() + policy.op_deadline; + let max_attempts = if class.retryable() { policy.max_attempts.max(1) } else { 1 }; + + let mut attempt_no = 0u32; + loop { + attempt_no += 1; + if cancel.is_cancelled() { + return Err(KmsError::operation_cancelled(format!( + "{operation} cancelled before attempt {attempt_no}" + ))); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(KmsError::operation_timed_out(format!( + "{operation} exceeded operation deadline of {:?}", + policy.op_deadline + ))); + } + + let attempt_budget = policy.attempt_timeout.min(remaining); + let outcome = tokio::select! { + biased; + _ = cancel.cancelled() => { + return Err(KmsError::operation_cancelled(format!("{operation} cancelled during attempt {attempt_no}"))); + } + outcome = tokio::time::timeout(attempt_budget, attempt()) => outcome, + }; + + let failure = match outcome { + Ok(Ok(value)) => return Ok(value), + Ok(Err(failure)) => failure, + Err(_) => AttemptError { + class: ErrorClass::RetryableConn, + error: KmsError::operation_timed_out(format!( + "{operation} attempt {attempt_no} timed out after {attempt_budget:?}" + )), + }, + }; + + if failure.class == ErrorClass::Fatal || attempt_no >= max_attempts { + return Err(failure.error); + } + + let backoff = jitter(backoff_cap(policy, attempt_no)); + if backoff >= deadline.saturating_duration_since(Instant::now()) { + // Not enough deadline budget left for another attempt. + return Err(failure.error); + } + tracing::warn!( + operation, + attempt = attempt_no, + error_class = ?failure.class, + backoff = ?backoff, + "KMS backend attempt failed with a retryable error; backing off before retry" + ); + tokio::select! { + biased; + _ = cancel.cancelled() => { + return Err(KmsError::operation_cancelled(format!("{operation} cancelled during retry backoff"))); + } + _ = tokio::time::sleep(backoff) => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicU32, Ordering}; + + type AttemptResult<T> = std::result::Result<T, AttemptError>; + + fn policy_of(attempt_timeout_ms: u64, op_deadline_ms: u64, max_attempts: u32, base_ms: u64, max_ms: u64) -> RetryPolicy { + RetryPolicy { + attempt_timeout: Duration::from_millis(attempt_timeout_ms), + op_deadline: Duration::from_millis(op_deadline_ms), + max_attempts, + base_backoff: Duration::from_millis(base_ms), + max_backoff: Duration::from_millis(max_ms), + } + } + + /// Deterministic jitter: always sleep the full backoff cap. + fn full_jitter(cap: Duration) -> Duration { + cap + } + + fn retryable_conn_error() -> AttemptError { + AttemptError { + class: ErrorClass::RetryableConn, + error: KmsError::backend_error("connection reset by peer"), + } + } + + #[tokio::test(start_paused = true)] + async fn hung_attempt_fails_within_attempt_timeout() { + let policy = policy_of(5_000, 60_000, 1, 100, 2_000); + let cancel = CancellationToken::new(); + let started = Instant::now(); + + let result: Result<()> = execute_with_jitter("encrypt", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, || { + std::future::pending::<AttemptResult<()>>() + }) + .await; + + assert!(matches!(result, Err(KmsError::OperationTimedOut { .. })), "got {result:?}"); + assert_eq!(started.elapsed(), Duration::from_millis(5_000)); + } + + #[tokio::test(start_paused = true)] + async fn retryable_status_retries_until_success() { + let policy = policy_of(1_000, 60_000, 3, 100, 2_000); + let cancel = CancellationToken::new(); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + let started = Instant::now(); + + let result = + execute_with_jitter("generate_data_key", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + if calls.fetch_add(1, Ordering::SeqCst) < 2 { + Err(AttemptError { + class: ErrorClass::RetryableStatus, + error: KmsError::backend_error("throttled (429)"), + }) + } else { + Ok(7u32) + } + } + }) + .await; + + assert_eq!(result.expect("retries within budget must succeed"), 7); + assert_eq!(calls.load(Ordering::SeqCst), 3); + // Full-cap backoff: 100ms after attempt 1, 200ms after attempt 2. + assert_eq!(started.elapsed(), Duration::from_millis(300)); + } + + #[tokio::test(start_paused = true)] + async fn fatal_error_is_not_retried() { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + let started = Instant::now(); + + let result: Result<()> = + execute_with_jitter("decrypt", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::access_denied("permission denied (403)"), + }) + } + }) + .await; + + assert!(matches!(result, Err(KmsError::AccessDenied { .. })), "got {result:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(started.elapsed(), Duration::ZERO); + } + + #[tokio::test(start_paused = true)] + async fn mutating_non_idempotent_runs_exactly_once() { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + + let result: Result<()> = + execute_with_jitter("rotate_key", OpClass::MutatingNonIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Err(retryable_conn_error()) + } + }) + .await; + + // Even a retryable failure must not replay a non-idempotent mutation. + assert!(matches!(result, Err(KmsError::BackendError { .. })), "got {result:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test(start_paused = true)] + async fn cancel_aborts_backoff_immediately() { + // Long backoff (10s cap) so a prompt return can only come from cancellation. + let policy = policy_of(1_000, 600_000, 5, 10_000, 10_000); + let cancel = CancellationToken::new(); + let canceller = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + canceller.cancel(); + }); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + let started = Instant::now(); + + let result: Result<()> = + execute_with_jitter("decrypt", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Err(retryable_conn_error()) + } + }) + .await; + + assert!(matches!(result, Err(KmsError::OperationCancelled { .. })), "got {result:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(started.elapsed(), Duration::from_millis(500)); + } + + #[tokio::test(start_paused = true)] + async fn cancelled_token_short_circuits_before_first_attempt() { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + cancel.cancel(); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + + let result: Result<()> = + execute_with_jitter("list_keys", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + }) + .await; + + assert!(matches!(result, Err(KmsError::OperationCancelled { .. })), "got {result:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn total_duration_never_exceeds_deadline() { + // Worst case without a deadline would be 5 * 10s + backoff; the 25s + // deadline must cut both the attempt count and the final attempt short. + let policy = policy_of(10_000, 25_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + let started = Instant::now(); + + let result: Result<()> = + execute_with_jitter("describe_key", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + calls_in_attempt.fetch_add(1, Ordering::SeqCst); + std::future::pending::<AttemptResult<()>>() + }) + .await; + + assert!(matches!(result, Err(KmsError::OperationTimedOut { .. })), "got {result:?}"); + // attempt 1: 10s + 100ms backoff; attempt 2: 10s + 200ms backoff; + // attempt 3 runs with the residual 4.7s budget, then the deadline is + // exhausted and no further backoff or attempt happens. + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(started.elapsed(), Duration::from_millis(25_000)); + } + + #[tokio::test(start_paused = true)] + async fn default_jitter_stays_within_equal_jitter_bounds() { + let policy = policy_of(1_000, 60_000, 3, 100, 2_000); + let cancel = CancellationToken::new(); + let calls = Arc::new(AtomicU32::new(0)); + let calls_in_attempt = calls.clone(); + let started = Instant::now(); + + let result = execute("health_check", OpClass::ReadIdempotent, &policy, &cancel, move || { + let calls = calls_in_attempt.clone(); + async move { + if calls.fetch_add(1, Ordering::SeqCst) < 2 { + Err(retryable_conn_error()) + } else { + Ok(()) + } + } + }) + .await; + + result.expect("retries within budget must succeed"); + let elapsed = started.elapsed(); + // Equal jitter sleeps within [cap / 2, cap]; caps are 100ms then 200ms. + assert!( + elapsed >= Duration::from_millis(150) && elapsed <= Duration::from_millis(300), + "elapsed {elapsed:?} outside equal-jitter bounds" + ); + } + + #[test] + fn equal_jitter_is_seed_deterministic_and_bounded() { + let caps = [Duration::from_millis(100), Duration::from_millis(250), Duration::from_secs(2)]; + let mut first = StdRng::seed_from_u64(1569); + let mut second = StdRng::seed_from_u64(1569); + for cap in caps { + let a = equal_jitter(&mut first, cap); + let b = equal_jitter(&mut second, cap); + assert_eq!(a, b, "same seed must yield the same jitter sequence"); + assert!(a >= cap / 2 && a <= cap, "jitter {a:?} outside [{:?}, {cap:?}]", cap / 2); + } + } + + #[test] + fn backoff_caps_double_up_to_max() { + let policy = policy_of(1_000, 60_000, 6, 100, 700); + let caps: Vec<Duration> = (1..=5).map(|completed| backoff_cap(&policy, completed)).collect(); + assert_eq!( + caps, + vec![ + Duration::from_millis(100), + Duration::from_millis(200), + Duration::from_millis(400), + Duration::from_millis(700), + Duration::from_millis(700), + ] + ); + } + + #[test] + fn retry_policy_from_config_applies_clamps() { + let config = KmsConfig { + timeout: Duration::from_secs(3_600), + retry_attempts: 50, + ..KmsConfig::default() + }; + let policy = RetryPolicy::from_config(&config); + assert_eq!(policy.attempt_timeout, Duration::from_secs(300)); + assert_eq!(policy.max_attempts, 10); + // The deadline must cover the full worst case so it never cuts a + // policy-conformant operation short. + assert!(policy.op_deadline >= policy.attempt_timeout.saturating_mul(policy.max_attempts)); + + let in_range = KmsConfig::default(); + let policy = RetryPolicy::from_config(&in_range); + assert_eq!(policy.attempt_timeout, in_range.timeout); + assert_eq!(policy.max_attempts, in_range.retry_attempts); + } + + #[test] + fn classify_vaultrs_matrix() { + use rustify::errors::ClientError as RestError; + use vaultrs::error::ClientError; + + let api = |code: u16| ClientError::APIError { code, errors: vec![] }; + for code in [429u16, 500, 502, 503, 504] { + assert_eq!(classify_vaultrs(&api(code)), ErrorClass::RetryableStatus, "status {code}"); + } + for code in [400u16, 401, 403, 404, 405, 412, 501] { + assert_eq!(classify_vaultrs(&api(code)), ErrorClass::Fatal, "status {code}"); + } + + // Status errors whose body could not be parsed stay wrapped in the + // rustify error; classification must still see the code. + let raw_status = |code: u16| ClientError::RestClientError { + source: RestError::ServerResponseError { code, content: None }, + }; + assert_eq!(classify_vaultrs(&raw_status(503)), ErrorClass::RetryableStatus); + assert_eq!(classify_vaultrs(&raw_status(403)), ErrorClass::Fatal); + + let send_failure = ClientError::RestClientError { + source: RestError::RequestError { + source: anyhow::anyhow!("connection refused"), + url: "http://127.0.0.1:8200/v1/sys/health".to_string(), + method: "GET".to_string(), + }, + }; + assert_eq!(classify_vaultrs(&send_failure), ErrorClass::RetryableConn); + + let read_failure = ClientError::RestClientError { + source: RestError::ResponseError { + source: anyhow::anyhow!("connection reset by peer"), + }, + }; + assert_eq!(classify_vaultrs(&read_failure), ErrorClass::RetryableConn); + + let malformed = ClientError::JsonParseError { + source: serde_json::from_str::<serde_json::Value>("{").expect_err("truncated JSON must not parse"), + }; + assert_eq!(classify_vaultrs(&malformed), ErrorClass::Fatal); + assert_eq!(classify_vaultrs(&ClientError::ResponseEmptyError), ErrorClass::Fatal); + assert_eq!(classify_vaultrs(&ClientError::InvalidLoginMethodError), ErrorClass::Fatal); + } +} From 19cdd806a27f84d0311449e5ca50eb10161b55d6 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Thu, 30 Jul 2026 22:56:30 +0800 Subject: [PATCH 16/52] fix(kms): fail closed on missing or corrupt key material (#5475) --- crates/kms/src/backends/local.rs | 158 ++++++++++++++++++-- crates/kms/src/backends/vault.rs | 176 ++++++++++++++++------- crates/kms/src/backends/vault_transit.rs | 9 +- crates/kms/src/error.rs | 48 +++++++ rustfs/src/admin/handlers/kms_keys.rs | 14 ++ rustfs/src/error.rs | 23 +++ 6 files changed, 370 insertions(+), 58 deletions(-) diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 064afff16..d07904350 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -658,7 +658,20 @@ impl LocalKmsClient { } let content = fs::read(&key_path).await?; - let stored_key: StoredMasterKey = serde_json::from_slice(&content)?; + + // Two-stage parse so an unrecognised protection marker is reported as an + // unsupported format (a newer build may still read the key) instead of being + // folded into generic corruption with every other malformed record. + let raw: serde_json::Value = serde_json::from_slice(&content) + .map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record is not valid JSON: {e}")))?; + if let Some(marker) = raw.get("at_rest_protection") + && serde_json::from_value::<StoredKeyProtection>(marker.clone()).is_err() + { + let version = marker.as_str().map(str::to_owned).unwrap_or_else(|| marker.to_string()); + return Err(KmsError::unsupported_format_version(key_id, version)); + } + let stored_key: StoredMasterKey = serde_json::from_value(raw) + .map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record does not deserialize: {e}")))?; if stored_key.key_id != key_id { return Err(KmsError::invalid_key(format!( "Local KMS key file identity mismatch: expected {key_id:?}, found {:?}", @@ -666,9 +679,15 @@ impl LocalKmsClient { ))); } + // An empty material field is a damaged record, whatever the protection marker + // says. Fail closed: reads must never backfill or regenerate master key material. + if stored_key.encrypted_key_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + let encrypted_bytes = BASE64 .decode(&stored_key.encrypted_key_material) - .map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))?; + .map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?; let effective_protection = if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified { if stored_key.nonce.is_empty() { @@ -692,7 +711,10 @@ impl LocalKmsClient { )) })?; if stored_key.nonce.len() != 12 { - return Err(KmsError::cryptographic_error("nonce", "Invalid nonce length")); + return Err(KmsError::material_corrupt( + key_id, + format!("stored nonce has invalid length ({} bytes, expected 12)", stored_key.nonce.len()), + )); } let mut nonce_array = [0u8; 12]; @@ -701,7 +723,7 @@ impl LocalKmsClient { match cipher.decrypt(&nonce, encrypted_bytes.as_ref()) { Ok(key_material) => key_material, - Err(current_error) if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified => { + Err(_) if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified => { let legacy_cipher = self.legacy_master_cipher.as_ref().ok_or_else(|| { KmsError::configuration_error(format!( "Local KMS key {key_id} is encrypted at rest and requires a configured master key" @@ -709,9 +731,9 @@ impl LocalKmsClient { })?; legacy_cipher .decrypt(&nonce, encrypted_bytes.as_ref()) - .map_err(|_| KmsError::cryptographic_error("decrypt", current_error.to_string()))? + .map_err(|_| KmsError::material_authentication_failed(key_id))? } - Err(error) => return Err(KmsError::cryptographic_error("decrypt", error.to_string())), + Err(_) => return Err(KmsError::material_authentication_failed(key_id)), } } StoredKeyProtection::PlaintextDevOnly | StoredKeyProtection::LegacyUnspecified => { @@ -1597,6 +1619,124 @@ mod tests { assert_eq!(decrypted, plaintext, "master key material must survive status transitions"); } + /// Snapshot every file in the key directory as (name, content SHA-256, mtime). + /// Comparing snapshots proves the read paths performed no persistent write at all — + /// no rewrite, no "repair", no temp-file leftovers. + async fn snapshot_key_dir(dir: &std::path::Path) -> Vec<(String, Vec<u8>, std::time::SystemTime)> { + let mut entries = fs::read_dir(dir).await.expect("read key dir"); + let mut snapshot = Vec::new(); + while let Some(entry) = entries.next_entry().await.expect("next key dir entry") { + let content = fs::read(entry.path()).await.expect("read key dir file"); + let modified = entry + .metadata() + .await + .expect("key dir file metadata") + .modified() + .expect("key dir file mtime"); + snapshot.push(( + entry.file_name().to_string_lossy().into_owned(), + Sha256::digest(&content).to_vec(), + modified, + )); + } + snapshot.sort(); + snapshot + } + + /// Poison matrix guard: every corruption class must surface its precise typed error + /// from every read path (get_key_material / describe_key / decrypt), and the key + /// directory must stay byte-for-byte identical. Restoring any historical "self-heal" + /// behaviour (regenerating or rewriting material on a failed read) flips either the + /// error assertion (read succeeds) or the snapshot assertion (directory changed). + #[tokio::test] + async fn read_paths_fail_closed_never_write_on_poisoned_key_files() { + let (client, temp_dir) = create_test_client().await; + let key_id = "poisoned-key"; + client.create_key(key_id, "AES_256", None).await.expect("create key"); + + // Wrap a DEK while the key is healthy so the decrypt path can be exercised + // against each poisoned state of its master key. + let request = GenerateKeyRequest::new(key_id.to_string(), "AES_256".to_string()); + let data_key = client.generate_data_key(&request, None).await.expect("generate data key"); + let envelope_ciphertext = data_key.ciphertext.clone(); + + let key_path = client.master_key_path(key_id).expect("valid key id"); + let pristine: serde_json::Value = + serde_json::from_slice(&fs::read(&key_path).await.expect("read pristine key file")).expect("decode pristine record"); + let pristine_bytes = serde_json::to_vec_pretty(&pristine).expect("encode pristine record"); + + let with_field = |field: &str, value: serde_json::Value| { + let mut record = pristine.clone(); + record[field] = value; + serde_json::to_vec_pretty(&record).expect("encode poisoned record") + }; + + let tampered_material = { + let mut material = BASE64 + .decode(pristine["encrypted_key_material"].as_str().expect("material is a string")) + .expect("decode pristine material"); + *material.last_mut().expect("material is not empty") ^= 0x01; + BASE64.encode(&material) + }; + + type PoisonCase = (&'static str, Vec<u8>, fn(&KmsError) -> bool); + let poisons: Vec<PoisonCase> = vec![ + ("empty material", with_field("encrypted_key_material", serde_json::json!("")), |e| { + matches!(e, KmsError::MaterialMissing { .. }) + }), + ("truncated JSON", pristine_bytes[..pristine_bytes.len() / 2].to_vec(), |e| { + matches!(e, KmsError::MaterialCorrupt { .. }) + }), + ( + "invalid base64", + with_field("encrypted_key_material", serde_json::json!("!!!not-base64!!!")), + |e| matches!(e, KmsError::MaterialCorrupt { .. }), + ), + ("wrong nonce length", with_field("nonce", serde_json::json!([0, 1, 2])), |e| { + matches!(e, KmsError::MaterialCorrupt { .. }) + }), + ( + "tampered AEAD", + with_field("encrypted_key_material", serde_json::json!(tampered_material)), + |e| matches!(e, KmsError::MaterialAuthenticationFailed { .. }), + ), + ( + "unknown protection marker", + with_field("at_rest_protection", serde_json::json!("post-quantum-v2")), + |e| matches!(e, KmsError::UnsupportedFormatVersion { version, .. } if version == "post-quantum-v2"), + ), + ]; + + for (name, poisoned_content, expected) in poisons { + fs::write(&key_path, &poisoned_content).await.expect("write poisoned record"); + let before = snapshot_key_dir(temp_dir.path()).await; + + let error = client + .get_key_material(key_id) + .await + .expect_err("get_key_material must fail on poisoned material"); + assert!(expected(&error), "{name}: get_key_material returned wrong variant: {error:?}"); + + let error = client + .describe_key(key_id, None) + .await + .expect_err("describe_key must fail on poisoned material"); + assert!(expected(&error), "{name}: describe_key returned wrong variant: {error:?}"); + + let error = client + .decrypt(&DecryptRequest::new(envelope_ciphertext.clone()), None) + .await + .expect_err("decrypt must fail on poisoned material"); + assert!(expected(&error), "{name}: decrypt returned wrong variant: {error:?}"); + + assert_eq!( + snapshot_key_dir(temp_dir.path()).await, + before, + "{name}: read paths must not write to the key directory" + ); + } + } + #[tokio::test] async fn test_encryption_operations() { let (client, _temp_dir) = create_test_client().await; @@ -1651,7 +1791,7 @@ mod tests { Ok(_) => panic!("wrong master key must fail initialization"), Err(error) => error, }; - assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. })); + assert!(matches!(wrong_master_error, KmsError::MaterialAuthenticationFailed { .. })); } #[tokio::test] @@ -1906,7 +2046,7 @@ mod tests { .get_key_material("beta5-explicit-key") .await .expect_err("explicit current protection must not fall back to the beta.5 KDF"); - assert!(matches!(explicit_error, KmsError::CryptographicError { .. })); + assert!(matches!(explicit_error, KmsError::MaterialAuthenticationFailed { .. })); let wrong_key_error = match LocalKmsClient::new(LocalConfig { key_dir: temp_dir.path().to_path_buf(), @@ -1918,7 +2058,7 @@ mod tests { Ok(_) => panic!("wrong beta.5 master key must fail initialization"), Err(error) => error, }; - assert!(matches!(wrong_key_error, KmsError::CryptographicError { .. })); + assert!(matches!(wrong_key_error, KmsError::MaterialAuthenticationFailed { .. })); } /// R03-CAN-072 / R03-CAN-073: key identifiers arrive from request input, so every path diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index b9f8e6364..bc89a54cb 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -66,6 +66,35 @@ struct VaultKeyData { encrypted_key_material: String, } +/// Decode and validate the stored master key material of a [`VaultKeyData`] record. +/// +/// This is the single read-side gate for KV2 key material: missing or undecodable +/// material must fail closed with a typed error and must never be regenerated or +/// written back (regenerating would orphan every DEK wrapped by the original key). +/// Kept synchronous and free of Vault I/O so the poison matrix is unit-testable +/// without a live Vault. +fn decode_stored_key_material(key_id: &str, encrypted_material: &str) -> Result<Vec<u8>> { + if encrypted_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + + // Mirrors `decrypt_key_material`: stored material is currently base64 without an + // additional encryption layer. + let key_material = general_purpose::STANDARD + .decode(encrypted_material) + .map_err(|e| KmsError::material_corrupt(key_id, format!("stored key material is not valid base64: {e}")))?; + + // Key material must be exactly 32 bytes for AES-256. + if key_material.len() != 32 { + return Err(KmsError::material_corrupt( + key_id, + format!("stored key material has invalid length ({} bytes, expected 32)", key_material.len()), + )); + } + + Ok(key_material) +} + impl VaultKmsClient { /// Create a new Vault KMS client /// @@ -139,47 +168,10 @@ impl VaultKmsClient { /// Get the actual key material for a master key async fn get_key_material(&self, key_id: &str) -> Result<Vec<u8>> { - let mut key_data = self.get_key_data(key_id).await?; - - // If encrypted_key_material is empty, generate and store it (fix for old keys) - if key_data.encrypted_key_material.is_empty() { - warn!(key_id, "Vault KMS key material missing; regenerating"); - let key_material = generate_key_material(&key_data.algorithm)?; - key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; - // Store the updated key data back to Vault - self.store_key_data(key_id, &key_data).await?; - return Ok(key_material); - } - - let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await { - Ok(km) => km, - Err(e) => { - // Never regenerate/overwrite the master key on a decrypt failure: that would - // destroy the original material and make every DEK wrapped by this key - // permanently undecryptable. Surface the error so the read fails recoverably - // instead of causing silent data loss. - warn!(key_id, error = %e, "Vault KMS key material could not be decoded"); - return Err(KmsError::cryptographic_error( - "decrypt", - format!("Stored key material for {key_id} is corrupted: {e}"), - )); - } - }; - - // Validate key material length (should be 32 bytes for AES-256). - if key_material.len() != 32 { - // As above: do not overwrite the stored key. Report the fault instead. - warn!(key_id, len = key_material.len(), "Vault KMS key material has invalid length"); - return Err(KmsError::cryptographic_error( - "decrypt", - format!( - "Stored key material for {key_id} has invalid length ({} bytes, expected 32)", - key_material.len() - ), - )); - } - - Ok(key_material) + let key_data = self.get_key_data(key_id).await?; + decode_stored_key_material(key_id, &key_data.encrypted_key_material).inspect_err(|error| { + warn!(key_id, %error, "Vault KMS key material failed validation"); + }) } /// Encrypt data using a master key @@ -213,14 +205,15 @@ impl VaultKmsClient { // Get existing key data to preserve encrypted_key_material and other fields // This is called after create_key, so the key should already exist - let mut existing_key_data = self.get_key_data(key_id).await?; + let existing_key_data = self.get_key_data(key_id).await?; - // If encrypted_key_material is empty, generate it (this handles the case where - // an old key was created without proper key material) + // A key that was just created must already carry material; an empty value means + // the create flow failed to persist it. Fail closed instead of minting replacement + // material: silently generating a new key here would mask the broken create and + // orphan any DEK already wrapped by a different copy of this key. if existing_key_data.encrypted_key_material.is_empty() { warn!(key_id, "Vault KMS key metadata missing encrypted key material"); - let key_material = generate_key_material(&existing_key_data.algorithm)?; - existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; + return Err(KmsError::material_missing(key_id)); } // Update only the metadata fields, preserving the encrypted_key_material @@ -623,6 +616,14 @@ impl VaultKmsBackend { // Get the current key data from Vault let mut key_data = self.client.get_key_data(key_id).await?; + // This is a read-modify-write of the whole VaultKeyData document. Refuse to write + // back a record whose key material is missing: persisting it would cement the + // empty-material state under a fresh document version. A damaged key must go + // through an explicit repair operation, not a metadata update. + if key_data.encrypted_key_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + // Update the status based on the new metadata key_data.status = match metadata.key_state { KeyState::Enabled => KeyStatus::Active, @@ -843,6 +844,46 @@ mod tests { use super::*; use crate::config::{VaultAuthMethod, VaultConfig}; + /// Poison matrix for the read-side material gate. Every corruption class must fail + /// closed with its typed error; reintroducing any "self-heal" (regenerate on empty or + /// undecodable material) turns one of these expected errors into an Ok and fails the + /// test. Offline on purpose: `decode_stored_key_material` has no Vault I/O. + #[test] + fn decode_stored_key_material_fails_closed_on_poisoned_values() { + // Empty material means the record lost its key, not that a new one may be minted. + assert!(matches!( + decode_stored_key_material("poisoned", ""), + Err(KmsError::MaterialMissing { key_id }) if key_id == "poisoned" + )); + + // Invalid base64. + assert!(matches!( + decode_stored_key_material("poisoned", "!!!not-base64!!!"), + Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned" + )); + + // Truncated material: valid base64 of fewer than 32 bytes. + let truncated = general_purpose::STANDARD.encode([0x42u8; 16]); + assert!(matches!( + decode_stored_key_material("poisoned", &truncated), + Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned" + )); + + // Oversized material: valid base64 of more than 32 bytes. + let oversized = general_purpose::STANDARD.encode([0x42u8; 33]); + assert!(matches!( + decode_stored_key_material("poisoned", &oversized), + Err(KmsError::MaterialCorrupt { key_id, .. }) if key_id == "poisoned" + )); + + // Well-formed material still decodes. + let valid = general_purpose::STANDARD.encode([0x42u8; 32]); + assert_eq!( + decode_stored_key_material("healthy", &valid).expect("valid material must decode"), + vec![0x42u8; 32] + ); + } + #[tokio::test] #[ignore] // Requires a running Vault instance async fn test_vault_client_integration() { @@ -928,9 +969,13 @@ mod tests { client.store_key_data(&key_id, &key_data).await.expect("store corrupt"); // Reading the material must now ERROR, not silently regenerate + overwrite. + let error = client + .get_key_material(&key_id) + .await + .expect_err("corrupted key material must yield an error, not a fresh key"); assert!( - client.get_key_material(&key_id).await.is_err(), - "corrupted key material must yield an error, not a fresh key" + matches!(error, KmsError::MaterialCorrupt { .. }), + "expected MaterialCorrupt, got {error:?}" ); // And the stored (corrupted) material must be UNCHANGED. @@ -941,6 +986,41 @@ mod tests { ); } + #[tokio::test] + #[ignore] // Requires a running Vault instance (dev mode) + async fn test_empty_key_material_does_not_regenerate() { + // Regression: get_key_material previously treated empty stored material as a + // bootstrap case and silently generated + persisted a fresh master key on the + // read path. Empty material must instead fail closed as MaterialMissing and + // leave the stored record untouched. + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"); + + let key_id = format!("empty-{}", uuid::Uuid::new_v4()); + client.create_key(&key_id, "AES_256", None).await.expect("create"); + + let mut key_data = client.get_key_data(&key_id).await.expect("read"); + key_data.encrypted_key_material = String::new(); + client.store_key_data(&key_id, &key_data).await.expect("store empty"); + + let error = client + .get_key_material(&key_id) + .await + .expect_err("empty key material must yield an error, not a fresh key"); + assert!( + matches!(error, KmsError::MaterialMissing { .. }), + "expected MaterialMissing, got {error:?}" + ); + + // The stored record must still hold the empty value: no regeneration, no write. + let after = client.get_key_data(&key_id).await.expect("reread"); + assert!( + after.encrypted_key_material.is_empty(), + "get_key_material must not backfill missing master key material" + ); + } + #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) async fn test_vault_cancel_key_deletion_persists_state() { diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 9b95f826f..193856115 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -307,10 +307,17 @@ impl VaultTransitKmsClient { return Ok(persisted); } + // Deliberate exemption from the "read paths never write" rule (rustfs#4256 / + // rustfs#4262): transit keys created before metadata persistence existed have no + // KV record at all, so failing closed here would brick every pre-existing transit + // key. The synthesised record only describes metadata — key material lives solely + // inside Vault's transit engine and is never generated or written by this path. + // // Verify the transit key actually exists in Vault before synthesising. self.read_transit_key(key_id).await?; let metadata = TransitKeyMetadata::synthesized(); - // Persist the synthesised metadata so future cache misses pick it up. + // Persist the synthesised metadata so future cache misses pick it up (best + // effort: the KV write failing must not fail the read). let _ = self.write_metadata_to_kv(key_id, &metadata).await; self.metadata_cache.write().await.insert(key_id.to_string(), metadata.clone()); Ok(metadata) diff --git a/crates/kms/src/error.rs b/crates/kms/src/error.rs index d1031a32f..57b2e1785 100644 --- a/crates/kms/src/error.rs +++ b/crates/kms/src/error.rs @@ -98,6 +98,28 @@ pub enum KmsError { /// Backend operation aborted by cancellation or shutdown #[error("Operation cancelled: {message}")] OperationCancelled { message: String }, + + // New variants must be appended below (never inserted above) so that + // concurrent additions rebase without conflicts. + /// Persisted key material is absent from an otherwise readable key record + #[error( + "Key material missing for key {key_id}: the stored record has no key material; restore it from backup or repair the key explicitly" + )] + MaterialMissing { key_id: String }, + + /// Persisted key material exists but cannot be decoded + #[error("Key material corrupt for key {key_id}: {message}")] + MaterialCorrupt { key_id: String, message: String }, + + /// Persisted key material failed authenticated decryption + #[error( + "Key material authentication failed for key {key_id}: the stored material cannot be decrypted with the configured master key" + )] + MaterialAuthenticationFailed { key_id: String }, + + /// Persisted key record uses a format version unknown to this build + #[error("Unsupported key format version {version:?} for key {key_id}")] + UnsupportedFormatVersion { key_id: String, version: String }, } impl KmsError { @@ -205,6 +227,32 @@ impl KmsError { pub fn operation_cancelled<S: Into<String>>(message: S) -> Self { Self::OperationCancelled { message: message.into() } } + + /// Create a material missing error + pub fn material_missing<S: Into<String>>(key_id: S) -> Self { + Self::MaterialMissing { key_id: key_id.into() } + } + + /// Create a material corrupt error + pub fn material_corrupt<S1: Into<String>, S2: Into<String>>(key_id: S1, message: S2) -> Self { + Self::MaterialCorrupt { + key_id: key_id.into(), + message: message.into(), + } + } + + /// Create a material authentication failed error + pub fn material_authentication_failed<S: Into<String>>(key_id: S) -> Self { + Self::MaterialAuthenticationFailed { key_id: key_id.into() } + } + + /// Create an unsupported format version error + pub fn unsupported_format_version<S1: Into<String>, S2: Into<String>>(key_id: S1, version: S2) -> Self { + Self::UnsupportedFormatVersion { + key_id: key_id.into(), + version: version.into(), + } + } } /// Convert from standard library errors diff --git a/rustfs/src/admin/handlers/kms_keys.rs b/rustfs/src/admin/handlers/kms_keys.rs index 9363164f3..decc97247 100644 --- a/rustfs/src/admin/handlers/kms_keys.rs +++ b/rustfs/src/admin/handlers/kms_keys.rs @@ -851,6 +851,13 @@ impl Operation for DeleteKmsKeyHandler { let status = match &e { KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST, + // Damaged or missing key material is an integrity fault of an existing + // key: it must surface as a server error, never as NOT_FOUND (the key + // exists) and never as a retryable backend outage. + KmsError::MaterialMissing { .. } + | KmsError::MaterialCorrupt { .. } + | KmsError::MaterialAuthenticationFailed { .. } + | KmsError::UnsupportedFormatVersion { .. } => StatusCode::INTERNAL_SERVER_ERROR, _ => StatusCode::INTERNAL_SERVER_ERROR, }; let response = DeleteKmsKeyResponse { @@ -1265,6 +1272,13 @@ impl Operation for DescribeKmsKeyHandler { let status = match &e { KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, KmsError::InvalidOperation { .. } => StatusCode::BAD_REQUEST, + // Damaged or missing key material is an integrity fault of an existing + // key: it must surface as a server error, never as NOT_FOUND (the key + // exists) and never as a retryable backend outage. + KmsError::MaterialMissing { .. } + | KmsError::MaterialCorrupt { .. } + | KmsError::MaterialAuthenticationFailed { .. } + | KmsError::UnsupportedFormatVersion { .. } => StatusCode::INTERNAL_SERVER_ERROR, _ => StatusCode::INTERNAL_SERVER_ERROR, }; diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 63e6f3bf8..e5a243630 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -538,6 +538,29 @@ mod tests { assert_eq!(api_error.code, S3ErrorCode::InternalError); } + #[test] + fn test_kms_material_faults_map_to_internal_error_not_retryable_or_not_found() { + // Missing/corrupt key material is a persistent integrity fault of an existing key. + // It must not be disguised as a retryable backend outage (503 invites pointless + // retries) nor as NoSuchKey/404 (which suggests the key can be recreated — + // recreating it would orphan every DEK wrapped by the original material). + let material_faults = [ + rustfs_kms::KmsError::material_missing("key-a"), + rustfs_kms::KmsError::material_corrupt("key-a", "truncated record"), + rustfs_kms::KmsError::material_authentication_failed("key-a"), + rustfs_kms::KmsError::unsupported_format_version("key-a", "v99"), + ]; + + for fault in material_faults { + let description = fault.to_string(); + let api_error = ApiError::from(StorageError::other(fault)); + + assert_eq!(api_error.code, S3ErrorCode::InternalError, "wrong code for: {description}"); + assert_ne!(api_error.code, S3ErrorCode::ServiceUnavailable, "must not be retryable: {description}"); + assert_ne!(api_error.code, S3ErrorCode::NoSuchKey, "must not report key-not-found: {description}"); + } + } + #[test] fn test_api_error_from_storage_error_mappings() { let test_cases = vec![ From d4f2efa2ad12c259131292c758ffed03f7995709 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Thu, 30 Jul 2026 22:56:43 +0800 Subject: [PATCH 17/52] fix(kms): report accurate Vault KV2 security contract and disable unsafe rotation (#5474) --- crates/kms/src/api_types.rs | 61 +++++++++++- crates/kms/src/backends/vault.rs | 98 +++++++++++++------- crates/kms/src/config.rs | 117 ++++++++++++++++++++++-- crates/kms/src/lib.rs | 2 +- docs/operations/kms-backend-security.md | 78 ++++++++++++++++ rustfs/src/config/cli.rs | 2 +- 6 files changed, 308 insertions(+), 50 deletions(-) create mode 100644 docs/operations/kms-backend-security.md diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index 3af643c48..221529f5c 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -71,7 +71,10 @@ impl fmt::Debug for ConfigureLocalKmsRequest { } } -/// Request to configure KMS with Vault KV v2 + Transit backend +/// Request to configure KMS with the Vault KV v2 storage backend. +/// +/// This backend stores master key material directly in KV v2; confidentiality relies on +/// Vault ACLs and KV v2 at-rest encryption, with no Transit wrapping involved. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ConfigureVaultKmsRequest { @@ -82,7 +85,8 @@ pub struct ConfigureVaultKmsRequest { pub auth_method: VaultAuthMethod, /// Vault namespace (Vault Enterprise, optional) pub namespace: Option<String>, - /// Transit engine mount path + /// Deprecated: legacy Transit engine mount path. Still accepted so older clients keep + /// working, but the Vault KV2 backend never uses it. pub mount_path: Option<String>, /// KV engine mount path for storing keys pub kv_mount: Option<String>, @@ -192,7 +196,7 @@ pub enum ConfigureKmsRequest { /// Configure with Local backend #[serde(alias = "local", alias = "Local")] Local(ConfigureLocalKmsRequest), - /// Configure with Vault KV v2 + Transit backend + /// Configure with the Vault KV v2 storage backend #[serde( rename = "VaultKV2", alias = "Vault", @@ -333,7 +337,7 @@ pub enum BackendSummary { /// File permissions (octal) file_permissions: Option<u32>, }, - /// Vault KV v2 + Transit backend summary + /// Vault KV v2 storage backend summary #[serde(alias = "vault")] VaultKv2 { /// Vault server address @@ -344,7 +348,8 @@ pub enum BackendSummary { has_stored_credentials: bool, /// Namespace (if configured) namespace: Option<String>, - /// Transit engine mount path + /// Deprecated: legacy Transit mount path. Unused by the backend; kept only so the + /// serialized response shape stays stable for existing consumers. mount_path: String, /// KV engine mount path kv_mount: String, @@ -621,6 +626,52 @@ mod tests { } } + #[test] + fn test_deserialize_vault_kv2_configure_request_mount_path_optional_but_accepted() { + // deny_unknown_fields regression guard: mount_path is deprecated but must remain + // accepted so older clients that still send it do not get a 400. + let with_mount_path = serde_json::json!({ + "backend_type": "VaultKV2", + "address": "http://127.0.0.1:8200", + "auth_method": { "Token": { "token": "dev-root-token" } }, + "mount_path": "transit" + }); + let request: ConfigureKmsRequest = + serde_json::from_value(with_mount_path).expect("request with deprecated mount_path should deserialize"); + let config = request.to_kms_config(); + assert_eq!(config.vault_config().expect("vault-kv2 config").mount_path, "transit"); + + let without_mount_path = serde_json::json!({ + "backend_type": "VaultKV2", + "address": "http://127.0.0.1:8200", + "auth_method": { "Token": { "token": "dev-root-token" } } + }); + let request: ConfigureKmsRequest = + serde_json::from_value(without_mount_path).expect("request without mount_path should deserialize"); + let config = request.to_kms_config(); + assert_eq!(config.vault_config().expect("vault-kv2 config").mount_path, "transit"); + } + + #[test] + fn test_vault_kv2_status_summary_does_not_mention_transit() { + let config = KmsConfig::vault( + url::Url::parse("https://vault.example.com:8200").expect("vault URL"), + "summary-token".to_string(), + ); + let response = KmsStatusResponse { + status: KmsServiceStatus::Running, + backend_type: Some(config.backend.clone()), + healthy: Some(true), + config_summary: Some(KmsConfigSummary::from(&config)), + }; + + let json = serde_json::to_string(&response).expect("kms status response should serialize"); + assert!( + !json.contains("Transit"), + "vault-kv2 status output must not describe the backend as Transit: {json}" + ); + } + #[test] fn test_deserialize_vault_transit_configure_request() { let cases = ["VaultTransit", "vault-transit", "vault_transit"]; diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index bc89a54cb..1a72bff12 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -150,17 +150,18 @@ impl VaultKmsClient { format!("{}/{}", self.key_path_prefix, key_id) } - /// Encrypt key material using Vault's transit engine + /// Encode key material for KV2 storage. + /// + /// This is plain Base64 encoding, not encryption: the KV2 backend stores master key + /// material as-is and relies on Vault ACLs plus KV2 at-rest encryption for + /// confidentiality. Any identity with KV read access to the key path can recover the + /// plaintext master key. async fn encrypt_key_material(&self, key_material: &[u8]) -> Result<String> { - // For simplicity, we'll base64 encode the key material - // In a production setup, you would use Vault's transit engine for additional encryption Ok(general_purpose::STANDARD.encode(key_material)) } - /// Decrypt key material + /// Decode key material from KV2 storage (plain Base64, see `encrypt_key_material`). async fn decrypt_key_material(&self, encrypted_material: &str) -> Result<Vec<u8>> { - // For simplicity, we'll base64 decode the key material - // In a production setup, you would use Vault's transit engine for decryption general_purpose::STANDARD .decode(encrypted_material) .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string())) @@ -529,33 +530,13 @@ impl KmsClient for VaultKmsClient { Ok(()) } - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { - debug!("Rotating key: {}", key_id); - - let mut key_data = self.get_key_data(key_id).await?; - key_data.version += 1; - - // Generate new key material - let key_material = generate_key_material(&key_data.algorithm)?; - key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?; - - self.store_key_data(key_id, &key_data).await?; - - let master_key = MasterKeyInfo { - key_id: key_id.to_string(), - version: key_data.version, - algorithm: key_data.algorithm, - usage: key_data.usage, - status: key_data.status, - description: None, // Rotate preserves existing description (would need key lookup) - metadata: key_data.metadata, - created_at: key_data.created_at, - rotated_at: Some(Zoned::now()), - created_by: None, - }; - - debug!(key_id, "Vault KMS key rotated"); - Ok(master_key) + async fn rotate_key(&self, _key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + // Rotation previously overwrote the stored master key with fresh material, which + // permanently orphaned every DEK wrapped by the prior version. Reject before any + // storage access so existing material can never be touched. + Err(KmsError::invalid_operation( + "Vault KV2 rotation is unavailable until versioned key material retention lands", + )) } async fn health_check(&self) -> Result<()> { @@ -585,6 +566,9 @@ impl KmsClient for VaultKmsClient { BackendInfo::new("vault-kv2".to_string(), "0.1.0".to_string(), self.config.address.clone(), true) .with_metadata("kv_mount".to_string(), self.kv_mount.clone()) .with_metadata("key_prefix".to_string(), self.key_path_prefix.clone()) + // Master key material is protected only by Vault ACLs and KV2 at-rest + // encryption; there is no additional cryptographic wrapping. + .with_metadata("at_rest_protection".to_string(), "vault-kv2-acl".to_string()) } } @@ -950,6 +934,54 @@ mod tests { } } + #[tokio::test] + async fn test_vault_kv2_rotate_key_rejected_without_touching_storage() { + // No Vault instance needed: rotation must be rejected before any storage access, + // so the call cannot read or overwrite key material. + let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + + let err = client + .rotate_key("any-key", None) + .await + .expect_err("Vault KV2 rotation must be rejected"); + assert!(matches!(err, KmsError::InvalidOperation { .. }), "expected InvalidOperation, got {err:?}"); + assert!(err.to_string().contains("rotation is unavailable")); + } + + #[tokio::test] + async fn test_vault_kv2_backend_info_reports_at_rest_protection() { + let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + + let info = client.backend_info(); + assert_eq!(info.backend_type, "vault-kv2"); + assert_eq!(info.metadata.get("at_rest_protection").map(String::as_str), Some("vault-kv2-acl")); + // The KV2 backend must not present itself as Transit-backed. + assert!(!format!("{info:?}").contains("Transit")); + } + + #[tokio::test] + #[ignore] // Requires a running Vault instance (dev mode) + async fn test_vault_kv2_rotate_rejected_and_material_untouched() { + let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + + let key_id = format!("rotate-{}", uuid::Uuid::new_v4()); + client.create_key(&key_id, "AES_256", None).await.expect("create"); + let before = client.get_key_data(&key_id).await.expect("read"); + + let err = client + .rotate_key(&key_id, None) + .await + .expect_err("Vault KV2 rotation must be rejected"); + assert!(matches!(err, KmsError::InvalidOperation { .. })); + + let after = client.get_key_data(&key_id).await.expect("reread"); + assert_eq!( + after.encrypted_key_material, before.encrypted_key_material, + "rejected rotation must leave stored key material untouched" + ); + assert_eq!(after.version, before.version, "rejected rotation must not bump the key version"); + } + #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) async fn test_corrupted_key_material_does_not_regenerate() { diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 04aab91d6..2e2f5e80e 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -49,6 +49,10 @@ fn default_vault_transit_metadata_key_prefix() -> String { DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX.to_string() } +fn default_vault_kv2_mount_path() -> String { + "transit".to_string() +} + pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[ RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"), RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"), @@ -100,7 +104,9 @@ pub(crate) fn redacted_secret_option(value: Option<&str>) -> Option<&'static str /// KMS backend types #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub enum KmsBackend { - /// Vault KV v2 + Transit backend (key metadata in KV, wrapping via Transit) + /// Vault KV v2 storage backend: master key material is stored directly in KV v2. + /// Confidentiality relies on Vault ACLs, KV v2 at-rest encryption, and TLS; the + /// backend performs no Transit wrapping of key material. #[serde(rename = "VaultKV2", alias = "Vault")] VaultKv2, /// Vault Transit backend using Vault as the cryptographic source of truth @@ -162,7 +168,7 @@ impl Default for KmsConfig { pub enum BackendConfig { /// Local backend configuration Local(LocalConfig), - /// Vault KV v2 + Transit backend configuration + /// Vault KV v2 storage backend configuration #[serde(rename = "VaultKV2", alias = "Vault")] VaultKv2(Box<VaultConfig>), /// Vault Transit backend configuration @@ -270,7 +276,11 @@ impl StaticConfig { } } -/// Vault KV v2 + Transit backend configuration (metadata in KV, key wrapping via Transit) +/// Vault KV v2 backend configuration. +/// +/// Key material and metadata are stored directly in KV v2; any identity with KV read +/// access to the key path can recover plaintext master key material. Use the Vault +/// Transit backend when cryptographic isolation of key material is required. #[derive(Clone, Serialize, Deserialize)] pub struct VaultConfig { /// Vault server URL @@ -279,7 +289,10 @@ pub struct VaultConfig { pub auth_method: VaultAuthMethod, /// Vault namespace (Vault Enterprise) pub namespace: Option<String>, - /// Transit engine mount path + /// Deprecated: legacy Transit engine mount path. The Vault KV2 backend never calls + /// the Transit engine, so this value is unused; the field is retained (and + /// defaulted) only so previously persisted configurations keep deserializing. + #[serde(default = "default_vault_kv2_mount_path")] pub mount_path: String, /// KV engine mount path for storing keys pub kv_mount: String, @@ -439,7 +452,12 @@ impl KmsConfig { } } - /// Create a new KMS configuration for Vault backend with token authentication (recommended for production) + /// Create a new KMS configuration for the Vault KV v2 backend with token authentication. + /// + /// Master key material is stored directly in Vault KV v2: confidentiality relies on + /// Vault ACLs, KV v2 at-rest encryption, and TLS. KV read access to the key path is + /// equivalent to holding the plaintext master keys. Use [`KmsConfig::vault_transit`] + /// when key material must never be readable through Vault storage APIs. pub fn vault(address: Url, token: String) -> Self { Self { backend: KmsBackend::VaultKv2, @@ -452,7 +470,10 @@ impl KmsConfig { } } - /// Create a new KMS configuration for Vault backend with AppRole authentication (recommended for production) + /// Create a new KMS configuration for the Vault KV v2 backend with AppRole authentication. + /// + /// Shares the security boundary described on [`KmsConfig::vault`]: key material lives + /// in KV v2 and is protected only by Vault ACLs and KV v2 at-rest encryption. pub fn vault_approle(address: Url, role_id: String, secret_id: String) -> Self { Self { backend: KmsBackend::VaultKv2, @@ -620,9 +641,8 @@ impl KmsConfig { validate_vault_development_defaults("Vault KV2", &config.address, &config.auth_method, config.tls.as_ref())?; } - if config.mount_path.is_empty() { - return Err(KmsError::configuration_error("Vault KV2 mount path cannot be empty")); - } + // `mount_path` is deprecated and unused by this backend, so an empty value + // is deliberately not an error. // Validate TLS configuration if using HTTPS if config.address.starts_with("https://") @@ -747,11 +767,21 @@ impl KmsConfig { let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"); let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); + let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") { + Some(path) => { + tracing::warn!( + "RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused" + ); + path + } + None => default_vault_kv2_mount_path(), + }; + config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig { address, auth_method: VaultAuthMethod::Token { token }, namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), - mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"), + mount_path, kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"), tls: vault_tls_config(skip_tls_verify), @@ -1044,6 +1074,73 @@ mod tests { assert!(config.vault_config().is_some()); } + #[test] + fn test_persisted_vault_kv2_config_without_mount_path_deserializes() { + // Configurations persisted after mount_path was deprecated may omit the field; + // it must default instead of failing deserialization. + let raw = r#"{ + "backend": "VaultKV2", + "backend_config": { + "VaultKV2": { + "address": "http://127.0.0.1:8200", + "auth_method": { "Token": { "token": "t" } }, + "namespace": null, + "kv_mount": "secret", + "key_path_prefix": "rustfs/kms/keys", + "tls": null + } + }, + "default_key_id": null, + "timeout": {"secs": 30, "nanos": 0}, + "retry_attempts": 3, + "enable_cache": true, + "cache_config": { + "max_keys": 1000, + "ttl": {"secs": 3600, "nanos": 0}, + "enable_metrics": true + } + }"#; + let config: KmsConfig = serde_json::from_str(raw).expect("persisted kms config without mount_path"); + assert_eq!(config.backend, KmsBackend::VaultKv2); + let vault = config.vault_config().expect("vault-kv2 config"); + assert_eq!(vault.mount_path, "transit"); + } + + #[test] + fn test_vault_kv2_empty_mount_path_passes_validation() { + let address = Url::parse("https://vault.example.com:8200").expect("Valid URL"); + let mut config = KmsConfig::vault(address, "test-token".to_string()); + if let BackendConfig::VaultKv2(vault) = &mut config.backend_config { + vault.mount_path = String::new(); + } + assert!(config.validate().is_ok(), "deprecated mount_path must not be required"); + } + + #[test] + fn test_vault_kv2_sources_do_not_claim_transit_wrapping() { + let sources = [ + ("config.rs", include_str!("config.rs")), + ("api_types.rs", include_str!("api_types.rs")), + ("backends/vault.rs", include_str!("backends/vault.rs")), + ("lib.rs", include_str!("lib.rs")), + ]; + // Assemble the needles at runtime so this guard does not match its own source. + let needles = [ + format!("wrapping via {}", "Transit"), + format!("KV v2 + {}", "Transit"), + format!("KV2+{}", "Transit"), + format!("you would use Vault's {} engine", "transit"), + ]; + for (name, source) in sources { + for needle in &needles { + assert!( + !source.contains(needle.as_str()), + "{name} still describes the Vault KV2 backend with `{needle}`" + ); + } + } + } + #[test] fn test_legacy_persisted_vault_transit_config_uses_metadata_defaults() { let raw = r#"{ diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index d15718ef9..3867ef7b0 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -20,7 +20,7 @@ //! //! ## Features //! -//! - **Multiple Backends**: Local file storage, Vault KV2+Transit, and Vault Transit (optional) +//! - **Multiple Backends**: Local file storage, Vault KV2 (plain KV storage), and Vault Transit (optional) //! - **Object Encryption**: Transparent S3-compatible object encryption //! - **Streaming Encryption**: Memory-efficient encryption for large files //! - **Key Management**: Full lifecycle management of encryption keys diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md new file mode 100644 index 000000000..31c0d2e67 --- /dev/null +++ b/docs/operations/kms-backend-security.md @@ -0,0 +1,78 @@ +# KMS backend security properties + +RustFS ships several KMS backends. They differ not only in deployment effort +but in **where master key material lives and who can read it**. Pick a backend +based on the confidentiality boundary you need, not on the name alone. + +## Backend comparison + +| Backend | Config tag | Master key material location | At-rest protection of key material | Rotation | Intended use | +| --- | --- | --- | --- | --- | --- | +| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Rejected (no versioned retention yet) | Development; single-node setups that accept host-level trust | +| Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | Rejected (read-only backend) | Simple deployments with an external secret manager | +| Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Rejected (no versioned retention yet) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary | +| Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | + +## Vault KV2: what the backend does and does not do + +The Vault KV2 backend uses Vault purely as a **secure storage** service: + +- Master key material is generated by RustFS and written to KV v2 as a + Base64-encoded value (`encrypted_key_material` is an encoding, not a + ciphertext). +- The backend never calls the Vault Transit engine. The `mount_path` + configuration field and the `RUSTFS_KMS_VAULT_MOUNT_PATH` environment + variable are deprecated leftovers: they are accepted for compatibility and + ignored. +- Data-encryption keys (DEKs) handed to the object-encryption path are still + wrapped with AES-256-GCM under the master key; the statement above concerns + the master key's storage in Vault, not the DEK envelope. +- The backend reports this boundary in its `backend_info` metadata as + `at_rest_protection: vault-kv2-acl`. +- Key rotation is rejected (`InvalidOperation`) until versioned key material + retention lands; rotating by overwriting the stored key would orphan every + DEK wrapped by the previous version. + +> **Warning: KV read access is equivalent to holding the master keys.** +> Any Vault identity (token, AppRole, or policy) that can `read` the RustFS +> key path in KV v2 can recover the plaintext master key material and decrypt +> every object protected by those keys. Treat KV read grants on that path with +> the same care as handing out the keys themselves. If this is not acceptable, +> use the Vault Transit backend instead. + +## Minimal Vault policy for the KV2 backend + +Scope the RustFS token/AppRole to exactly the KV v2 mount and key prefix it is +configured with (defaults shown: mount `secret`, prefix `rustfs/kms/keys`), +and grant no other identity read access to that subtree: + +```hcl +# RustFS KMS (Vault KV2 backend) — key storage only, no Transit access needed. +path "secret/data/rustfs/kms/keys/*" { + capabilities = ["create", "read", "update"] +} + +path "secret/metadata/rustfs/kms/keys/*" { + capabilities = ["list", "read", "delete"] +} +``` + +Notes: + +- `delete` on the metadata path is required for permanent key deletion + (`force_immediate`); drop it if you never hard-delete keys. +- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the + KV2 backend does not use them. +- Auditing KV reads on the key prefix is strongly recommended: every read + event is a potential master-key disclosure. + +## Choosing between Vault KV2 and Vault Transit + +Use **Vault Transit** (`VaultTransit`) when key material must be +cryptographically isolated from anyone holding storage-level read access: +Transit keeps key-encryption keys inside Vault and only ever returns +ciphertext, and supports server-side key versioning/rotation. + +Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* +the confidentiality boundary and you want the operational simplicity of a +single KV mount. diff --git a/rustfs/src/config/cli.rs b/rustfs/src/config/cli.rs index 35328037e..30332d128 100644 --- a/rustfs/src/config/cli.rs +++ b/rustfs/src/config/cli.rs @@ -314,7 +314,7 @@ pub struct ServerOpts { #[arg(long, default_value_t = false, env = "RUSTFS_KMS_ENABLE")] pub kms_enable: bool, - /// KMS backend type: local, vault or vault-kv2 (Vault KV2+Transit), vault-transit + /// KMS backend type: local, vault or vault-kv2 (plain Vault KV v2 storage), vault-transit #[arg(long, default_value_t = rustfs_config::DEFAULT_KMS_BACKEND.to_string(), env = "RUSTFS_KMS_BACKEND")] pub kms_backend: String, From 7662b2436a6b02d1f16d13078ab5e9c874a7725d Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 00:14:45 +0800 Subject: [PATCH 18/52] docs(kms): document local backend durability and deployment support matrix (#5478) * docs(kms): document local backend durability and deployment support matrix * docs(kms): reflow backend security doc to one line per paragraph --- docs/operations/kms-backend-security.md | 111 ++++++++++++++---------- 1 file changed, 66 insertions(+), 45 deletions(-) diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 31c0d2e67..08ad04b75 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -1,50 +1,32 @@ # KMS backend security properties -RustFS ships several KMS backends. They differ not only in deployment effort -but in **where master key material lives and who can read it**. Pick a backend -based on the confidentiality boundary you need, not on the name alone. +RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone. ## Backend comparison -| Backend | Config tag | Master key material location | At-rest protection of key material | Rotation | Intended use | -| --- | --- | --- | --- | --- | --- | -| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Rejected (no versioned retention yet) | Development; single-node setups that accept host-level trust | -| Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | Rejected (read-only backend) | Simple deployments with an external secret manager | -| Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Rejected (no versioned retention yet) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary | -| Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | +| Backend | Config tag | Master key material location | At-rest protection of key material | Durability | Rotation | Intended use | +| --- | --- | --- | --- | --- | --- | --- | +| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Crash-durable commits on local filesystems only; see [Local backend durability and deployment support matrix](#local-backend-durability-and-deployment-support-matrix) | Rejected (no versioned retention yet) | Development; single-node setups that accept host-level trust | +| Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | No state persisted by RustFS | Rejected (read-only backend) | Simple deployments with an external secret manager | +| Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Delegated to Vault storage | Rejected (no versioned retention yet) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary | +| Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | ## Vault KV2: what the backend does and does not do The Vault KV2 backend uses Vault purely as a **secure storage** service: -- Master key material is generated by RustFS and written to KV v2 as a - Base64-encoded value (`encrypted_key_material` is an encoding, not a - ciphertext). -- The backend never calls the Vault Transit engine. The `mount_path` - configuration field and the `RUSTFS_KMS_VAULT_MOUNT_PATH` environment - variable are deprecated leftovers: they are accepted for compatibility and - ignored. -- Data-encryption keys (DEKs) handed to the object-encryption path are still - wrapped with AES-256-GCM under the master key; the statement above concerns - the master key's storage in Vault, not the DEK envelope. -- The backend reports this boundary in its `backend_info` metadata as - `at_rest_protection: vault-kv2-acl`. -- Key rotation is rejected (`InvalidOperation`) until versioned key material - retention lands; rotating by overwriting the stored key would orphan every - DEK wrapped by the previous version. +- Master key material is generated by RustFS and written to KV v2 as a Base64-encoded value (`encrypted_key_material` is an encoding, not a ciphertext). +- The backend never calls the Vault Transit engine. The `mount_path` configuration field and the `RUSTFS_KMS_VAULT_MOUNT_PATH` environment variable are deprecated leftovers: they are accepted for compatibility and ignored. +- Data-encryption keys (DEKs) handed to the object-encryption path are still wrapped with AES-256-GCM under the master key; the statement above concerns the master key's storage in Vault, not the DEK envelope. +- The backend reports this boundary in its `backend_info` metadata as `at_rest_protection: vault-kv2-acl`. +- Key rotation is rejected (`InvalidOperation`) until versioned key material retention lands; rotating by overwriting the stored key would orphan every DEK wrapped by the previous version. > **Warning: KV read access is equivalent to holding the master keys.** -> Any Vault identity (token, AppRole, or policy) that can `read` the RustFS -> key path in KV v2 can recover the plaintext master key material and decrypt -> every object protected by those keys. Treat KV read grants on that path with -> the same care as handing out the keys themselves. If this is not acceptable, -> use the Vault Transit backend instead. +> Any Vault identity (token, AppRole, or policy) that can `read` the RustFS key path in KV v2 can recover the plaintext master key material and decrypt every object protected by those keys. Treat KV read grants on that path with the same care as handing out the keys themselves. If this is not acceptable, use the Vault Transit backend instead. ## Minimal Vault policy for the KV2 backend -Scope the RustFS token/AppRole to exactly the KV v2 mount and key prefix it is -configured with (defaults shown: mount `secret`, prefix `rustfs/kms/keys`), -and grant no other identity read access to that subtree: +Scope the RustFS token/AppRole to exactly the KV v2 mount and key prefix it is configured with (defaults shown: mount `secret`, prefix `rustfs/kms/keys`), and grant no other identity read access to that subtree: ```hcl # RustFS KMS (Vault KV2 backend) — key storage only, no Transit access needed. @@ -59,20 +41,59 @@ path "secret/metadata/rustfs/kms/keys/*" { Notes: -- `delete` on the metadata path is required for permanent key deletion - (`force_immediate`); drop it if you never hard-delete keys. -- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the - KV2 backend does not use them. -- Auditing KV reads on the key prefix is strongly recommended: every read - event is a potential master-key disclosure. +- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. +- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them. +- Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure. ## Choosing between Vault KV2 and Vault Transit -Use **Vault Transit** (`VaultTransit`) when key material must be -cryptographically isolated from anyone holding storage-level read access: -Transit keeps key-encryption keys inside Vault and only ever returns -ciphertext, and supports server-side key versioning/rotation. +Use **Vault Transit** (`VaultTransit`) when key material must be cryptographically isolated from anyone holding storage-level read access: Transit keeps key-encryption keys inside Vault and only ever returns ciphertext, and supports server-side key versioning/rotation. -Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* -the confidentiality boundary and you want the operational simplicity of a -single KV mount. +Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* the confidentiality boundary and you want the operational simplicity of a single KV mount. + +## Local backend durability and deployment support matrix + +The Local backend stores one JSON record per key (`<key_id>.key`) plus an Argon2id salt file (`.master-key.salt`) inside the configured `key_dir`. This section documents which deployments that layout supports and how the backend recovers from a crash or power loss. For where the key material lives and who can read it, see the [backend comparison](#backend-comparison) above. + +### Positioning + +The facts today: + +- `Local` is the current default backend (`kms_backend` defaults to `local`). +- The RustFS Kubernetes operator places the key directory on a PersistentVolumeClaim, so the keys survive pod rescheduling. +- The in-code documentation labels the backend "for development and testing only", and configuration validation enforces stricter rules outside explicit development mode: a master key is required and `key_dir` must not live under the process temp directory. +- Production multi-node deployments should use the Vault Transit backend. + +The backend's final support level is positioning under review (internal tracking); this section describes what the implementation guarantees, not a commitment to a support tier. + +### Deployment support matrix + +| Deployment | Supported | Notes | +| --- | --- | --- | +| Local filesystem (ext4, XFS, APFS, ...) | Yes | The commit protocol relies on POSIX `rename`/`hard_link` atomicity and `fsync` durability, which local filesystems provide | +| Kubernetes PVC | Yes | Only when the PersistentVolume is backed by a local or block filesystem; this is how the RustFS operator provisions the key directory | +| NFS or other shared/network filesystems | No | Network filesystems do not reliably provide the atomicity and fsync semantics the commit protocol depends on; an NFS-backed PersistentVolume is this case, not the PVC case above | +| Multiple RustFS processes sharing one `key_dir` | No | Concurrent key **creation** is linearized (`hard_link` refuses to clobber an existing key), but every other write — status updates, deletion, cancellation — is a read-modify-write with no cross-process lock, so concurrent writers can silently lose updates | + +Within a single process, per-key write locks serialize read-modify-write updates, so concurrent API calls against one RustFS instance are safe. + +### Crash recovery behavior + +Every mutation of the key directory uses a durable commit protocol: + +1. A temp file (`<name>.tmp-<uuid>`) is created exclusively in `key_dir`. +2. The content is written and fsynced (`sync_all`). +3. The file is published atomically: `rename` to replace an existing file, `hard_link` to create a new one without clobbering. +4. The parent directory is fsynced so the new directory entry is durable. + +Deletion mirrors the tail of the protocol (`remove_file` followed by a parent directory fsync), so a deleted key cannot resurface after power loss. A crash at any step leaves either the complete old state or the complete new state, plus at most an unpublished temp file. + +On startup the backend then: + +- **Removes orphaned commit temp files.** The matcher is strict (`<prefix>.tmp-<uuid>`, never anything ending in `.key`), so published key files — including a key the user named to look like a temp file — are never touched. Publishing is atomic, so a matching leftover can only be an unpublished remnant of an interrupted commit. +- **Validates every published `.key` file.** A record that fails to decode fails startup rather than being silently skipped. +- **Guards the salt file.** If `.master-key.salt` is missing but the directory contains keys marked `encrypted-master-key`, initialization fails closed with a configuration error naming the salt path. A regenerated salt derives a different master key and can never decrypt those keys, so the correct recovery is to **restore the salt file (or the whole directory) from backup**, never to let a fresh salt be generated. An empty directory, or a legacy directory predating the salt file, still initializes normally. + +### Backing up the key directory + +Back up `key_dir` as a whole, including the hidden `.master-key.salt` file. A key file on its own is not restorable: decrypting it requires the master key derived from the configured `master_key` **and** the persisted salt. Restoring a partial directory — key files without the salt, or the salt without the key files — leaves the backend unable to decrypt, and the salt guard above will (correctly) refuse to start with encrypted keys and no salt. Losing the salt file with no backup means every key encrypted under it is unrecoverable. From 35a20622f14b4f5c51218daf13c60443f19f0e42 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 00:43:10 +0800 Subject: [PATCH 19/52] feat(kms): add master key version to data key envelope contract (#5480) * fix(kms): restore vault backend test compilation after timeout parameter PR #5472 added an attempt_timeout parameter to VaultKmsClient::new while PR #5474 landed tests still using the one-argument form, leaving 'cargo test -p rustfs-kms' unable to compile on main. Pass the same 30-second timeout the surrounding integration tests already use. * feat(kms): add master key version to data key envelope contract DataKeyEnvelope gains an optional master_key_version field recording which KEK version wrapped the DEK, so rotation-aware backends can load the matching historical material on decrypt. The field is skipped when None, keeping envelopes from non-rotating backends byte-identical to the historical seven-field JSON shape, and legacy envelopes without the field deserialize to None. The envelope discriminator marker is untouched, so mixed-format routing is unchanged in both directions. Adds the KeyVersionNotFound typed error for version-addressed material lookups that must fail closed instead of falling back to the current version. Refs rustfs/backlog#1565 --- crates/kms/src/backends/local.rs | 4 +- crates/kms/src/backends/static_kms.rs | 4 ++ crates/kms/src/backends/vault.rs | 13 +++-- crates/kms/src/backends/vault_transit.rs | 3 ++ crates/kms/src/encryption/dek.rs | 69 +++++++++++++++++++++++- crates/kms/src/error.rs | 12 +++++ 6 files changed, 100 insertions(+), 5 deletions(-) diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index d07904350..38c80477d 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -942,7 +942,8 @@ impl KmsClient for LocalKmsClient { // Encrypt the data key with the master key let (encrypted_key, nonce) = self.encrypt_with_master_key(&request.master_key_id, &plaintext_key).await?; - // Create data key envelope with master key version for rotation support + // Local rotation is rejected, so every envelope is wrapped by the key's sole + // material and needs no master key version. let envelope = DataKeyEnvelope { key_id: uuid::Uuid::new_v4().to_string(), master_key_id: request.master_key_id.clone(), @@ -951,6 +952,7 @@ impl KmsClient for LocalKmsClient { nonce, encryption_context: request.encryption_context.clone(), created_at: Zoned::now(), + master_key_version: None, }; // Serialize the envelope as the ciphertext diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 8372429a9..8dd4af518 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -137,6 +137,8 @@ impl KmsClient for StaticKmsBackend { nonce: nonce_bytes.to_vec(), encryption_context: request.encryption_context.clone(), created_at: Zoned::now(), + // The static backend has a single fixed key with no rotation. + master_key_version: None, }; let ciphertext = serde_json::to_vec(&envelope)?; @@ -181,6 +183,8 @@ impl KmsClient for StaticKmsBackend { nonce: nonce_bytes.to_vec(), encryption_context: request.encryption_context.clone(), created_at: Zoned::now(), + // The static backend has a single fixed key with no rotation. + master_key_version: None, }; let ciphertext = serde_json::to_vec(&envelope)?; diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 1a72bff12..f24b44427 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -311,6 +311,7 @@ impl KmsClient for VaultKmsClient { nonce, encryption_context: request.encryption_context.clone(), created_at: Zoned::now(), + master_key_version: None, }; // Serialize the envelope as the ciphertext @@ -938,7 +939,9 @@ mod tests { async fn test_vault_kv2_rotate_key_rejected_without_touching_storage() { // No Vault instance needed: rotation must be rejected before any storage access, // so the call cannot read or overwrite key material. - let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"); let err = client .rotate_key("any-key", None) @@ -950,7 +953,9 @@ mod tests { #[tokio::test] async fn test_vault_kv2_backend_info_reports_at_rest_protection() { - let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"); let info = client.backend_info(); assert_eq!(info.backend_type, "vault-kv2"); @@ -962,7 +967,9 @@ mod tests { #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) async fn test_vault_kv2_rotate_rejected_and_material_untouched() { - let client = VaultKmsClient::new(integration_vault_config()).await.expect("client"); + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"); let key_id = format!("rotate-{}", uuid::Uuid::new_v4()); client.create_key(&key_id, "AES_256", None).await.expect("create"); diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 193856115..72a895c2d 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -406,6 +406,9 @@ impl KmsClient for VaultTransitKmsClient { nonce: Vec::new(), encryption_context: request.encryption_context.clone(), created_at: Zoned::now(), + // Transit ciphertext already self-describes its key version + // ("vault:vN:..."), so the envelope never carries one. + master_key_version: None, }; let ciphertext = serde_json::to_vec(&envelope)?; diff --git a/crates/kms/src/encryption/dek.rs b/crates/kms/src/encryption/dek.rs index cf6fee2da..8ee252a56 100644 --- a/crates/kms/src/encryption/dek.rs +++ b/crates/kms/src/encryption/dek.rs @@ -32,7 +32,10 @@ use std::collections::HashMap; /// /// This structure stores the encrypted DEK along with metadata needed for decryption. /// The `master_key_version` field records which version of the KEK (Key Encryption Key) -/// was used to encrypt this DEK, enabling proper key rotation support. +/// wrapped this DEK so rotation-aware backends can load the matching historical +/// material. Envelopes written before versioning carry `None`; backends must resolve +/// `None` to a deterministic baseline version recorded in key metadata, never +/// implicitly to whatever version is current. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataKeyEnvelope { pub key_id: String, @@ -43,6 +46,12 @@ pub struct DataKeyEnvelope { pub encryption_context: HashMap<String, String>, #[serde(with = "crate::time_serde::zoned")] pub created_at: Zoned, + /// KEK version that wrapped `encrypted_key`; `None` on pre-versioning envelopes. + /// + /// Optional and omitted when `None` so envelopes from non-rotating backends stay + /// byte-identical to the historical seven-field JSON shape. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub master_key_version: Option<u32>, } #[derive(Deserialize)] @@ -307,6 +316,7 @@ mod tests { map }, created_at: Zoned::now(), + master_key_version: None, }; // Test serialization @@ -336,6 +346,8 @@ mod tests { let deserialized: DataKeyEnvelope = serde_json::from_str(envelope_json).expect("Should deserialize current format"); assert_eq!(deserialized.key_id, "test-key-id"); assert_eq!(deserialized.master_key_id, "master-key-id"); + // Envelopes persisted before versioning must parse with no master key version. + assert_eq!(deserialized.master_key_version, None); } #[tokio::test] @@ -353,6 +365,50 @@ mod tests { let deserialized: DataKeyEnvelope = serde_json::from_str(envelope_json).expect("Should deserialize legacy format"); assert_eq!(deserialized.key_id, "test-key-id"); assert_eq!(deserialized.master_key_id, "master-key-id"); + assert_eq!(deserialized.master_key_version, None); + } + + #[test] + fn test_data_key_envelope_none_version_serializes_without_field() { + // A `None` version must keep the serialized envelope on the historical + // seven-field JSON shape so non-rotating backends emit byte-compatible + // envelopes that older readers accept unchanged. + let envelope = DataKeyEnvelope { + key_id: "test-key-id".to_string(), + master_key_id: "master-key-id".to_string(), + key_spec: "AES_256".to_string(), + encrypted_key: vec![1, 2, 3, 4], + nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + encryption_context: HashMap::new(), + created_at: Zoned::now(), + master_key_version: None, + }; + + let value = serde_json::to_value(&envelope).expect("serialize envelope"); + let object = value.as_object().expect("envelope serializes to an object"); + assert!(!object.contains_key("master_key_version")); + assert_eq!(object.len(), 7, "None version must not change the seven-field JSON shape"); + } + + #[test] + fn test_data_key_envelope_version_round_trip() { + let envelope = DataKeyEnvelope { + key_id: "test-key-id".to_string(), + master_key_id: "master-key-id".to_string(), + key_spec: "AES_256".to_string(), + encrypted_key: vec![1, 2, 3, 4], + nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + encryption_context: HashMap::new(), + created_at: Zoned::now(), + master_key_version: Some(7), + }; + + let serialized = serde_json::to_vec(&envelope).expect("serialize envelope"); + let value: serde_json::Value = serde_json::from_slice(&serialized).expect("parse serialized envelope"); + assert_eq!(value.get("master_key_version"), Some(&serde_json::json!(7))); + + let deserialized: DataKeyEnvelope = serde_json::from_slice(&serialized).expect("deserialize envelope"); + assert_eq!(deserialized.master_key_version, Some(7)); } #[test] @@ -368,8 +424,19 @@ mod tests { }"#; let minio_legacy = br#"{"aead":"AES-256-GCM-HMAC-SHA-256","iv":[1],"nonce":[2],"bytes":[3]}"#; let duplicate_key_id = [b"{\"key_id\":\"duplicate\",".as_slice(), &kms_envelope[1..]].concat(); + // Rotation-aware envelope: the optional master_key_version field must not + // change how mixed batches of old and new envelopes are routed. + let versioned_envelope = { + let mut value: serde_json::Value = serde_json::from_slice(kms_envelope).expect("parse KMS envelope fixture"); + value + .as_object_mut() + .expect("KMS envelope fixture is an object") + .insert("master_key_version".to_string(), serde_json::json!(2)); + serde_json::to_vec(&value).expect("serialize versioned envelope") + }; assert!(is_data_key_envelope(kms_envelope)); + assert!(is_data_key_envelope(&versioned_envelope)); assert!(is_data_key_envelope(&[b" \n".as_slice(), kms_envelope].concat())); assert!(!is_data_key_envelope(&duplicate_key_id)); assert!(!is_data_key_envelope(b"bm9uY2U=:Y2lwaGVydGV4dA==")); diff --git a/crates/kms/src/error.rs b/crates/kms/src/error.rs index 57b2e1785..8a7ef6563 100644 --- a/crates/kms/src/error.rs +++ b/crates/kms/src/error.rs @@ -120,6 +120,10 @@ pub enum KmsError { /// Persisted key record uses a format version unknown to this build #[error("Unsupported key format version {version:?} for key {key_id}")] UnsupportedFormatVersion { key_id: String, version: String }, + + /// Requested master key version has no persisted material for the key + #[error("Key version {version} not found for key {key_id}")] + KeyVersionNotFound { key_id: String, version: u32 }, } impl KmsError { @@ -253,6 +257,14 @@ impl KmsError { version: version.into(), } } + + /// Create a key version not found error + pub fn key_version_not_found<S: Into<String>>(key_id: S, version: u32) -> Self { + Self::KeyVersionNotFound { + key_id: key_id.into(), + version, + } + } } /// Convert from standard library errors From 704ea43da526afdbe41aa1fa8586274d9e5da186 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 00:44:21 +0800 Subject: [PATCH 20/52] fix(kms): pass attempt timeout to Vault test clients (#5479) PR #5472 added an attempt_timeout parameter to VaultKmsClient::new while PR #5474 was developed in parallel and added three tests using the old single-argument signature. Both merged cleanly at the text level, leaving main unable to compile the rustfs-kms test target. Align the three call sites with the current constructor signature. From 699ef14dddf8f8f4a8f71fb7e450d370cf22730e Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 00:47:06 +0800 Subject: [PATCH 21/52] fix(kms): pass attempt timeout in vault kv2 tests (#5482) PR #5472 added a required attempt_timeout parameter to VaultKmsClient::new, while PR #5474 (developed in parallel) added three tests calling it with the old single-argument signature, breaking compilation of cargo test -p rustfs-kms. Pass the same 30-second timeout the neighboring tests use. From 8368017fb2c9d5027ed518f3740b8572d6ef339f Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 00:48:24 +0800 Subject: [PATCH 22/52] refactor(kms): route Vault clients through a rotatable credential provider (#5481) * fix(kms): repair vault test call sites missed by the timeout refactor Three offline tests still constructed VaultKmsClient with the pre-#5472 single-argument signature, leaving cargo test -p rustfs-kms unable to compile. Pass the same 30s attempt timeout the neighbouring tests use. * refactor(kms): route Vault clients through a rotatable credential provider Both Vault backends previously built a VaultClient in their constructor and held it for the lifetime of the backend, which leaves no seam for re-authentication: rotating credentials would require tearing down the whole backend. Introduce backends/vault_credentials with a TokenSource trait (only StaticToken for now; AppRole login and agent token files land in follow-ups) and a VaultCredentialProvider that owns the authenticated client behind an ArcSwap. Request paths take a per-call snapshot via current(), so a future rotation swaps in a new client generation without interrupting calls already in flight. Tokens held by this crate are zeroized on drop, and Debug output of every credential-carrying type is redacted (covered by a leak regression test). Behavior is unchanged: static token, namespace, and per-attempt timeout feed the same VaultClientSettings as before, and AppRole configurations are still rejected at construction with the same message. --- crates/kms/src/backends/mod.rs | 1 + crates/kms/src/backends/vault.rs | 75 ++--- crates/kms/src/backends/vault_credentials.rs | 304 +++++++++++++++++++ crates/kms/src/backends/vault_transit.rs | 86 +++--- 4 files changed, 373 insertions(+), 93 deletions(-) create mode 100644 crates/kms/src/backends/vault_credentials.rs diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index 158bb4cc0..76da244f0 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -22,6 +22,7 @@ use std::collections::HashMap; pub mod local; pub mod static_kms; pub mod vault; +pub(crate) mod vault_credentials; pub mod vault_transit; /// Abstract KMS client interface that all backends must implement diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index f24b44427..109b9b12d 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -14,6 +14,7 @@ //! Vault-based KMS backend implementation using vaultrs +use crate::backends::vault_credentials::{VaultClientHandle, VaultConnectionSettings, VaultCredentialProvider, token_source_for}; use crate::backends::{BackendInfo, KmsBackend, KmsClient}; use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -24,16 +25,14 @@ use base64::{Engine as _, engine::general_purpose}; use jiff::Zoned; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use tracing::{debug, info, warn}; -use vaultrs::{ - client::{VaultClient, VaultClientSettingsBuilder}, - kv2, -}; +use vaultrs::kv2; /// Vault KMS client implementation pub struct VaultKmsClient { - client: VaultClient, + credentials: VaultCredentialProvider, config: VaultConfig, /// Mount path for the KV engine (typically "kv" or "secret") kv_mount: String, @@ -100,44 +99,18 @@ impl VaultKmsClient { /// /// `attempt_timeout` caps every HTTP request issued through this client. pub async fn new(config: VaultConfig, attempt_timeout: Duration) -> Result<Self> { - // Create client settings - let mut settings_builder = VaultClientSettingsBuilder::default(); - settings_builder.address(&config.address); - // Defense in depth against stalled connections: vaultrs leaves the - // underlying reqwest client without any timeout by default, so a hung - // request would otherwise wait forever regardless of the - // operation-level retry policy. - settings_builder.timeout(Some(attempt_timeout)); - - // Set authentication token based on method - let token = match &config.auth_method { - crate::config::VaultAuthMethod::Token { token } => token.clone(), - crate::config::VaultAuthMethod::AppRole { .. } => { - // For AppRole authentication, we would need to first authenticate - // and get a token. For simplicity, we'll require a token for now. - return Err(KmsError::backend_error( - "AppRole authentication not yet implemented. Please use token authentication.", - )); - } + let source = token_source_for(&config.auth_method)?; + let settings = VaultConnectionSettings { + address: config.address.clone(), + namespace: config.namespace.clone(), + attempt_timeout, }; - - settings_builder.token(&token); - - if let Some(namespace) = &config.namespace { - settings_builder.namespace(Some(namespace.clone())); - } - - let settings = settings_builder - .build() - .map_err(|e| KmsError::backend_error(format!("Failed to build Vault client settings: {e}")))?; - - let client = - VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?; + let credentials = VaultCredentialProvider::new(settings, source).await?; info!(address = %config.address, "Vault KMS backend connected"); Ok(Self { - client, + credentials, kv_mount: config.kv_mount.clone(), key_path_prefix: config.key_path_prefix.clone(), config, @@ -145,6 +118,14 @@ impl VaultKmsClient { }) } + /// Snapshot the authenticated Vault client for a single request. + /// + /// Every Vault call takes its own snapshot so a credential rotation + /// applies to subsequent calls without interrupting in-flight ones. + fn vault(&self) -> Arc<VaultClientHandle> { + self.credentials.current() + } + /// Get the full path for a key in Vault fn key_path(&self, key_id: &str) -> String { format!("{}/{}", self.key_path_prefix, key_id) @@ -193,7 +174,7 @@ impl VaultKmsClient { async fn store_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> { let path = self.key_path(key_id); - kv2::set(&self.client, &self.kv_mount, &path, key_data) + kv2::set(&self.vault().client, &self.kv_mount, &path, key_data) .await .map_err(|e| KmsError::backend_error(format!("Failed to store key in Vault: {e}")))?; @@ -242,11 +223,13 @@ impl VaultKmsClient { async fn get_key_data(&self, key_id: &str) -> Result<VaultKeyData> { let path = self.key_path(key_id); - let secret: VaultKeyData = kv2::read(&self.client, &self.kv_mount, &path).await.map_err(|e| match e { - vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id), - vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), - _ => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), - })?; + let secret: VaultKeyData = kv2::read(&self.vault().client, &self.kv_mount, &path) + .await + .map_err(|e| match e { + vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id), + vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), + _ => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), + })?; debug!("Retrieved key {} from Vault, tags: {:?}", key_id, secret.tags); Ok(secret) @@ -255,7 +238,7 @@ impl VaultKmsClient { /// List all keys stored in Vault async fn list_vault_keys(&self) -> Result<Vec<String>> { // List keys under the prefix - match kv2::list(&self.client, &self.kv_mount, &self.key_path_prefix).await { + match kv2::list(&self.vault().client, &self.kv_mount, &self.key_path_prefix).await { Ok(keys) => { debug!("Found {} keys in Vault", keys.len()); Ok(keys) @@ -279,7 +262,7 @@ impl VaultKmsClient { // For this specific key path, we can safely delete the metadata // since each key has its own unique path under the prefix - kv2::delete_metadata(&self.client, &self.kv_mount, &path) + kv2::delete_metadata(&self.vault().client, &self.kv_mount, &path) .await .map_err(|e| match e { vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), diff --git a/crates/kms/src/backends/vault_credentials.rs b/crates/kms/src/backends/vault_credentials.rs new file mode 100644 index 000000000..486100dcd --- /dev/null +++ b/crates/kms/src/backends/vault_credentials.rs @@ -0,0 +1,304 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Credential plumbing shared by the Vault KV2 and Transit backends. +//! +//! [`VaultCredentialProvider`] owns the authenticated [`VaultClient`] and hands +//! out per-request snapshots. Backends take a fresh snapshot via +//! [`VaultCredentialProvider::current`] for every Vault call instead of holding +//! a client for their own lifetime: a future credential rotation then applies +//! to the next call, while calls already in flight finish on the generation +//! they captured (their `Arc` keeps it alive). + +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use arc_swap::ArcSwap; +use async_trait::async_trait; +use vaultrs::client::{VaultClient, VaultClientSettingsBuilder}; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +use crate::config::{VaultAuthMethod, redacted_secret}; +use crate::error::{KmsError, Result}; + +/// A Vault token handed out by a [`TokenSource`]. +/// +/// The crate's copy of the token is zeroized on drop. This cannot cover the +/// copy `vaultrs` keeps inside its client settings (or the HTTP headers built +/// from it); it bounds how long the token lingers in memory owned by this +/// module. +/// +/// AppRole login (PR-2) will extend this with the lease metadata returned by +/// the login endpoint (`lease_duration`, `renewable`, accessor). +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +pub(crate) struct TokenLease { + token: String, +} + +impl TokenLease { + pub(crate) fn new(token: String) -> Self { + Self { token } + } + + /// Expose the raw token for handing to the Vault client builder. + pub(crate) fn expose(&self) -> &str { + &self.token + } +} + +impl fmt::Debug for TokenLease { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TokenLease") + .field("token", &redacted_secret(&self.token)) + .finish() + } +} + +/// Source of Vault authentication tokens. +/// +/// Only [`StaticToken`] exists today. The trait is async and fallible so +/// future sources can perform I/O when acquiring a token without changing the +/// provider: +/// - `AppRoleLogin` (PR-2): performs an `auth/approle/login` round trip and +/// returns the issued token with its lease metadata; +/// - `TokenFile` (PR-3): re-reads an agent-managed token file. +#[async_trait] +pub(crate) trait TokenSource: fmt::Debug + Send + Sync { + /// Acquire a token for a new client generation. + /// + /// Called once at provider construction today; rotation (PR-2) will call + /// it again for every re-authentication. + async fn acquire(&self) -> Result<TokenLease>; +} + +/// Token source for [`VaultAuthMethod::Token`]: always yields the token fixed +/// at configuration time. +pub(crate) struct StaticToken { + token: TokenLease, +} + +impl StaticToken { + pub(crate) fn new(token: String) -> Self { + Self { + token: TokenLease::new(token), + } + } +} + +#[async_trait] +impl TokenSource for StaticToken { + async fn acquire(&self) -> Result<TokenLease> { + Ok(self.token.clone()) + } +} + +impl fmt::Debug for StaticToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TokenLease::fmt already redacts the token value. + f.debug_struct("StaticToken").field("token", &self.token).finish() + } +} + +/// Map the configured auth method onto a token source. +/// +/// AppRole is still rejected at construction time; PR-2 replaces this arm with +/// an `AppRoleLogin` source. +pub(crate) fn token_source_for(auth_method: &VaultAuthMethod) -> Result<Box<dyn TokenSource>> { + match auth_method { + VaultAuthMethod::Token { token } => Ok(Box::new(StaticToken::new(token.clone()))), + VaultAuthMethod::AppRole { .. } => Err(KmsError::backend_error( + "AppRole authentication not yet implemented. Please use token authentication.", + )), + } +} + +/// Connection parameters shared by every client generation. +#[derive(Debug, Clone)] +pub(crate) struct VaultConnectionSettings { + pub(crate) address: String, + pub(crate) namespace: Option<String>, + /// Per-attempt HTTP timeout applied to the underlying reqwest client. + pub(crate) attempt_timeout: Duration, +} + +impl VaultConnectionSettings { + /// Build an authenticated client for one generation. + fn build_client(&self, token: &TokenLease) -> Result<VaultClient> { + let mut settings_builder = VaultClientSettingsBuilder::default(); + settings_builder.address(&self.address); + // Defense in depth against stalled connections: vaultrs leaves the + // underlying reqwest client without any timeout by default, so a hung + // request would otherwise wait forever regardless of the + // operation-level retry policy. + settings_builder.timeout(Some(self.attempt_timeout)); + settings_builder.token(token.expose()); + + if let Some(namespace) = &self.namespace { + settings_builder.namespace(Some(namespace.clone())); + } + + let settings = settings_builder + .build() + .map_err(|e| KmsError::backend_error(format!("Failed to build Vault client settings: {e}")))?; + + VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}"))) + } +} + +/// One authenticated client generation. +/// +/// Request paths hold the handle (via `Arc`) for the duration of a single +/// Vault call, so a rotation that swaps in a newer generation never tears the +/// client out from under an in-flight request. +pub(crate) struct VaultClientHandle { + /// Monotonic counter identifying the credential generation this client was + /// built from. Static tokens never rotate, so only generation 0 exists + /// today; rotation (PR-2) bumps it on every re-authentication. + pub(crate) generation: u64, + pub(crate) client: VaultClient, +} + +impl fmt::Debug for VaultClientHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // `VaultClient` embeds its settings, including the token, so it must + // never appear in Debug output. + f.debug_struct("VaultClientHandle") + .field("generation", &self.generation) + .finish_non_exhaustive() + } +} + +/// Owns the authenticated Vault client for a backend and hands out +/// per-request snapshots. +/// +/// The provider keeps neither the settings nor the source after construction +/// because a static token can never be refreshed. Rotation (PR-2) will retain +/// both and add a refresh path that acquires a fresh lease, rebuilds the +/// client, and stores it under a bumped generation. +pub(crate) struct VaultCredentialProvider { + current: ArcSwap<VaultClientHandle>, +} + +impl VaultCredentialProvider { + /// Authenticate with `source` and build the initial client generation. + pub(crate) async fn new(settings: VaultConnectionSettings, source: Box<dyn TokenSource>) -> Result<Self> { + let lease = source.acquire().await?; + let client = settings.build_client(&lease)?; + Ok(Self { + current: ArcSwap::from_pointee(VaultClientHandle { generation: 0, client }), + }) + } + + /// Snapshot the current client generation. + /// + /// Take one snapshot per Vault call: the returned `Arc` pins the + /// generation for exactly that call, so a concurrent rotation applies to + /// the next call without interrupting this one. + pub(crate) fn current(&self) -> Arc<VaultClientHandle> { + self.current.load_full() + } +} + +impl fmt::Debug for VaultCredentialProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VaultCredentialProvider") + .field("current", &self.current.load()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::REDACTED_SECRET; + + const TEST_TOKEN: &str = "vault-token-debug-leak-canary"; + + fn test_settings() -> VaultConnectionSettings { + VaultConnectionSettings { + address: "http://127.0.0.1:8200".to_string(), + namespace: Some("team-namespace".to_string()), + attempt_timeout: Duration::from_secs(30), + } + } + + /// Building a provider never contacts Vault, so these tests run offline. + async fn test_provider() -> VaultCredentialProvider { + VaultCredentialProvider::new(test_settings(), Box::new(StaticToken::new(TEST_TOKEN.to_string()))) + .await + .expect("provider construction must not require a live Vault") + } + + #[tokio::test] + async fn test_static_token_snapshots_pin_one_generation() { + let provider = test_provider().await; + + let first = provider.current(); + let second = provider.current(); + + assert_eq!(first.generation, 0); + assert!( + Arc::ptr_eq(&first, &second), + "without rotation every snapshot must return the same client generation" + ); + } + + #[tokio::test] + async fn test_static_token_source_yields_configured_token() { + let source = token_source_for(&VaultAuthMethod::Token { + token: TEST_TOKEN.to_string(), + }) + .expect("token auth must map to a source"); + + let lease = source.acquire().await.expect("static acquire cannot fail"); + assert_eq!(lease.expose(), TEST_TOKEN); + } + + /// Behavior pin: AppRole keeps failing at construction with the same + /// user-visible message until the login source lands (PR-2). + #[test] + fn test_approle_auth_method_still_rejected() { + let error = token_source_for(&VaultAuthMethod::AppRole { + role_id: "role".to_string(), + secret_id: "approle-secret-canary".to_string(), + }) + .expect_err("approle must stay rejected until the login source lands"); + + let rendered = error.to_string(); + assert!(rendered.contains("AppRole authentication not yet implemented"), "got: {rendered}"); + assert!(!rendered.contains("approle-secret-canary"), "error must not echo the secret id"); + } + + /// Leak regression: the Debug output of every credential-carrying type + /// must stay free of the token literal. + #[tokio::test] + async fn test_credential_types_debug_redacts_token() { + let provider = test_provider().await; + let handle = provider.current(); + let lease = TokenLease::new(TEST_TOKEN.to_string()); + let source = StaticToken::new(TEST_TOKEN.to_string()); + + for rendered in [ + format!("{provider:?}"), + format!("{handle:?}"), + format!("{lease:?}"), + format!("{source:?}"), + ] { + assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}"); + } + + assert!(format!("{lease:?}").contains(REDACTED_SECRET)); + } +} diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 72a895c2d..8ba3535a9 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -14,6 +14,7 @@ //! Vault Transit-based KMS backend. +use crate::backends::vault_credentials::{VaultClientHandle, VaultConnectionSettings, VaultCredentialProvider, token_source_for}; use crate::backends::{BackendInfo, KmsBackend, KmsClient}; use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; @@ -24,6 +25,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use jiff::Zoned; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; use std::time::Duration; use tokio::sync::RwLock; use vaultrs::{ @@ -33,7 +35,6 @@ use vaultrs::{ CreateKeyRequestBuilder, DecryptDataRequestBuilder, EncryptDataRequestBuilder, UpdateKeyConfigurationRequestBuilder, }, }, - client::{VaultClient, VaultClientSettingsBuilder}, kv2, transit::{data, key}, }; @@ -128,7 +129,7 @@ impl From<TransitKeyMetadataPersisted> for TransitKeyMetadata { } pub struct VaultTransitKmsClient { - client: VaultClient, + credentials: VaultCredentialProvider, config: VaultTransitConfig, /// KV v2 mount path for persisting transit key metadata metadata_kv_mount: String, @@ -142,38 +143,16 @@ impl VaultTransitKmsClient { /// /// `attempt_timeout` caps every HTTP request issued through this client. pub async fn new(config: VaultTransitConfig, attempt_timeout: Duration) -> Result<Self> { - let mut settings_builder = VaultClientSettingsBuilder::default(); - settings_builder.address(&config.address); - // Defense in depth against stalled connections: vaultrs leaves the - // underlying reqwest client without any timeout by default, so a hung - // request would otherwise wait forever regardless of the - // operation-level retry policy. - settings_builder.timeout(Some(attempt_timeout)); - - let token = match &config.auth_method { - crate::config::VaultAuthMethod::Token { token } => token.clone(), - crate::config::VaultAuthMethod::AppRole { .. } => { - return Err(KmsError::backend_error( - "AppRole authentication not yet implemented. Please use token authentication.", - )); - } + let source = token_source_for(&config.auth_method)?; + let settings = VaultConnectionSettings { + address: config.address.clone(), + namespace: config.namespace.clone(), + attempt_timeout, }; - - settings_builder.token(&token); - - if let Some(namespace) = &config.namespace { - settings_builder.namespace(Some(namespace.clone())); - } - - let settings = settings_builder - .build() - .map_err(|e| KmsError::backend_error(format!("Failed to build Vault client settings: {e}")))?; - - let client = - VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?; + let credentials = VaultCredentialProvider::new(settings, source).await?; Ok(Self { - client, + credentials, metadata_kv_mount: config.metadata_kv_mount.clone(), metadata_key_prefix: config.metadata_key_prefix.clone(), config, @@ -181,6 +160,14 @@ impl VaultTransitKmsClient { }) } + /// Snapshot the authenticated Vault client for a single request. + /// + /// Every Vault call takes its own snapshot so a credential rotation + /// applies to subsequent calls without interrupting in-flight ones. + fn vault(&self) -> Arc<VaultClientHandle> { + self.credentials.current() + } + fn canonicalize_context(encryption_context: &HashMap<String, String>) -> Result<Option<String>> { if encryption_context.is_empty() { return Ok(None); @@ -205,7 +192,7 @@ impl VaultTransitKmsClient { } async fn read_transit_key(&self, key_id: &str) -> Result<vaultrs::api::transit::responses::ReadKeyResponse> { - key::read(&self.client, &self.config.mount_path, key_id) + key::read(&self.vault().client, &self.config.mount_path, key_id) .await .or_else(|e| Self::map_vault_error(key_id, e, "read")) } @@ -213,7 +200,7 @@ impl VaultTransitKmsClient { async fn create_transit_key(&self, key_id: &str) -> Result<()> { let mut builder = CreateKeyRequestBuilder::default(); builder.key_type(KeyType::Aes256Gcm96); - key::create(&self.client, &self.config.mount_path, key_id, Some(&mut builder)) + key::create(&self.vault().client, &self.config.mount_path, key_id, Some(&mut builder)) .await .map_err(|e| KmsError::backend_error(format!("Failed to create Vault Transit key {key_id}: {e}"))) } @@ -230,7 +217,7 @@ impl VaultTransitKmsClient { builder.associated_data(aad); } - let response = data::encrypt(&self.client, &self.config.mount_path, key_id, &plaintext_b64, Some(&mut builder)) + let response = data::encrypt(&self.vault().client, &self.config.mount_path, key_id, &plaintext_b64, Some(&mut builder)) .await .map_err(|e| KmsError::backend_error(format!("Failed to encrypt data with Vault Transit key {key_id}: {e}")))?; @@ -248,7 +235,7 @@ impl VaultTransitKmsClient { builder.associated_data(aad); } - let response = data::decrypt(&self.client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) + let response = data::decrypt(&self.vault().client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) .await .map_err(|e| KmsError::backend_error(format!("Failed to decrypt data with Vault Transit key {key_id}: {e}")))?; @@ -263,7 +250,7 @@ impl VaultTransitKmsClient { async fn read_metadata_from_kv(&self, key_id: &str) -> Result<Option<TransitKeyMetadata>> { let path = self.metadata_key_path(key_id); - match kv2::read::<TransitKeyMetadataPersisted>(&self.client, &self.metadata_kv_mount, &path).await { + match kv2::read::<TransitKeyMetadataPersisted>(&self.vault().client, &self.metadata_kv_mount, &path).await { Ok(persisted) => Ok(Some(persisted.into())), Err(vaultrs::error::ClientError::ResponseWrapError) | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(None), @@ -274,7 +261,7 @@ impl VaultTransitKmsClient { async fn write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { let path = self.metadata_key_path(key_id); let persisted: TransitKeyMetadataPersisted = metadata.clone().into(); - kv2::set(&self.client, &self.metadata_kv_mount, &path, &persisted) + kv2::set(&self.vault().client, &self.metadata_kv_mount, &path, &persisted) .await .map(|_| ()) .map_err(|e| KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}"))) @@ -282,7 +269,7 @@ impl VaultTransitKmsClient { async fn delete_metadata_from_kv(&self, key_id: &str) -> Result<()> { let path = self.metadata_key_path(key_id); - match kv2::delete_metadata(&self.client, &self.metadata_kv_mount, &path).await { + match kv2::delete_metadata(&self.vault().client, &self.metadata_kv_mount, &path).await { Ok(_) => Ok(()), Err(vaultrs::error::ClientError::ResponseWrapError) | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(()), @@ -496,7 +483,7 @@ impl KmsClient for VaultTransitKmsClient { } async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { - let all_keys = key::list(&self.client, &self.config.mount_path) + let all_keys = key::list(&self.vault().client, &self.config.mount_path) .await .map_err(|e| KmsError::backend_error(format!("Failed to list Vault Transit keys: {e}")))? .keys; @@ -566,7 +553,7 @@ impl KmsClient for VaultTransitKmsClient { } async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { - key::rotate(&self.client, &self.config.mount_path, key_id) + key::rotate(&self.vault().client, &self.config.mount_path, key_id) .await .map_err(|e| KmsError::backend_error(format!("Failed to rotate Vault Transit key {key_id}: {e}")))?; @@ -589,7 +576,7 @@ impl KmsClient for VaultTransitKmsClient { } async fn health_check(&self) -> Result<()> { - key::list(&self.client, &self.config.mount_path) + key::list(&self.vault().client, &self.config.mount_path) .await .map(|_| ()) .map_err(|e| KmsError::backend_error(format!("Vault Transit health check failed: {e}"))) @@ -710,13 +697,18 @@ impl KmsBackend for VaultTransitKmsBackend { if !self.client.read_transit_key(&key_id).await?.deletion_allowed { let mut update_builder = UpdateKeyConfigurationRequestBuilder::default(); update_builder.deletion_allowed(true); - key::update(&self.client.client, &self.client.config.mount_path, &key_id, Some(&mut update_builder)) - .await - .map_err(|e| { - KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")) - })?; + key::update( + &self.client.vault().client, + &self.client.config.mount_path, + &key_id, + Some(&mut update_builder), + ) + .await + .map_err(|e| { + KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")) + })?; } - key::delete(&self.client.client, &self.client.config.mount_path, &key_id) + key::delete(&self.client.vault().client, &self.client.config.mount_path, &key_id) .await .map_err(|e| KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")))?; self.client.delete_key_metadata(&key_id).await?; From 1d3ba1eb8bdc8b52b562f057cb807dfa4c208f54 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 01:48:10 +0800 Subject: [PATCH 23/52] feat(kms): retain historical master key versions for Vault KV2 rotation (#5484) Rotation previously had to be rejected outright because replacing the stored material would orphan every DEK wrapped by earlier versions. Vault KV2 now keeps each version's material in an immutable, create-only record at {prefix}/{key_id}/versions/{N} and treats the top-level record as the current-version pointer plus fast-path material copy: - decrypt resolves the envelope's master_key_version to its version record; a missing version fails closed with KeyVersionNotFound and never falls back to the current material - envelopes without a version (pre-versioning writers) resolve to the baseline_version frozen at the key's first rotation, so never-rotated keys behave exactly as before - generate_data_key stamps the wrapping version from the same key record snapshot that supplied the material - rotate_key commits in check-and-set order: freeze baseline, persist the next version's material, then switch the current pointer; any failure leaves the current pointer untouched, and concurrent rotations serialize on the CAS writes with monotonically unique versions - key listings drop the versions/ directory entries and physical key deletion purges version records before the key record Refs rustfs/backlog#1565 --- crates/kms/src/backends/vault.rs | 635 ++++++++++++++++++++++++++++--- 1 file changed, 582 insertions(+), 53 deletions(-) diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 109b9b12d..01cc4f042 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -28,7 +28,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tracing::{debug, info, warn}; -use vaultrs::kv2; +use vaultrs::{api::kv2::requests::SetSecretRequestOptions, error::ClientError, kv2}; /// Vault KMS client implementation pub struct VaultKmsClient { @@ -63,6 +63,70 @@ struct VaultKeyData { tags: HashMap<String, String>, /// Encrypted key material (base64 encoded) encrypted_key_material: String, + /// Version that pre-versioning envelopes (no `master_key_version`) resolve to. + /// + /// Recorded once, at the key's first rotation, when the then-current material is + /// frozen as an immutable version record. `None` means the key has never been + /// rotated, so legacy envelopes keep resolving to the current version — exactly + /// the pre-versioning behavior. Optional so records written by older builds keep + /// deserializing. + #[serde(default)] + baseline_version: Option<u32>, +} + +/// Immutable per-version master key material record stored under +/// `{prefix}/{key_id}/versions/{N}`. +/// +/// Version records are created with a KV2 check-and-set of 0 (create-only) and are +/// never rewritten, so every master key version that ever wrapped a DEK stays +/// readable after rotation. The top-level `{prefix}/{key_id}` record keeps a copy of +/// the current material as a fast path and so binaries that predate versioned +/// storage can still read never-rotated keys. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct VaultKeyVersionRecord { + /// Master key version this record holds material for + version: u32, + /// Encrypted key material (base64 encoded) + encrypted_key_material: String, + /// When this version's material was created + created_at: Zoned, +} + +/// Sub-path (under each key path) reserved for immutable version records. +const KEY_VERSIONS_SUBPATH: &str = "versions"; + +/// Drop KV2 directory entries from a key listing. +/// +/// Once a key has version records, listing the key prefix returns both the key +/// record itself ("my-key") and a directory entry for its version sub-path +/// ("my-key/"); only the former is a key. +fn filter_key_directory_entries(keys: Vec<String>) -> Vec<String> { + keys.into_iter().filter(|key| !key.ends_with('/')).collect() +} + +/// Resolve which master key version wrapped an envelope. +/// +/// `Some` versions are honored verbatim: if the record for that version is missing +/// the lookup must fail closed with [`KmsError::KeyVersionNotFound`], never fall +/// back to the current material. `None` (a pre-versioning envelope) resolves to the +/// key's baseline version — the deterministic version whose material was current +/// before the first rotation froze it — and, for keys that were never rotated and +/// thus have no baseline, to the current version, which matches pre-versioning +/// behavior exactly. +fn resolve_envelope_master_key_version( + envelope_version: Option<u32>, + baseline_version: Option<u32>, + current_version: u32, +) -> u32 { + envelope_version.or(baseline_version).unwrap_or(current_version) +} + +/// Whether a KV2 write failed its check-and-set precondition. +fn is_cas_conflict(error: &ClientError) -> bool { + matches!( + error, + ClientError::APIError { code: 400, errors } if errors.iter().any(|message| message.contains("check-and-set")) + ) } /// Decode and validate the stored master key material of a [`VaultKeyData`] record. @@ -131,6 +195,16 @@ impl VaultKmsClient { format!("{}/{}", self.key_path_prefix, key_id) } + /// Get the path of the immutable record holding one version's material + fn key_version_path(&self, key_id: &str, version: u32) -> String { + format!("{}/{}/{}/{}", self.key_path_prefix, key_id, KEY_VERSIONS_SUBPATH, version) + } + + /// Get the directory path holding a key's version records + fn key_versions_dir(&self, key_id: &str) -> String { + format!("{}/{}/{}", self.key_path_prefix, key_id, KEY_VERSIONS_SUBPATH) + } + /// Encode key material for KV2 storage. /// /// This is plain Base64 encoding, not encryption: the KV2 backend stores master key @@ -148,26 +222,117 @@ impl VaultKmsClient { .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string())) } - /// Get the actual key material for a master key - async fn get_key_material(&self, key_id: &str) -> Result<Vec<u8>> { - let key_data = self.get_key_data(key_id).await?; - decode_stored_key_material(key_id, &key_data.encrypted_key_material).inspect_err(|error| { - warn!(key_id, %error, "Vault KMS key material failed validation"); + /// Read the immutable material record of one key version. + /// + /// A missing record fails closed with [`KmsError::KeyVersionNotFound`]; falling + /// back to the current material would decrypt with the wrong key at best and + /// mask a tampered envelope version at worst. + async fn get_key_version_record(&self, key_id: &str, version: u32) -> Result<VaultKeyVersionRecord> { + let path = self.key_version_path(key_id, version); + + let record: VaultKeyVersionRecord = + kv2::read(&self.vault().client, &self.kv_mount, &path) + .await + .map_err(|e| match e { + ClientError::ResponseWrapError => KmsError::key_version_not_found(key_id, version), + ClientError::APIError { code: 404, .. } => KmsError::key_version_not_found(key_id, version), + _ => KmsError::backend_error(format!("Failed to read key version record from Vault: {e}")), + })?; + + if record.version != version { + return Err(KmsError::material_corrupt( + key_id, + format!("version record at {path} claims version {} instead of {version}", record.version), + )); + } + + Ok(record) + } + + /// Load master key material for a specific key version. + /// + /// The top-level record is the authoritative copy for the current version (a + /// never-rotated key has no version records at all); any other version must have + /// an immutable version record. + async fn get_key_material_for_version(&self, key_id: &str, key_data: &VaultKeyData, version: u32) -> Result<Vec<u8>> { + let encrypted_material = if version == key_data.version { + key_data.encrypted_key_material.clone() + } else { + self.get_key_version_record(key_id, version).await?.encrypted_key_material + }; + + decode_stored_key_material(key_id, &encrypted_material).inspect_err(|error| { + warn!(key_id, version, %error, "Vault KMS key material failed validation"); }) } - /// Encrypt data using a master key - async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> { - // Load the actual master key material - let key_material = self.get_key_material(key_id).await?; - self.dek_crypto.encrypt(&key_material, plaintext).await + /// Read the key record together with the KV2 secret version holding it, so a + /// later write can be check-and-set against exactly this snapshot. + async fn get_key_data_versioned(&self, key_id: &str) -> Result<(u32, VaultKeyData)> { + let path = self.key_path(key_id); + + let metadata = kv2::read_metadata(&self.vault().client, &self.kv_mount, &path) + .await + .map_err(|e| match e { + ClientError::ResponseWrapError => KmsError::key_not_found(key_id), + ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), + _ => KmsError::backend_error(format!("Failed to read key metadata from Vault: {e}")), + })?; + let cas = u32::try_from(metadata.current_version) + .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32")))?; + + // Read the exact secret version from the metadata to keep the (cas, data) + // pair consistent even if another writer lands in between. + let key_data: VaultKeyData = kv2::read_version(&self.vault().client, &self.kv_mount, &path, metadata.current_version) + .await + .map_err(|e| match e { + ClientError::ResponseWrapError => KmsError::key_not_found(key_id), + ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), + _ => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), + })?; + + Ok((cas, key_data)) } - /// Decrypt data using a master key - async fn decrypt_with_master_key(&self, key_id: &str, ciphertext: &[u8], nonce: &[u8]) -> Result<Vec<u8>> { - // Load the actual master key material - let key_material = self.get_key_material(key_id).await?; - self.dek_crypto.decrypt(&key_material, ciphertext, nonce).await + /// Check-and-set write of the key record. + /// + /// `cas` must match the KV2 secret version currently holding the record. + /// Returns the secret version created by this write so a caller can chain + /// further check-and-set writes. + async fn cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result<u32> { + let path = self.key_path(key_id); + + let written = + kv2::set_with_options(&self.vault().client, &self.kv_mount, &path, key_data, SetSecretRequestOptions { cas }) + .await + .map_err(|e| { + if is_cas_conflict(&e) { + KmsError::invalid_operation(format!( + "Concurrent modification of key {key_id} detected, retry the rotation" + )) + } else { + KmsError::backend_error(format!("Failed to store key in Vault: {e}")) + } + })?; + + u32::try_from(written.version) + .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32"))) + } + + /// Create-only write of an immutable version record (KV2 check-and-set of 0). + /// + /// Returns `Ok(true)` when this call created the record and `Ok(false)` when a + /// record already exists at that version; the caller decides whether the + /// existing record is acceptable. The record is never overwritten. + async fn try_create_key_version_record(&self, key_id: &str, record: &VaultKeyVersionRecord) -> Result<bool> { + let path = self.key_version_path(key_id, record.version); + + match kv2::set_with_options(&self.vault().client, &self.kv_mount, &path, record, SetSecretRequestOptions { cas: 0 }).await + { + Ok(_) => Ok(true), + Err(e) if is_cas_conflict(&e) => Ok(false), + Err(e) => Err(KmsError::backend_error(format!("Failed to store key version record in Vault: {e}"))), + } } /// Store key data in Vault @@ -209,6 +374,7 @@ impl VaultKmsClient { metadata: existing_key_data.metadata.clone(), tags: request.tags.clone(), encrypted_key_material: existing_key_data.encrypted_key_material.clone(), // Preserve the key material + baseline_version: existing_key_data.baseline_version, }; debug!( @@ -240,6 +406,7 @@ impl VaultKmsClient { // List keys under the prefix match kv2::list(&self.vault().client, &self.kv_mount, &self.key_path_prefix).await { Ok(keys) => { + let keys = filter_key_directory_entries(keys); debug!("Found {} keys in Vault", keys.len()); Ok(keys) } @@ -260,6 +427,24 @@ impl VaultKmsClient { async fn delete_key(&self, key_id: &str) -> Result<()> { let path = self.key_path(key_id); + // Purge immutable version records first: if any purge fails, the top-level + // record still exists and the deletion can be retried. The reverse order + // would leave orphaned master key material in Vault after the key vanished. + let versions_dir = self.key_versions_dir(key_id); + match kv2::list(&self.vault().client, &self.kv_mount, &versions_dir).await { + Ok(versions) => { + for version in versions { + let version_path = format!("{versions_dir}/{version}"); + kv2::delete_metadata(&self.vault().client, &self.kv_mount, &version_path) + .await + .map_err(|e| KmsError::backend_error(format!("Failed to delete key version record from Vault: {e}")))?; + } + } + // No version records exist (the key was never rotated). + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => {} + Err(e) => return Err(KmsError::backend_error(format!("Failed to list key version records in Vault: {e}"))), + } + // For this specific key path, we can safely delete the metadata // since each key has its own unique path under the prefix kv2::delete_metadata(&self.vault().client, &self.kv_mount, &path) @@ -282,8 +467,16 @@ impl KmsClient for VaultKmsClient { // Generate random data key material using the existing method let plaintext_key = generate_key_material(&request.key_spec)?; - // Encrypt the data key with the master key - let (encrypted_key, nonce) = self.encrypt_with_master_key(&request.master_key_id, &plaintext_key).await?; + // Encrypt the data key with the current master key material. Single read of + // the key record: the material we wrap with and the version we stamp into + // the envelope must come from the same snapshot, or a concurrent rotation + // could stamp a version that never wrapped this DEK. + let key_data = self.get_key_data(&request.master_key_id).await?; + let key_material = + decode_stored_key_material(&request.master_key_id, &key_data.encrypted_key_material).inspect_err(|error| { + warn!(key_id = %request.master_key_id, %error, "Vault KMS key material failed validation"); + })?; + let (encrypted_key, nonce) = self.dek_crypto.encrypt(&key_material, &plaintext_key).await?; // Create data key envelope with master key version for rotation support let envelope = DataKeyEnvelope { @@ -294,7 +487,7 @@ impl KmsClient for VaultKmsClient { nonce, encryption_context: request.encryption_context.clone(), created_at: Zoned::now(), - master_key_version: None, + master_key_version: Some(key_data.version), }; // Serialize the envelope as the ciphertext @@ -354,9 +547,16 @@ impl KmsClient for VaultKmsClient { } } - // Decrypt the data key + // Decrypt the data key with the master key version that wrapped it + let key_data = self.get_key_data(&envelope.master_key_id).await?; + let version = + resolve_envelope_master_key_version(envelope.master_key_version, key_data.baseline_version, key_data.version); + let key_material = self + .get_key_material_for_version(&envelope.master_key_id, &key_data, version) + .await?; let plaintext = self - .decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce) + .dek_crypto + .decrypt(&key_material, &envelope.encrypted_key, &envelope.nonce) .await?; debug!("Vault KMS data decrypted"); @@ -386,6 +586,7 @@ impl KmsClient for VaultKmsClient { metadata: HashMap::new(), tags: HashMap::new(), encrypted_key_material: encrypted_material, + baseline_version: None, }; // Store in Vault @@ -514,13 +715,102 @@ impl KmsClient for VaultKmsClient { Ok(()) } - async fn rotate_key(&self, _key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { - // Rotation previously overwrote the stored master key with fresh material, which - // permanently orphaned every DEK wrapped by the prior version. Reject before any - // storage access so existing material can never be touched. - Err(KmsError::invalid_operation( - "Vault KV2 rotation is unavailable until versioned key material retention lands", - )) + /// Rotate the master key while keeping every historical version decryptable. + /// + /// Commit protocol (all writes check-and-set, in this order): + /// 1. First rotation only: freeze the current material as an immutable version + /// record and persist `baseline_version` so pre-versioning envelopes resolve + /// to it deterministically. + /// 2. Persist the next version's material as an immutable version record + /// (create-only) before anything references it. + /// 3. Switch the current pointer: bump `version` and mirror the new material + /// into the top-level record in a single check-and-set write. + /// + /// If any step fails the current pointer is untouched, so a failed, cancelled, + /// or interrupted rotation never exposes half-committed material. Concurrent + /// rotations are serialized by the check-and-set writes: at most one caller + /// commits each version and the losers fail without side effects on current. + async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + debug!("Rotating master key: {}", key_id); + + let (mut cas, mut key_data) = self.get_key_data_versioned(key_id).await?; + + // The material about to be frozen must be decodable: freezing poisoned + // material would give legacy envelopes a permanently broken baseline. This + // surfaces the same typed Material* errors as the read path. + decode_stored_key_material(key_id, &key_data.encrypted_key_material) + .inspect_err(|error| warn!(key_id, %error, "Vault KMS key material failed validation"))?; + + // Step 1: freeze the baseline on first rotation. + if key_data.baseline_version.is_none() { + let baseline = VaultKeyVersionRecord { + version: key_data.version, + encrypted_key_material: key_data.encrypted_key_material.clone(), + created_at: key_data.created_at.clone(), + }; + if !self.try_create_key_version_record(key_id, &baseline).await? { + // Either a previous rotation attempt crashed between freezing the + // baseline and recording it in metadata, or a concurrent rotation + // got here first. Both are benign only if the existing record holds + // exactly the material being frozen; anything else means the + // version history is inconsistent and rotation must not proceed. + let existing = self.get_key_version_record(key_id, key_data.version).await?; + if existing.encrypted_key_material != key_data.encrypted_key_material { + return Err(KmsError::internal_error(format!( + "version record {} of key {key_id} does not match the current key material; refusing to rotate", + key_data.version + ))); + } + } + key_data.baseline_version = Some(key_data.version); + cas = self.cas_store_key_data(key_id, &key_data, cas).await?; + } + + // Step 2: durably persist the next version's material before it can become + // current. + let new_version = key_data + .version + .checked_add(1) + .ok_or_else(|| KmsError::internal_error(format!("key {key_id} exhausted the version space")))?; + let generated = generate_key_material(&key_data.algorithm)?; + let mut new_material = self.encrypt_key_material(&generated).await?; + let record = VaultKeyVersionRecord { + version: new_version, + encrypted_key_material: new_material.clone(), + created_at: Zoned::now(), + }; + if !self.try_create_key_version_record(key_id, &record).await? { + // A record for the next version already exists: an interrupted rotation + // persisted it and stopped before switching the current pointer, or a + // concurrent rotation just created it. Adopt the persisted material — + // it is immutable, fully durable, and has never been current — instead + // of failing the create-only write forever. The check-and-set switch + // below still lets at most one caller commit this version. + let existing = self.get_key_version_record(key_id, new_version).await?; + decode_stored_key_material(key_id, &existing.encrypted_key_material)?; + new_material = existing.encrypted_key_material; + } + + // Step 3: switch the current pointer. The top-level copy of the material is + // the fast path for new encryptions and must always match `version`. + key_data.version = new_version; + key_data.encrypted_key_material = new_material; + self.cas_store_key_data(key_id, &key_data, cas).await?; + + info!(key_id, version = new_version, "Vault KMS master key rotated"); + + Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: new_version, + algorithm: key_data.algorithm.clone(), + usage: key_data.usage.clone(), + status: key_data.status, + description: key_data.description.clone(), + metadata: key_data.metadata.clone(), + created_at: key_data.created_at.clone(), + rotated_at: Some(Zoned::now()), + created_by: None, + }) } async fn health_check(&self) -> Result<()> { @@ -919,19 +1209,88 @@ mod tests { } #[tokio::test] - async fn test_vault_kv2_rotate_key_rejected_without_touching_storage() { - // No Vault instance needed: rotation must be rejected before any storage access, - // so the call cannot read or overwrite key material. + async fn test_key_version_paths_stay_under_the_key() { let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) .await .expect("client"); - let err = client - .rotate_key("any-key", None) - .await - .expect_err("Vault KV2 rotation must be rejected"); - assert!(matches!(err, KmsError::InvalidOperation { .. }), "expected InvalidOperation, got {err:?}"); - assert!(err.to_string().contains("rotation is unavailable")); + assert_eq!(client.key_path("my-key"), "rustfs/kms/keys/my-key"); + assert_eq!(client.key_versions_dir("my-key"), "rustfs/kms/keys/my-key/versions"); + assert_eq!(client.key_version_path("my-key", 3), "rustfs/kms/keys/my-key/versions/3"); + } + + #[test] + fn test_filter_key_directory_entries_drops_version_dirs() { + // Listing the key prefix returns "my-key/" as a directory entry once + // my-key has version records; only real key records may be listed. + let listed = vec!["alpha".to_string(), "alpha/".to_string(), "beta".to_string()]; + assert_eq!(filter_key_directory_entries(listed), vec!["alpha".to_string(), "beta".to_string()]); + } + + #[test] + fn test_resolve_envelope_master_key_version_rules() { + // An explicit envelope version is honored verbatim, even when it differs + // from both the baseline and the current version: whether material exists + // for it is decided by the versioned lookup, never by falling back. + assert_eq!(resolve_envelope_master_key_version(Some(2), Some(1), 5), 2); + assert_eq!(resolve_envelope_master_key_version(Some(9), Some(1), 5), 9); + + // A pre-versioning envelope resolves to the frozen baseline, not to + // whatever version happens to be current. + assert_eq!(resolve_envelope_master_key_version(None, Some(1), 5), 1); + + // Never-rotated keys have no baseline; the current version is the only + // material that ever existed, matching pre-versioning behavior. + assert_eq!(resolve_envelope_master_key_version(None, None, 1), 1); + } + + #[test] + fn test_vault_key_data_without_baseline_version_deserializes() { + // Key records written before versioned storage have no baseline_version + // field and must keep deserializing with None. + let key_data = VaultKeyData { + algorithm: "AES_256".to_string(), + usage: KeyUsage::EncryptDecrypt, + created_at: Zoned::now(), + status: KeyStatus::Active, + version: 1, + description: None, + metadata: HashMap::new(), + tags: HashMap::new(), + encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]), + baseline_version: Some(1), + }; + + let mut value = serde_json::to_value(&key_data).expect("serialize key data"); + value + .as_object_mut() + .expect("key data serializes to an object") + .remove("baseline_version"); + + let legacy: VaultKeyData = serde_json::from_value(value).expect("legacy record must deserialize"); + assert_eq!(legacy.baseline_version, None); + assert_eq!(legacy.version, 1); + } + + #[test] + fn test_is_cas_conflict_only_matches_cas_failures() { + let cas = ClientError::APIError { + code: 400, + errors: vec!["check-and-set parameter did not match the current version".to_string()], + }; + assert!(is_cas_conflict(&cas)); + + let other_400 = ClientError::APIError { + code: 400, + errors: vec!["invalid request".to_string()], + }; + assert!(!is_cas_conflict(&other_400)); + + let not_found = ClientError::APIError { + code: 404, + errors: Vec::new(), + }; + assert!(!is_cas_conflict(¬_found)); } #[tokio::test] @@ -947,29 +1306,197 @@ mod tests { assert!(!format!("{info:?}").contains("Transit")); } + fn integration_generate_request(key_id: &str) -> GenerateKeyRequest { + GenerateKeyRequest { + master_key_id: key_id.to_string(), + key_spec: "AES_256".to_string(), + key_length: Some(32), + encryption_context: Default::default(), + grant_tokens: Vec::new(), + } + } + + fn integration_decrypt_request(ciphertext: Vec<u8>) -> DecryptRequest { + DecryptRequest { + ciphertext, + encryption_context: Default::default(), + grant_tokens: Vec::new(), + } + } + #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) - async fn test_vault_kv2_rotate_rejected_and_material_untouched() { + async fn test_vault_kv2_decrypt_after_rotate() { let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) .await .expect("client"); - let key_id = format!("rotate-{}", uuid::Uuid::new_v4()); + let key_id = format!("rotate-retain-{}", uuid::Uuid::new_v4()); client.create_key(&key_id, "AES_256", None).await.expect("create"); - let before = client.get_key_data(&key_id).await.expect("read"); + let request = integration_generate_request(&key_id); - let err = client - .rotate_key(&key_id, None) + let dk_v1 = client.generate_data_key(&request, None).await.expect("generate under v1"); + let env_v1: DataKeyEnvelope = serde_json::from_slice(&dk_v1.ciphertext).expect("parse v1 envelope"); + assert_eq!(env_v1.master_key_version, Some(1)); + + let rotated = client.rotate_key(&key_id, None).await.expect("rotate to v2"); + assert_eq!(rotated.version, 2); + let dk_v2 = client.generate_data_key(&request, None).await.expect("generate under v2"); + let env_v2: DataKeyEnvelope = serde_json::from_slice(&dk_v2.ciphertext).expect("parse v2 envelope"); + assert_eq!(env_v2.master_key_version, Some(2), "new envelopes must carry the latest version"); + + let rotated = client.rotate_key(&key_id, None).await.expect("rotate to v3"); + assert_eq!(rotated.version, 3); + let dk_v3 = client.generate_data_key(&request, None).await.expect("generate under v3"); + let env_v3: DataKeyEnvelope = serde_json::from_slice(&dk_v3.ciphertext).expect("parse v3 envelope"); + assert_eq!(env_v3.master_key_version, Some(3)); + + // A mixed batch of envelopes from every historical version must decrypt. + for (data_key, label) in [(&dk_v1, "v1"), (&dk_v3, "v3"), (&dk_v2, "v2"), (&dk_v1, "v1 again")] { + let plaintext = client + .decrypt(&integration_decrypt_request(data_key.ciphertext.clone()), None) + .await + .unwrap_or_else(|error| panic!("envelope wrapped under {label} must stay decryptable: {error}")); + assert_eq!(Some(plaintext), data_key.plaintext, "{label} plaintext must round-trip"); + } + } + + #[tokio::test] + #[ignore] // Requires a running Vault instance (dev mode) + async fn test_vault_kv2_rotate_does_not_orphan_legacy_envelopes() { + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) .await - .expect_err("Vault KV2 rotation must be rejected"); - assert!(matches!(err, KmsError::InvalidOperation { .. })); + .expect("client"); - let after = client.get_key_data(&key_id).await.expect("reread"); - assert_eq!( - after.encrypted_key_material, before.encrypted_key_material, - "rejected rotation must leave stored key material untouched" + let key_id = format!("rotate-legacy-{}", uuid::Uuid::new_v4()); + client.create_key(&key_id, "AES_256", None).await.expect("create"); + + // Simulate an envelope written by a pre-versioning build: same wrapped DEK, + // but without the master_key_version field. + let data_key = client + .generate_data_key(&integration_generate_request(&key_id), None) + .await + .expect("generate"); + let mut envelope: serde_json::Value = serde_json::from_slice(&data_key.ciphertext).expect("parse envelope"); + envelope + .as_object_mut() + .expect("envelope is an object") + .remove("master_key_version"); + let legacy_ciphertext = serde_json::to_vec(&envelope).expect("serialize legacy envelope"); + + client.rotate_key(&key_id, None).await.expect("rotate to v2"); + client.rotate_key(&key_id, None).await.expect("rotate to v3"); + + // The baseline rule must route the legacy envelope to the frozen version 1 + // material even though the current version has moved on. + let plaintext = client + .decrypt(&integration_decrypt_request(legacy_ciphertext), None) + .await + .expect("legacy envelope must stay decryptable after rotation"); + assert_eq!(Some(plaintext), data_key.plaintext); + + let key_data = client.get_key_data(&key_id).await.expect("read"); + assert_eq!(key_data.baseline_version, Some(1), "first rotation must pin the baseline"); + assert_eq!(key_data.version, 3); + } + + #[tokio::test] + #[ignore] // Requires a running Vault instance (dev mode) + async fn test_vault_kv2_envelope_version_tampering_fails_closed() { + let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"); + + let key_id = format!("rotate-tamper-{}", uuid::Uuid::new_v4()); + client.create_key(&key_id, "AES_256", None).await.expect("create"); + let data_key = client + .generate_data_key(&integration_generate_request(&key_id), None) + .await + .expect("generate"); + client.rotate_key(&key_id, None).await.expect("rotate"); + + // Point the envelope at a version that has no material record. + let mut envelope: serde_json::Value = serde_json::from_slice(&data_key.ciphertext).expect("parse envelope"); + envelope + .as_object_mut() + .expect("envelope is an object") + .insert("master_key_version".to_string(), serde_json::json!(999)); + let tampered = serde_json::to_vec(&envelope).expect("serialize tampered envelope"); + + let error = client + .decrypt(&integration_decrypt_request(tampered), None) + .await + .expect_err("nonexistent version must fail closed, not fall back to current"); + assert!( + matches!(error, KmsError::KeyVersionNotFound { version: 999, key_id: ref error_key_id } if *error_key_id == key_id), + "expected KeyVersionNotFound for version 999, got {error:?}" ); - assert_eq!(after.version, before.version, "rejected rotation must not bump the key version"); + + // The untampered envelope still decrypts through its recorded version. + let plaintext = client + .decrypt(&integration_decrypt_request(data_key.ciphertext.clone()), None) + .await + .expect("untampered envelope must still decrypt"); + assert_eq!(Some(plaintext), data_key.plaintext); + } + + #[tokio::test] + #[ignore] // Requires a running Vault instance (dev mode) + async fn test_vault_kv2_concurrent_rotate_versions_monotonic() { + use std::sync::Arc; + + let client = Arc::new( + VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + .await + .expect("client"), + ); + + let key_id = format!("rotate-concurrent-{}", uuid::Uuid::new_v4()); + client.create_key(&key_id, "AES_256", None).await.expect("create"); + + let attempts = 4; + let tasks: Vec<_> = (0..attempts) + .map(|_| { + let client = Arc::clone(&client); + let key_id = key_id.clone(); + tokio::spawn(async move { client.rotate_key(&key_id, None).await }) + }) + .collect(); + + let mut successes = 0u32; + for task in tasks { + // Losing a check-and-set race is an expected error; committing is not + // required, but every commit must account for exactly one version bump. + if task.await.expect("join rotate task").is_ok() { + successes += 1; + } + } + assert!(successes >= 1, "at least one rotation must commit"); + + let key_data = client.get_key_data(&key_id).await.expect("read"); + assert_eq!( + key_data.version, + 1 + successes, + "each successful rotation must commit exactly one new version" + ); + assert_eq!(key_data.baseline_version, Some(1)); + + // Every version has an immutable record with unique material, and the + // top-level fast-path copy matches the current version's record. + let mut materials = std::collections::HashSet::new(); + for version in 1..=key_data.version { + let record = client + .get_key_version_record(&key_id, version) + .await + .unwrap_or_else(|error| panic!("version {version} must have a record: {error}")); + assert_eq!(record.version, version); + assert!(materials.insert(record.encrypted_key_material), "version materials must be unique"); + } + let current_record = client + .get_key_version_record(&key_id, key_data.version) + .await + .expect("current version record"); + assert_eq!(current_record.encrypted_key_material, key_data.encrypted_key_material); } #[tokio::test] @@ -991,8 +1518,9 @@ mod tests { client.store_key_data(&key_id, &key_data).await.expect("store corrupt"); // Reading the material must now ERROR, not silently regenerate + overwrite. + let poisoned = client.get_key_data(&key_id).await.expect("read poisoned"); let error = client - .get_key_material(&key_id) + .get_key_material_for_version(&key_id, &poisoned, poisoned.version) .await .expect_err("corrupted key material must yield an error, not a fresh key"); assert!( @@ -1004,7 +1532,7 @@ mod tests { let after = client.get_key_data(&key_id).await.expect("reread"); assert_eq!( after.encrypted_key_material, "!!!not-base64!!!", - "get_key_material must not overwrite stored master key material on failure" + "the material read path must not overwrite stored master key material on failure" ); } @@ -1026,8 +1554,9 @@ mod tests { key_data.encrypted_key_material = String::new(); client.store_key_data(&key_id, &key_data).await.expect("store empty"); + let poisoned = client.get_key_data(&key_id).await.expect("read poisoned"); let error = client - .get_key_material(&key_id) + .get_key_material_for_version(&key_id, &poisoned, poisoned.version) .await .expect_err("empty key material must yield an error, not a fresh key"); assert!( @@ -1039,7 +1568,7 @@ mod tests { let after = client.get_key_data(&key_id).await.expect("reread"); assert!( after.encrypted_key_material.is_empty(), - "get_key_material must not backfill missing master key material" + "the material read path must not backfill missing master key material" ); } From 7051a5ce41ec567b1f0431fb799e6c5e4028de0b Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Fri, 31 Jul 2026 01:49:19 +0800 Subject: [PATCH 24/52] feat: add opt-in hotpath profiling (#5488) * feat: add opt-in hotpath profiling Co-Authored-By: heihutu <heihutu@gmail.com> * test: fix vault kms client construction Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> --- Cargo.lock | 39 ++++++++++++++++++++ Cargo.toml | 2 +- crates/ecstore/Cargo.toml | 18 ++++++++- crates/ecstore/src/disk/local.rs | 4 +- crates/ecstore/src/erasure/coding/bitrot.rs | 6 +-- crates/ecstore/src/erasure/coding/decode.rs | 4 +- crates/ecstore/src/erasure/coding/encode.rs | 8 ++-- crates/ecstore/src/erasure/coding/erasure.rs | 10 ++--- crates/ecstore/src/lib.rs | 2 +- crates/ecstore/src/set_disk/read.rs | 14 +++---- crates/ecstore/src/store/multipart.rs | 2 +- crates/ecstore/src/store/object.rs | 4 +- crates/filemeta/Cargo.toml | 5 ++- crates/filemeta/src/filemeta/codec.rs | 6 +-- crates/rio/Cargo.toml | 5 ++- rustfs/Cargo.toml | 15 +++++++- rustfs/src/main.rs | 19 +++------- rustfs/src/startup_entrypoint.rs | 2 + 18 files changed, 113 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0acca68ae..f5bd0f4e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4916,19 +4916,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66750a77f4f6b408a148be5102ef1f3ba7172def7ee92b1cfc75d9f7a3870453" dependencies = [ "arc-swap", + "async-channel", + "async-trait", "cfg-if", "crossbeam-channel", + "futures-channel", "futures-util", "hdrhistogram", "hotpath-macros", + "hotpath-meta", + "http 1.5.0", "libc", + "parking_lot", "pin-project-lite", "prettytable-rs", "quanta", "regex", + "reqwest", + "reqwest-middleware", "serde", "serde_json", "tiny_http", + "tokio", ] [[package]] @@ -4942,6 +4951,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "hotpath-macros-meta" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21f2f70b29b6f42311acd2fb0b2f91a34d9a573c76fb8a7f51970670a1673a49" + +[[package]] +name = "hotpath-meta" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b771f77b2f409086bb40ff029f850ce99d17118d4e207df0f28cb32bd7568c7" +dependencies = [ + "hotpath-macros-meta", +] + [[package]] name = "htmlescape" version = "0.3.1" @@ -8514,6 +8538,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http 1.5.0", + "reqwest", + "serde", + "thiserror 2.0.19", + "tower-service", +] + [[package]] name = "resolv-conf" version = "0.7.6" diff --git a/Cargo.toml b/Cargo.toml index 34a4b6327..1b5bbe8b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -348,7 +348,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling mimalloc = "0.1.52" -hotpath = "0.22.0" +hotpath = { version = "0.22.0", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 78186b52a..3b6ce2ce8 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -33,14 +33,28 @@ workspace = true [features] default = [] rio-v2 = ["dep:rustfs-rio-v2"] -hotpath = ["dep:hotpath", "hotpath/hotpath", "rustfs-filemeta/hotpath", "rustfs-rio/hotpath"] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/async-channel", + "hotpath/parking_lot", + "hotpath/reqwest-0-13", + "rustfs-filemeta/hotpath", + "rustfs-rio/hotpath", +] +hotpath-alloc = [ + "hotpath/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", + "rustfs-rio/hotpath-alloc", +] # Exposes shared lifecycle/tier test utilities (MockWarmBackend, fault # injection, xl.meta transition assertions) via `api::tier::test_util`. # Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6). test-util = [] [dependencies] -hotpath = { workspace = true, optional = true } +hotpath.workspace = true rustfs-filemeta.workspace = true rustfs-utils = { workspace = true, features = ["full"] } rustfs-rio.workspace = true diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 6dc8a829a..6d309e4f7 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -5078,7 +5078,7 @@ impl LocalDisk { Ok((buf, mtime)) } - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> { check_path_length(file_path.as_ref().to_string_lossy().as_ref())?; @@ -5121,7 +5121,7 @@ impl LocalDisk { Ok((data, modtime)) } - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> { // TODO: timeout support let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?; diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 41705912b..006c9dea3 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -103,7 +103,7 @@ where /// or `out` is larger than one shard. On error `out`'s contents are /// unspecified but never contain bytes that failed the hash check — the copy /// happens only after verification. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> { let want = out.len(); self.begin_read(want)?; @@ -303,7 +303,7 @@ where /// Write a (hash+data) block. Returns the number of data bytes written. /// Returns an error if called after a short write or if data exceeds shard_size. - #[cfg_attr(feature = "hotpath", hotpath::measure(label = "BitrotWriter::write"))] + #[hotpath::measure(label = "BitrotWriter::write")] pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { if buf.is_empty() { return Ok(0); @@ -455,7 +455,7 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith /// stores those as whole-file bitrot with no interleaved hash, so the size guard /// on the next line would reject a genuinely healthy part. Reading legacy V1 /// whole-file-bitrot objects would need a separate verification path. -#[cfg_attr(feature = "hotpath", hotpath::measure)] +#[hotpath::measure] pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>( mut r: R, want_size: usize, diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index 7dc3af4d8..d073eea5c 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -691,7 +691,7 @@ impl<R> ParallelReader<R> where R: crate::erasure::coding::ShardSource, { - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) { // On the reconstruction-verifying GET path, read every live shard reader // in lockstep so all readers advance one block per stripe and stay @@ -1505,7 +1505,7 @@ where } impl Erasure { - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn decode<W, R>( &self, writer: &mut W, diff --git a/crates/ecstore/src/erasure/coding/encode.rs b/crates/ecstore/src/erasure/coding/encode.rs index 117f55da8..c4414723c 100644 --- a/crates/ecstore/src/erasure/coding/encode.rs +++ b/crates/ecstore/src/erasure/coding/encode.rs @@ -504,7 +504,7 @@ impl Erasure { Ok((reader, total)) } - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn encode<R>( self: Arc<Self>, reader: R, @@ -670,7 +670,7 @@ impl Erasure { Ok((reader, total)) } - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn encode_batched<R>( self: Arc<Self>, mut reader: R, @@ -798,7 +798,7 @@ impl Erasure { /// Fast path for small inline objects: skip tokio::spawn + mpsc channel. /// Reads all data, encodes directly, writes shards sequentially. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn encode_inline_small<R>( self: Arc<Self>, reader: R, @@ -813,7 +813,7 @@ impl Erasure { /// Fast path for single-block non-inline objects: avoids the producer/consumer /// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn encode_single_block_non_inline<R>( self: Arc<Self>, reader: R, diff --git a/crates/ecstore/src/erasure/coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs index 98a9d2003..5607a2302 100644 --- a/crates/ecstore/src/erasure/coding/erasure.rs +++ b/crates/ecstore/src/erasure/coding/erasure.rs @@ -640,7 +640,7 @@ impl Erasure { /// # Returns /// A vector of encoded shards as `Bytes`. #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> { let shard_size_fn = if self.uses_legacy { calc_shard_size_legacy @@ -688,7 +688,7 @@ impl Erasure { /// Encode owned data, avoiding a copy when the caller already has a heap buffer. /// Falls back to copying into a new buffer if zero-copy conversion fails. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> { let shard_size_fn = if self.uses_legacy { calc_shard_size_legacy @@ -752,7 +752,7 @@ impl Erasure { /// block), the `resize(need_total_size)` below stays within capacity for every /// `data_len <= block_size` — both shard-size formulas are monotone in /// `data_len` — so this function never reallocates the buffer. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> { let shard_size_fn = if self.uses_legacy { calc_shard_size_legacy @@ -805,7 +805,7 @@ impl Erasure { /// /// # Returns /// Ok if reconstruction succeeds, error otherwise. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> { if self.parity_shards > 0 { if self.uses_legacy { @@ -825,7 +825,7 @@ impl Erasure { } /// Decode and reconstruct missing data shards, then regenerate parity shards. - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> { if self.parity_shards > 0 { if self.uses_legacy { diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 47d25a03e..e0c7d7b6d 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. /// Scope-based hotpath measurement for `#[async_trait]` methods, where -/// `#[cfg_attr(feature = "hotpath", hotpath::measure)]` would only time the boxed-future construction. +/// `#[hotpath::measure]` would only time the boxed-future construction. /// The guard records wall time from this statement until the enclosing /// (desugared) async block completes, including early returns via `?`. #[cfg(feature = "hotpath")] diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index ebd4a7a60..aaec15180 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -199,7 +199,7 @@ impl SetDisks { ); } - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub async fn read_version_optimized( &self, bucket: &str, @@ -238,7 +238,7 @@ impl SetDisks { } #[tracing::instrument(level = "debug", skip(self))] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn get_object_fileinfo( &self, bucket: &str, @@ -410,7 +410,7 @@ impl SetDisks { Ok((fi, parts_metadata, op_online_disks)) } - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn get_object_info_and_quorum( &self, bucket: &str, @@ -605,7 +605,7 @@ impl SetDisks { } #[allow(clippy::too_many_arguments)] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn get_object_with_fileinfo<W>( // &self, bucket: &str, @@ -1140,7 +1140,7 @@ impl SetDisks { } #[allow(clippy::too_many_arguments)] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn get_object_decode_reader_with_fileinfo( bucket: &str, object: &str, @@ -1296,7 +1296,7 @@ impl SetDisks { } #[allow(clippy::too_many_arguments)] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] async fn build_codec_streaming_part_reader( bucket: &str, object: &str, @@ -1469,7 +1469,7 @@ fn multipart_part_checksum_algo(fi: &FileInfo, part_number: usize) -> HashAlgori /// `get_object_with_fileinfo` (backlog#870) so both report the same /// stage-duration semantics. #[allow(clippy::too_many_arguments)] -#[cfg_attr(feature = "hotpath", hotpath::measure)] +#[hotpath::measure] async fn setup_multipart_part_readers( files: &[FileInfo], disks: &[Option<DiskStore>], diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index 65f0128a3..32b21c11e 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -238,7 +238,7 @@ impl ECStore { } #[instrument(skip(self, data))] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn handle_put_object_part( &self, bucket: &str, diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 159fbafa3..6bfa27ed6 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -815,7 +815,7 @@ impl ECStore { } #[instrument(level = "debug", skip(self))] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn handle_get_object_reader( &self, bucket: &str, @@ -849,7 +849,7 @@ impl ECStore { } #[instrument(level = "debug", skip(self, data))] - #[cfg_attr(feature = "hotpath", hotpath::measure)] + #[hotpath::measure] pub(super) async fn handle_put_object( &self, bucket: &str, diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index f19c63ed8..3fdea23e0 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -27,10 +27,11 @@ documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/" [features] default = [] -hotpath = ["dep:hotpath", "hotpath/hotpath"] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath/hotpath-alloc"] [dependencies] -hotpath = { workspace = true, optional = true } +hotpath.workspace = true crc-fast = { workspace = true } rmp.workspace = true rmp-serde.workspace = true diff --git a/crates/filemeta/src/filemeta/codec.rs b/crates/filemeta/src/filemeta/codec.rs index bf3529bfa..0a5a39c6b 100644 --- a/crates/filemeta/src/filemeta/codec.rs +++ b/crates/filemeta/src/filemeta/codec.rs @@ -19,7 +19,7 @@ impl FileMeta { !matches!(Self::check_xl2_v1(buf), Err(_e)) } - #[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "FileMeta"))] + #[hotpath::measure(impl_type = "FileMeta")] pub fn load(buf: &[u8]) -> Result<FileMeta> { let mut xl = FileMeta::default(); xl.unmarshal_msg(buf)?; @@ -112,7 +112,7 @@ impl FileMeta { Ok((bin_len, remaining)) } - #[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "FileMeta"))] + #[hotpath::measure(impl_type = "FileMeta")] pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result<u64> { let i = buf.len() as u64; @@ -326,7 +326,7 @@ impl FileMeta { } } - #[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "FileMeta"))] + #[hotpath::measure(impl_type = "FileMeta")] pub fn marshal_msg(&self) -> Result<Vec<u8>> { let mut wr = Vec::new(); diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index f5745ec04..c0b7b52d6 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -30,10 +30,11 @@ workspace = true [features] default = [] -hotpath = ["dep:hotpath", "hotpath/hotpath"] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "hotpath/futures", "hotpath/reqwest-0-13"] +hotpath-alloc = ["hotpath/hotpath-alloc"] [dependencies] -hotpath = { workspace = true, optional = true } +hotpath.workspace = true arc-swap.workspace = true tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] } rand = { workspace = true, features = ["serde"] } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index ca5cff296..cbef7e4f0 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -53,18 +53,29 @@ pyroscope = ["rustfs-obs/pyroscope"] # Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`. dial9 = ["rustfs-obs/dial9"] hotpath = [ - "dep:hotpath", "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/async-channel", + "hotpath/crossbeam", + "hotpath/parking_lot", + "hotpath/reqwest-0-13", "rustfs-ecstore/hotpath", "rustfs-filemeta/hotpath", "rustfs-rio/hotpath", ] +hotpath-alloc = [ + "hotpath/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", + "rustfs-rio/hotpath-alloc", +] [lints] workspace = true [dependencies] -hotpath = { workspace = true, optional = true } +hotpath.workspace = true # RustFS Internal Crates rustfs-heal = { workspace = true } rustfs-audit = { workspace = true } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 43edb07a7..74d652be3 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -12,23 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +#[global_allocator] +static GLOBAL: hotpath::CountingAllocator = hotpath::CountingAllocator::new(); + +#[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))] #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; fn main() { - // Wall-time profiling guard for `--features hotpath` builds. The guard - // lives for the whole process; dropping it on normal exit (SIGINT/SIGTERM - // unwind back through run_process) prints the timing report. - // - // hotpath's alloc mode is intentionally NOT wired up: hotpath 0.21.x - // panics when a MeasurementGuardSync drops on a different thread than it - // was created on (TLS index mismatch, alloc/core.rs:151), which tokio's - // work-stealing runtime does constantly. See rustfs/backlog#935. - #[cfg(feature = "hotpath")] - let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main") - .percentiles(&[50.0, 95.0, 99.0]) - .functions_limit(0) - .build(); + let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build(); rustfs::startup_entrypoint::run_process(); } diff --git a/rustfs/src/startup_entrypoint.rs b/rustfs/src/startup_entrypoint.rs index 55c97645e..26b5c73a9 100644 --- a/rustfs/src/startup_entrypoint.rs +++ b/rustfs/src/startup_entrypoint.rs @@ -62,6 +62,8 @@ fn emit_fatal_stderr(context: &str, error: impl std::fmt::Display) { } async fn async_main() -> Result<()> { + hotpath::tokio_runtime!(); + let env_compat_report = bootstrap_external_prefix_compat()?; // Parse command line arguments From b457c6abcc91cd0dbe9dfcde8a6dbef117b8b78e Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 01:49:21 +0800 Subject: [PATCH 25/52] feat(kms): add backup manifest and responsibility contract types (#5483) Contract-only module for KMS backup/restore (no handler or backend wiring): versioned manifest schema with completeness marker and sealed digest, the (backend, at-rest protection) responsibility matrix, typed fail-closed errors, and the zero-write restore dry-run report. Fields whose shape depends on in-flight contracts are reserved and reject data in format version 1. --- crates/kms/src/backends/local.rs | 17 +- crates/kms/src/backup/capability.rs | 233 ++++++ crates/kms/src/backup/dry_run.rs | 280 +++++++ crates/kms/src/backup/error.rs | 133 +++ crates/kms/src/backup/manifest.rs | 1165 +++++++++++++++++++++++++++ crates/kms/src/backup/mod.rs | 63 ++ crates/kms/src/error.rs | 4 + crates/kms/src/lib.rs | 1 + 8 files changed, 1890 insertions(+), 6 deletions(-) create mode 100644 crates/kms/src/backup/capability.rs create mode 100644 crates/kms/src/backup/dry_run.rs create mode 100644 crates/kms/src/backup/error.rs create mode 100644 crates/kms/src/backup/manifest.rs create mode 100644 crates/kms/src/backup/mod.rs diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 38c80477d..1cbad8a1f 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -69,11 +69,14 @@ fn validate_key_id(key_id: &str) -> Result<()> { } const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt"; -const LOCAL_KMS_MASTER_KEY_SALT_LEN: usize = 16; -const LOCAL_KMS_MASTER_KEY_LEN: usize = 32; -const LOCAL_KMS_ARGON2_M_COST_KIB: u32 = 19 * 1024; -const LOCAL_KMS_ARGON2_T_COST: u32 = 2; -const LOCAL_KMS_ARGON2_P_COST: u32 = 1; +// The KDF parameters are pub(crate) so the backup manifest contract +// (`crate::backup`) records the exact compiled-in derivation instead of a +// copy that could drift. +pub(crate) const LOCAL_KMS_MASTER_KEY_SALT_LEN: usize = 16; +pub(crate) const LOCAL_KMS_MASTER_KEY_LEN: usize = 32; +pub(crate) const LOCAL_KMS_ARGON2_M_COST_KIB: u32 = 19 * 1024; +pub(crate) const LOCAL_KMS_ARGON2_T_COST: u32 = 2; +pub(crate) const LOCAL_KMS_ARGON2_P_COST: u32 = 1; /// Strict matcher for leftover commit temp files (`<prefix>.tmp-<uuid>`). /// @@ -396,9 +399,11 @@ pub struct LocalKmsClient { key_write_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>, } +// pub(crate) so the backup contract tests can anchor the manifest's +// protection-state wire names against the marker values written to disk. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] -enum StoredKeyProtection { +pub(crate) enum StoredKeyProtection { #[default] LegacyUnspecified, EncryptedMasterKey, diff --git a/crates/kms/src/backup/capability.rs b/crates/kms/src/backup/capability.rs new file mode 100644 index 000000000..97ea531bc --- /dev/null +++ b/crates/kms/src/backup/capability.rs @@ -0,0 +1,233 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Backup responsibility matrix: what a RustFS backup bundle owns per backend. +//! +//! The matrix is two-dimensional on purpose: responsibility is a function of +//! the backend *and* its at-rest protection state, not of the backend alone. +//! This keeps the schema stable when a backend changes protection direction — +//! switching Vault KV2 between storage-only and Transit-wrapped operation +//! selects a different existing row instead of changing the contract. +//! +//! These enums are backup-domain contract types. Once the backlog#1571 +//! capability-discovery contract lands, the discovery surface is expected to +//! converge on (or map onto) the states defined here. + +use serde::{Deserialize, Serialize}; + +/// Backend discriminant recorded in a backup manifest. +/// +/// Wire names are aligned with [`crate::config::BackendConfig`] and +/// [`crate::config::KmsBackend`] (including the legacy `Vault` alias) so that +/// a manifest and a persisted KMS configuration never disagree about how the +/// same backend is spelled. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum BackupBackendKind { + /// Local file-based backend. + Local, + /// Vault KV v2 storage backend. + #[serde(rename = "VaultKV2", alias = "Vault")] + VaultKv2, + /// Vault Transit backend. + VaultTransit, + /// Static single-key backend. + Static, +} + +/// At-rest protection state of master key material, as observed at snapshot +/// time. +/// +/// The first three states mirror the Local backend's on-disk protection +/// marker (`StoredKeyProtection` in `backends/local.rs`, kebab-case wire +/// names). The remaining states describe the non-local backends. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AtRestProtection { + /// Local key files AEAD-encrypted under the Argon2id-derived master key. + EncryptedMasterKey, + /// Local development-only plaintext key files. Such files must never + /// enter a bundle as-is; bundle artifacts are always re-wrapped under the + /// backup KEK. + PlaintextDevOnly, + /// Pre-beta.9 local key files without a protection marker; the effective + /// mode is resolved at read time. Treated like [`Self::PlaintextDevOnly`] + /// for bundling purposes: re-wrap is mandatory. + LegacyUnspecified, + /// Vault KV2 as currently shipped: material confidentiality relies on + /// Vault ACLs, KV2 at-rest encryption, and TLS only (the backend reports + /// `at_rest_protection = "vault-kv2-acl"`); RustFS applies no + /// cryptographic wrapping of its own. + StorageOnly, + /// Vault KV2 with material wrapped by Vault Transit before storage. Not + /// produced by any current backend; the row exists so a future direction + /// change selects a state instead of changing the schema. + TransitWrapped, + /// Vault Transit: the cryptographic root lives in Vault and is not + /// exportable. RustFS can only ever own metadata and references. + ExternalNonExportable, + /// Static backend: the secret is delivered externally at startup and + /// RustFS persists no key material at all. + ExternalSecretDelivery, +} + +/// What a RustFS backup bundle is responsible for, per (backend, protection) +/// combination. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum BackupResponsibility { + /// The bundle carries the complete recoverable state: encrypted key + /// material for every version, salt, metadata, and configuration. + /// + /// Restore precondition for the Local backend: the operator re-supplies + /// the master key out of band. The master key itself is outside the + /// backup domain; the manifest stores at most an opaque verifier. + FullMaterial, + /// The bundle carries only non-sensitive references and verification + /// information. The source of truth is external secret delivery and the + /// operator re-provides the secret during restore. Embedding the secret + /// itself in a bundle is forbidden. + ReferenceOnly, + /// The bundle carries RustFS-side metadata, configuration references and + /// verification data, while the cryptographic root is protected by the + /// external system's native snapshot/disaster-recovery flow (Vault/HSM). + /// Restore must re-establish the external trust root first. + MetadataPlusExternalRoot, +} + +impl BackupResponsibility { + /// Resolve the responsibility matrix for one (backend, protection) cell. + /// + /// Returns `None` for combinations that no supported deployment can + /// produce; manifests declaring such a combination are rejected as + /// corrupted. This function is total and the unit tests anchor every + /// cell, so any change to the matrix is a deliberate contract change. + pub fn for_backend(backend: BackupBackendKind, protection: AtRestProtection) -> Option<Self> { + use AtRestProtection::*; + use BackupBackendKind::*; + + match (backend, protection) { + (Local, EncryptedMasterKey | PlaintextDevOnly | LegacyUnspecified) => Some(Self::FullMaterial), + (Local, _) => None, + // Storage-only KV2 offers no external cryptographic root, so the + // bundle must own the material (re-wrapped under the backup KEK). + (VaultKv2, StorageOnly) => Some(Self::FullMaterial), + (VaultKv2, TransitWrapped) => Some(Self::MetadataPlusExternalRoot), + (VaultKv2, _) => None, + (VaultTransit, ExternalNonExportable) => Some(Self::MetadataPlusExternalRoot), + (VaultTransit, _) => None, + (Static, ExternalSecretDelivery) => Some(Self::ReferenceOnly), + (Static, _) => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::KmsBackend; + + fn json<T: Serialize>(value: &T) -> String { + serde_json::to_string(value).expect("serialization should succeed") + } + + #[test] + fn backend_kind_wire_names_match_kms_backend() { + let pairs = [ + (BackupBackendKind::Local, KmsBackend::Local), + (BackupBackendKind::VaultKv2, KmsBackend::VaultKv2), + (BackupBackendKind::VaultTransit, KmsBackend::VaultTransit), + (BackupBackendKind::Static, KmsBackend::Static), + ]; + for (backup_kind, config_kind) in pairs { + assert_eq!(json(&backup_kind), json(&config_kind), "wire name drifted for {backup_kind:?}"); + } + } + + #[test] + fn backend_kind_accepts_legacy_vault_alias() { + let decoded: BackupBackendKind = serde_json::from_str("\"Vault\"").expect("legacy alias should decode"); + assert_eq!(decoded, BackupBackendKind::VaultKv2); + } + + #[test] + fn responsibility_matrix_is_anchored_cell_by_cell() { + use AtRestProtection::*; + use BackupBackendKind::*; + use BackupResponsibility::*; + + // Every (backend, protection) cell, exhaustively. Changing any row is + // a contract change and must be made here consciously. + let matrix = [ + (Local, EncryptedMasterKey, Some(FullMaterial)), + (Local, PlaintextDevOnly, Some(FullMaterial)), + (Local, LegacyUnspecified, Some(FullMaterial)), + (Local, StorageOnly, None), + (Local, TransitWrapped, None), + (Local, ExternalNonExportable, None), + (Local, ExternalSecretDelivery, None), + (VaultKv2, EncryptedMasterKey, None), + (VaultKv2, PlaintextDevOnly, None), + (VaultKv2, LegacyUnspecified, None), + (VaultKv2, StorageOnly, Some(FullMaterial)), + (VaultKv2, TransitWrapped, Some(MetadataPlusExternalRoot)), + (VaultKv2, ExternalNonExportable, None), + (VaultKv2, ExternalSecretDelivery, None), + (VaultTransit, EncryptedMasterKey, None), + (VaultTransit, PlaintextDevOnly, None), + (VaultTransit, LegacyUnspecified, None), + (VaultTransit, StorageOnly, None), + (VaultTransit, TransitWrapped, None), + (VaultTransit, ExternalNonExportable, Some(MetadataPlusExternalRoot)), + (VaultTransit, ExternalSecretDelivery, None), + (Static, EncryptedMasterKey, None), + (Static, PlaintextDevOnly, None), + (Static, LegacyUnspecified, None), + (Static, StorageOnly, None), + (Static, TransitWrapped, None), + (Static, ExternalNonExportable, None), + (Static, ExternalSecretDelivery, Some(ReferenceOnly)), + ]; + assert_eq!(matrix.len(), 28, "matrix must stay exhaustive: 4 backends x 7 protection states"); + for (backend, protection, expected) in matrix { + assert_eq!( + BackupResponsibility::for_backend(backend, protection), + expected, + "matrix cell drifted for ({backend:?}, {protection:?})" + ); + } + } + + #[test] + fn local_protection_wire_names_match_stored_key_protection() { + use crate::backends::local::StoredKeyProtection; + + // The manifest must record exactly the marker values the Local + // backend writes to disk, or a restore could misread protection. + let pairs = [ + (AtRestProtection::EncryptedMasterKey, StoredKeyProtection::EncryptedMasterKey), + (AtRestProtection::PlaintextDevOnly, StoredKeyProtection::PlaintextDevOnly), + (AtRestProtection::LegacyUnspecified, StoredKeyProtection::LegacyUnspecified), + ]; + for (backup_state, stored_state) in pairs { + assert_eq!(json(&backup_state), json(&stored_state), "wire name drifted for {backup_state:?}"); + } + } + + #[test] + fn responsibility_wire_names_are_frozen() { + assert_eq!(json(&BackupResponsibility::FullMaterial), "\"full-material\""); + assert_eq!(json(&BackupResponsibility::ReferenceOnly), "\"reference-only\""); + assert_eq!(json(&BackupResponsibility::MetadataPlusExternalRoot), "\"metadata-plus-external-root\""); + } +} diff --git a/crates/kms/src/backup/dry_run.rs b/crates/kms/src/backup/dry_run.rs new file mode 100644 index 000000000..54bd7e42a --- /dev/null +++ b/crates/kms/src/backup/dry_run.rs @@ -0,0 +1,280 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Restore dry-run report contract. +//! +//! A restore dry-run is a zero-write preflight: it evaluates a bundle against +//! a target and reports every blocker, conflict, and external dependency +//! mismatch without modifying the target in any way. The report itself is +//! plain data — producing, serializing, or discarding it has no side effects, +//! and an implementation that writes anything during a dry-run violates this +//! contract. All values in a report are identifiers and references; secrets, +//! tokens, and key material never appear in it. + +use crate::backup::error::BackupError; +use serde::{Deserialize, Serialize}; + +/// Machine-readable category of a restore blocker. +/// +/// The first six codes mirror the [`BackupError`] variants; the remaining +/// codes cover preflight conditions that are not bundle defects. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RestoreBlockerCode { + /// The bundle failed structural or integrity validation. + BundleCorrupted, + /// The manifest input ended prematurely. + BundleTruncated, + /// The manifest format version is unknown to this build. + UnknownFormatVersion, + /// The supplied backup KEK does not match the bundle's KEK. + WrongBackupKek, + /// A required artifact is absent from the bundle. + MissingArtifact, + /// The bundle has no completeness marker or is marked in-progress. + IncompleteBundle, + /// The target backend cannot satisfy the bundle's responsibility model. + UnsupportedBackend, + /// The bundle was produced by a different deployment than the target and + /// no explicit cross-deployment authorization applies. + DeploymentMismatch, + /// An external dependency (Vault cluster, mount, Transit key, ...) that + /// the bundle references is unreachable or missing. + ExternalDependencyUnavailable, +} + +/// One condition that forbids the restore outright. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreBlocker { + /// Machine-readable category. + pub code: RestoreBlockerCode, + /// Human-readable detail. Identifiers only; never secrets or material. + pub detail: String, +} + +impl From<&BackupError> for RestoreBlocker { + fn from(error: &BackupError) -> Self { + let code = match error { + BackupError::Corrupted { .. } => RestoreBlockerCode::BundleCorrupted, + BackupError::Truncated { .. } => RestoreBlockerCode::BundleTruncated, + BackupError::UnknownVersion { .. } => RestoreBlockerCode::UnknownFormatVersion, + BackupError::WrongKek { .. } => RestoreBlockerCode::WrongBackupKek, + BackupError::MissingArtifact { .. } => RestoreBlockerCode::MissingArtifact, + BackupError::IncompleteBundle { .. } => RestoreBlockerCode::IncompleteBundle, + }; + Self { + code, + detail: error.to_string(), + } + } +} + +/// Kind of a conflict between bundle state and existing target state. +/// +/// Restore is non-destructive by default: every conflict blocks the restore +/// unless an explicit, audited conflict policy resolves it. Silent overwrite +/// or merge is never an option. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RestoreConflictKind { + /// The target already has a key with this stable id. + KeyAlreadyExists, + /// Restoring would lower a key version the target has already observed. + VersionRegression, + /// Restoring would lower the snapshot generation the target has already + /// observed. + GenerationRegression, + /// Restoring would revive a key the target has deleted or scheduled for + /// deletion. + StateRegression, +} + +/// One conflict with existing target state. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreConflict { + /// Stable key id the conflict concerns. + pub key_id: String, + /// Machine-readable category. + pub kind: RestoreConflictKind, + /// Human-readable detail. Identifiers only; never secrets or material. + pub detail: String, +} + +/// A mismatch between an external dependency reference recorded in the bundle +/// and what the target environment observes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExternalDependencyMismatch { + /// Which dependency is affected (for example a Vault mount or Transit + /// key name). References only; never credentials. + pub dependency: String, + /// The value the bundle recorded. + pub expected: String, + /// The value the target environment reports. + pub observed: String, +} + +/// Result of a restore dry-run preflight. +/// +/// # Zero-write contract +/// +/// A dry-run must not write to the target: no staging directories, no +/// repaired records, no metadata fixes triggered along the read path. The +/// report is pure data over values already known to the caller. +/// +/// ``` +/// use rustfs_kms::backup::{RestoreBlocker, RestoreBlockerCode, RestoreDryRunReport}; +/// +/// let clean = RestoreDryRunReport { +/// backup_id: "backup-0001".to_string(), +/// target_deployment_identity: "deployment-a".to_string(), +/// blockers: Vec::new(), +/// conflicts: Vec::new(), +/// external_mismatches: Vec::new(), +/// }; +/// assert!(clean.restore_permitted()); +/// +/// let blocked = RestoreDryRunReport { +/// blockers: vec![RestoreBlocker { +/// code: RestoreBlockerCode::IncompleteBundle, +/// detail: "manifest has no completeness marker".to_string(), +/// }], +/// ..clean +/// }; +/// assert!(!blocked.restore_permitted()); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreDryRunReport { + /// Identifier of the evaluated bundle. + pub backup_id: String, + /// Identity of the restore target the bundle was evaluated against. + pub target_deployment_identity: String, + /// Conditions that forbid the restore outright. + pub blockers: Vec<RestoreBlocker>, + /// Conflicts with existing target state. + pub conflicts: Vec<RestoreConflict>, + /// External dependency mismatches. + pub external_mismatches: Vec<ExternalDependencyMismatch>, +} + +impl RestoreDryRunReport { + /// Whether the restore may proceed: true only when the preflight found + /// no blockers, no conflicts, and no external dependency mismatches. + pub fn restore_permitted(&self) -> bool { + self.blockers.is_empty() && self.conflicts.is_empty() && self.external_mismatches.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_report() -> RestoreDryRunReport { + RestoreDryRunReport { + backup_id: "backup-0001".to_string(), + target_deployment_identity: "deployment-b".to_string(), + blockers: vec![RestoreBlocker { + code: RestoreBlockerCode::DeploymentMismatch, + detail: "bundle was produced by deployment-a".to_string(), + }], + conflicts: vec![RestoreConflict { + key_id: "object-key".to_string(), + kind: RestoreConflictKind::VersionRegression, + detail: "target observed version 5, bundle carries version 3".to_string(), + }], + external_mismatches: vec![ExternalDependencyMismatch { + dependency: "vault transit key rustfs-master".to_string(), + expected: "min_version=2".to_string(), + observed: "min_version=4".to_string(), + }], + } + } + + #[test] + fn report_round_trips_through_json() { + let report = sample_report(); + let json = serde_json::to_string(&report).expect("serialization should succeed"); + let decoded: RestoreDryRunReport = serde_json::from_str(&json).expect("deserialization should succeed"); + assert_eq!(decoded, report); + } + + #[test] + fn restore_permitted_requires_every_section_empty() { + assert!(!sample_report().restore_permitted()); + + let clean = RestoreDryRunReport { + blockers: Vec::new(), + conflicts: Vec::new(), + external_mismatches: Vec::new(), + ..sample_report() + }; + assert!(clean.restore_permitted()); + + for section in 0..3 { + let mut report = clean.clone(); + match section { + 0 => report.blockers = sample_report().blockers, + 1 => report.conflicts = sample_report().conflicts, + _ => report.external_mismatches = sample_report().external_mismatches, + } + assert!(!report.restore_permitted(), "section {section} alone must block the restore"); + } + } + + #[test] + fn every_backup_error_maps_to_a_blocker_code() { + let cases = [ + (BackupError::corrupted("x"), RestoreBlockerCode::BundleCorrupted), + (BackupError::truncated("x"), RestoreBlockerCode::BundleTruncated), + ( + BackupError::UnknownVersion { found: 2, supported: 1 }, + RestoreBlockerCode::UnknownFormatVersion, + ), + ( + BackupError::WrongKek { + required_kek_id: "a".to_string(), + required_kek_version: 1, + supplied_kek_id: "b".to_string(), + supplied_kek_version: 1, + }, + RestoreBlockerCode::WrongBackupKek, + ), + (BackupError::missing_artifact("key-material"), RestoreBlockerCode::MissingArtifact), + (BackupError::incomplete_bundle("x"), RestoreBlockerCode::IncompleteBundle), + ]; + for (error, expected_code) in cases { + let blocker = RestoreBlocker::from(&error); + assert_eq!(blocker.code, expected_code, "wrong code for {error:?}"); + assert_eq!(blocker.detail, error.to_string()); + } + } + + /// The zero-write contract in practice: a report is plain serializable + /// data with no handles, no I/O, and no drop side effects. + #[test] + fn report_types_are_plain_data() { + fn assert_plain_data<T>() + where + T: serde::Serialize + serde::de::DeserializeOwned + Clone + PartialEq + std::fmt::Debug + Send + Sync + 'static, + { + } + assert_plain_data::<RestoreDryRunReport>(); + assert_plain_data::<RestoreBlocker>(); + assert_plain_data::<RestoreConflict>(); + assert_plain_data::<ExternalDependencyMismatch>(); + } +} diff --git a/crates/kms/src/backup/error.rs b/crates/kms/src/backup/error.rs new file mode 100644 index 000000000..c0967b74a --- /dev/null +++ b/crates/kms/src/backup/error.rs @@ -0,0 +1,133 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Typed failures for the backup/restore bundle contract. + +use thiserror::Error; + +/// Typed failures raised while decoding or validating a backup bundle. +/// +/// Every variant is a fail-closed condition: a restore surface observing any +/// of them must abort before touching target state. Messages carry only +/// identifiers (backup ids, KEK ids, artifact kinds and paths) — never key +/// material, bundle plaintext, or credentials. +#[derive(Error, Debug, Clone, PartialEq, Eq)] +pub enum BackupError { + /// Manifest or bundle content fails structural, schema, or integrity + /// validation (unknown fields, duplicate fields, digest mismatch, + /// contradictory responsibility declarations, ...). + #[error("backup bundle corrupted: {reason}")] + Corrupted { reason: String }, + + /// Input ended before a complete manifest could be decoded. + #[error("backup manifest truncated: {reason}")] + Truncated { reason: String }, + + /// Manifest declares a format version this build does not understand. + /// Unknown versions are always rejected; there is no best-effort read. + #[error("unknown backup manifest format version {found} (this build supports version {supported})")] + UnknownVersion { found: u32, supported: u32 }, + + /// Bundle is protected by a different backup KEK than the one supplied. + #[error( + "backup bundle requires KEK '{required_kek_id}' version {required_kek_version}; \ + supplied KEK '{supplied_kek_id}' version {supplied_kek_version} cannot open it" + )] + WrongKek { + required_kek_id: String, + required_kek_version: u32, + supplied_kek_id: String, + supplied_kek_version: u32, + }, + + /// Manifest requires an artifact that is not present in the bundle. + #[error("backup bundle is missing a required artifact: {artifact}")] + MissingArtifact { artifact: String }, + + /// Bundle has no completeness marker or records an in-progress state. + /// A bundle that never reached its completeness marker must never be + /// restored, regardless of how much of it is readable. + #[error("backup bundle is incomplete ({reason}); incomplete bundles must never be restored")] + IncompleteBundle { reason: String }, +} + +impl BackupError { + /// Create a corrupted-bundle error. + pub fn corrupted<S: Into<String>>(reason: S) -> Self { + Self::Corrupted { reason: reason.into() } + } + + /// Create a truncated-manifest error. + pub fn truncated<S: Into<String>>(reason: S) -> Self { + Self::Truncated { reason: reason.into() } + } + + /// Create an incomplete-bundle error. + pub fn incomplete_bundle<S: Into<String>>(reason: S) -> Self { + Self::IncompleteBundle { reason: reason.into() } + } + + /// Create a missing-artifact error. + pub fn missing_artifact<S: Into<String>>(artifact: S) -> Self { + Self::MissingArtifact { + artifact: artifact.into(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::KmsError; + + #[test] + fn backup_errors_convert_into_kms_error_transparently() { + let error = BackupError::corrupted("manifest digest mismatch"); + let kms_error: KmsError = error.clone().into(); + assert_eq!(kms_error.to_string(), error.to_string()); + assert!(matches!(kms_error, KmsError::Backup(inner) if inner == error)); + } + + #[test] + fn error_messages_carry_identifiers_only() { + // The display strings must stay descriptive without ever embedding + // material or bundle plaintext; each variant only interpolates the + // identifiers below. + let wrong_kek = BackupError::WrongKek { + required_kek_id: "backup-kek-1".to_string(), + required_kek_version: 3, + supplied_kek_id: "backup-kek-2".to_string(), + supplied_kek_version: 1, + }; + assert_eq!( + wrong_kek.to_string(), + "backup bundle requires KEK 'backup-kek-1' version 3; supplied KEK 'backup-kek-2' version 1 cannot open it" + ); + + let unknown = BackupError::UnknownVersion { found: 9, supported: 1 }; + assert_eq!( + unknown.to_string(), + "unknown backup manifest format version 9 (this build supports version 1)" + ); + + assert_eq!( + BackupError::missing_artifact("key-material").to_string(), + "backup bundle is missing a required artifact: key-material" + ); + assert_eq!( + BackupError::incomplete_bundle("manifest has no completeness marker").to_string(), + "backup bundle is incomplete (manifest has no completeness marker); incomplete bundles must never be restored" + ); + } +} diff --git a/crates/kms/src/backup/manifest.rs b/crates/kms/src/backup/manifest.rs new file mode 100644 index 000000000..052642428 --- /dev/null +++ b/crates/kms/src/backup/manifest.rs @@ -0,0 +1,1165 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Versioned backup manifest schema (format version 1). +//! +//! Schema evolution policy: every struct rejects unknown fields, so adding, +//! removing, or reshaping any field requires bumping +//! [`BackupManifest::FORMAT_VERSION`]. Decoders reject unknown versions +//! outright — there is no best-effort read of a manifest this build does not +//! understand. + +use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; +use crate::backup::error::BackupError; +use jiff::Zoned; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::path::{Component, Path}; + +/// AEAD algorithm identifiers for bundle protection. +/// +/// This is deliberately not [`crate::types::EncryptionAlgorithm`]: that enum +/// carries S3-facing wire names and the non-AEAD `aws:kms` marker. The backup +/// domain only ever names a concrete AEAD. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum AeadAlgorithm { + /// AES-256-GCM. + #[serde(rename = "aes-256-gcm")] + Aes256Gcm, + /// ChaCha20-Poly1305. + #[serde(rename = "chacha20-poly1305")] + ChaCha20Poly1305, +} + +/// Digest algorithm identifiers for manifest and artifact digests. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DigestAlgorithm { + /// SHA-256. + #[serde(rename = "sha-256")] + Sha256, +} + +/// A content digest: algorithm plus lowercase hex value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContentDigest { + /// Digest algorithm. + pub algorithm: DigestAlgorithm, + /// Lowercase hex encoding of the digest value. + pub hex: String, +} + +impl ContentDigest { + /// Hex length of a SHA-256 digest. + pub const SHA256_HEX_LEN: usize = 64; + + /// Compute the SHA-256 digest of `bytes`. + pub fn sha256_of(bytes: &[u8]) -> Self { + Self { + algorithm: DigestAlgorithm::Sha256, + hex: hex::encode(Sha256::digest(bytes)), + } + } + + /// The placeholder written into the digest slot while computing the + /// canonical manifest bytes (see [`BackupManifest::compute_digest`]). + fn placeholder(algorithm: DigestAlgorithm) -> Self { + Self { + algorithm, + hex: String::new(), + } + } + + /// Whether the hex value is well-formed for the declared algorithm. + pub fn is_well_formed(&self) -> bool { + match self.algorithm { + DigestAlgorithm::Sha256 => { + self.hex.len() == Self::SHA256_HEX_LEN + && self.hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + } + } + } +} + +/// Identity of the backup KEK protecting a bundle. +/// +/// The backup KEK is a trust root separate from the business KMS hierarchy: +/// a bundle must never be encrypted with a key that is itself part of the +/// state being backed up. The manifest records only the KEK identity — never +/// material that could open the bundle. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BackupKekDescriptor { + /// Stable identifier of the backup KEK. + pub kek_id: String, + /// Version of the backup KEK used for this bundle. + pub kek_version: u32, + /// AEAD algorithm protecting the bundle artifacts. + pub aead_algorithm: AeadAlgorithm, +} + +impl BackupKekDescriptor { + /// Fail closed unless the supplied KEK identity matches the one this + /// bundle was sealed with. + pub fn ensure_matches(&self, supplied_kek_id: &str, supplied_kek_version: u32) -> Result<(), BackupError> { + if self.kek_id != supplied_kek_id || self.kek_version != supplied_kek_version { + return Err(BackupError::WrongKek { + required_kek_id: self.kek_id.clone(), + required_kek_version: self.kek_version, + supplied_kek_id: supplied_kek_id.to_string(), + supplied_kek_version, + }); + } + Ok(()) + } +} + +/// Kind of an artifact referenced by the manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ArtifactKind { + /// Encrypted master key material (all versions of one or more keys). + KeyMaterial, + /// Key metadata records (status, usage, timestamps, tags). + KeyMetadata, + /// The Local backend's persistent master-key KDF salt. + MasterKeySalt, + /// The sanitized KMS configuration. The persisted `KmsConfig` contains + /// plaintext credentials (Vault token / AppRole secret id); the config + /// artifact must be the sanitized form and never those secrets. + KmsConfig, + /// Reserved: no alias feature exists in the codebase. The name is frozen + /// so a future format version can use it; format version 1 manifests + /// containing it are rejected. + Alias, + /// Reserved: key policies are accepted on `CreateKeyRequest` but never + /// consumed. Same rejection rule as [`Self::Alias`]. + Policy, +} + +impl ArtifactKind { + /// Whether this kind is reserved for a future format version. + pub fn is_reserved(&self) -> bool { + matches!(self, Self::Alias | Self::Policy) + } +} + +/// One artifact in the bundle payload set. +/// +/// Every artifact payload is AEAD-encrypted under the backup KEK regardless +/// of how the source material was protected at rest. In particular, +/// plaintext-dev-only Local key files must never enter a bundle unwrapped. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactDescriptor { + /// What the artifact contains. + pub kind: ArtifactKind, + /// Bundle-relative path of the artifact payload. Absolute paths and path + /// traversal are rejected. + pub path: String, + /// Length of the encrypted payload in bytes. + pub len: u64, + /// AEAD algorithm the payload is encrypted with. + pub aead_algorithm: AeadAlgorithm, + /// Digest of the encrypted payload bytes (not of the plaintext, so + /// verification never requires decryption). + pub encrypted_digest: ContentDigest, +} + +/// Key-derivation description for a Local backend bundle. +/// +/// Values mirror the constants in `backends/local.rs`; recording them in the +/// manifest lets a restore detect a KDF parameter drift instead of silently +/// deriving a different master key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub enum LocalKeyDerivation { + /// Argon2id with explicit parameters and a persistent on-disk salt. + Argon2id { + /// Argon2 version (0x13 = 19). + version: u32, + /// Memory cost in KiB. + memory_kib: u32, + /// Iteration count (time cost). + iterations: u32, + /// Lane count (parallelism). + parallelism: u32, + /// Derived key length in bytes. + output_len: u32, + /// Persistent salt length in bytes. + salt_len: u32, + }, + /// Pre-beta.9 SHA-256 derivation without a salt. Recorded so legacy key + /// directories can be restored; new bundles always use Argon2id. + LegacySha256, +} + +impl LocalKeyDerivation { + /// The derivation currently performed by `backends/local.rs` + /// (`derive_master_key`): Argon2id v0x13 with the crate's compiled-in + /// parameters. + pub fn current_argon2id() -> Self { + use crate::backends::local::{ + LOCAL_KMS_ARGON2_M_COST_KIB, LOCAL_KMS_ARGON2_P_COST, LOCAL_KMS_ARGON2_T_COST, LOCAL_KMS_MASTER_KEY_LEN, + LOCAL_KMS_MASTER_KEY_SALT_LEN, + }; + + Self::Argon2id { + version: 0x13, + memory_kib: LOCAL_KMS_ARGON2_M_COST_KIB, + iterations: LOCAL_KMS_ARGON2_T_COST, + parallelism: LOCAL_KMS_ARGON2_P_COST, + output_len: LOCAL_KMS_MASTER_KEY_LEN as u32, + salt_len: LOCAL_KMS_MASTER_KEY_SALT_LEN as u32, + } + } + + fn validate(&self) -> Result<(), BackupError> { + match self { + Self::Argon2id { + version, + memory_kib, + iterations, + parallelism, + output_len, + salt_len, + } => { + if *version == 0 || *memory_kib == 0 || *iterations == 0 || *parallelism == 0 || *salt_len == 0 { + return Err(BackupError::corrupted("local KDF descriptor has zero-valued Argon2 parameters")); + } + if *output_len != 32 { + return Err(BackupError::corrupted("local KDF descriptor must derive a 32-byte key for AES-256")); + } + Ok(()) + } + Self::LegacySha256 => Ok(()), + } + } +} + +/// Local backend section of the manifest. +/// +/// The Local master key itself is outside the backup domain: it is supplied +/// via environment or configuration and the operator must re-supply it before +/// restore. The manifest stores at most an opaque one-way verifier. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalKdfDescriptor { + /// How the master key string is turned into the file-encryption key. + pub derivation: LocalKeyDerivation, + /// Protection states observed across the stored key files in this + /// snapshot (deduplicated). Restricted to the Local rows of the + /// responsibility matrix. + pub protection_modes: Vec<AtRestProtection>, + /// Opaque one-way verifier of the operator-supplied master key, used to + /// detect a wrong master key before restore touches anything. Never key + /// material; the exact derivation is defined by the Local export + /// implementation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub master_key_verifier: Option<String>, +} + +impl LocalKdfDescriptor { + /// Build a descriptor for the current compiled-in derivation. + pub fn current(protection_modes: Vec<AtRestProtection>, master_key_verifier: Option<String>) -> Self { + Self { + derivation: LocalKeyDerivation::current_argon2id(), + protection_modes, + master_key_verifier, + } + } + + fn validate(&self) -> Result<(), BackupError> { + self.derivation.validate()?; + if self.protection_modes.is_empty() { + return Err(BackupError::corrupted("local KDF descriptor lists no protection modes")); + } + let mut seen = BTreeSet::new(); + for mode in &self.protection_modes { + if BackupResponsibility::for_backend(BackupBackendKind::Local, *mode).is_none() { + return Err(BackupError::corrupted(format!( + "local KDF descriptor lists non-local protection mode {mode:?}" + ))); + } + if !seen.insert(format!("{mode:?}")) { + return Err(BackupError::corrupted(format!( + "local KDF descriptor lists protection mode {mode:?} more than once" + ))); + } + } + if self.master_key_verifier.as_deref() == Some("") { + return Err(BackupError::corrupted("local master key verifier must not be empty when present")); + } + Ok(()) + } +} + +/// Completeness marker of a bundle. +/// +/// A producer writes `in-progress` state (or no manifest at all) until the +/// final integrity checks pass, then seals the bundle as `complete`. Anything +/// other than `complete` must never be restored. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum CompletenessState { + /// The bundle never reached its final integrity checks. + InProgress, + /// The bundle passed its final integrity checks and was sealed. + Complete, +} + +/// Placeholder for a manifest field whose shape is intentionally not frozen +/// yet. +/// +/// Reserved fields hold their name in the schema but may not carry data in +/// format version 1: deserializing any non-null value fails, and +/// [`BackupManifest::validate`] rejects a populated slot. A later format +/// version replaces the slot with the real type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct ReservedSlot; + +impl<'de> Deserialize<'de> for ReservedSlot { + fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error> + where + D: serde::Deserializer<'de>, + { + Err(serde::de::Error::custom( + "reserved manifest field must not carry data in format version 1", + )) + } +} + +/// Version probe decoded before the full manifest so that version and +/// completeness failures surface as their own typed errors instead of +/// generic decode errors. +#[derive(Deserialize)] +struct ManifestProbe { + format_version: u32, + #[serde(default)] + completeness: Option<serde_json::Value>, +} + +/// Versioned backup manifest (format version 1). +/// +/// The manifest is the authoritative description of one backup bundle: what +/// was captured, under which snapshot generation, protected by which backup +/// KEK, and which restore responsibility applies. Field order is part of the +/// canonical digest form and is frozen for this format version. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BackupManifest { + /// Manifest format version; see [`Self::FORMAT_VERSION`]. + pub format_version: u32, + /// Unique identifier of this backup. + pub backup_id: String, + /// Creation time of the bundle. + #[serde(with = "crate::time_serde::zoned")] + pub created_at: Zoned, + /// RustFS version that produced the bundle. + pub rustfs_version: String, + /// Opaque identity of the producing deployment, used by restore preflight + /// to detect cross-deployment restores. The value source is decided by + /// the producing implementation; the contract freezes only that it is an + /// opaque non-empty string. + pub deployment_identity: String, + /// Backend the bundle was captured from. + pub backend: BackupBackendKind, + /// At-rest protection state observed at snapshot time. + pub at_rest_protection: AtRestProtection, + /// Declared restore responsibility. Must equal the responsibility matrix + /// row for `(backend, at_rest_protection)`; a mismatch is rejected. + pub responsibility: BackupResponsibility, + /// Monotonic snapshot generation. All state in one bundle belongs to this + /// single generation; restore preflight uses it to block rollbacks onto a + /// target that has already observed a higher generation. + pub snapshot_generation: u64, + /// Identity of the backup KEK protecting the bundle. + pub backup_kek: BackupKekDescriptor, + /// The bundle payload set. + pub artifacts: Vec<ArtifactDescriptor>, + /// Local backend section; required exactly when `backend` is `Local`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_kdf: Option<LocalKdfDescriptor>, + /// Reserved for the per-key version/envelope inventory defined by the + /// backlog#1565 contract. May not carry data in format version 1. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key_versions: Option<ReservedSlot>, + /// Reserved for the backlog#1571 capability-discovery contract. May not + /// carry data in format version 1. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capability_discovery: Option<ReservedSlot>, + /// Completeness marker; anything but `complete` is never restorable. + pub completeness: CompletenessState, + /// Digest over the canonical manifest bytes (see + /// [`Self::compute_digest`]); the final integrity anchor of the bundle. + pub manifest_digest: ContentDigest, +} + +impl BackupManifest { + /// The manifest format version this build reads and writes. + pub const FORMAT_VERSION: u32 = 1; + + /// Decode and fully validate a manifest from JSON bytes. + /// + /// Fail-closed order: truncated or malformed input, then unknown format + /// version, then a missing completeness marker, then schema decoding + /// (unknown fields, duplicate fields, missing fields), then semantic + /// validation including digest verification. + pub fn decode(bytes: &[u8]) -> Result<Self, BackupError> { + let probe: ManifestProbe = serde_json::from_slice(bytes).map_err(map_serde_error)?; + if probe.format_version != Self::FORMAT_VERSION { + return Err(BackupError::UnknownVersion { + found: probe.format_version, + supported: Self::FORMAT_VERSION, + }); + } + if probe.completeness.is_none() { + return Err(BackupError::incomplete_bundle("manifest has no completeness marker")); + } + let manifest: Self = serde_json::from_slice(bytes).map_err(map_serde_error)?; + manifest.validate()?; + Ok(manifest) + } + + /// Serialize a sealed, valid manifest to its canonical JSON bytes. + /// + /// Encoding validates first so an unsealed or inconsistent manifest can + /// never be published. + pub fn encode(&self) -> Result<Vec<u8>, BackupError> { + self.validate()?; + serde_json::to_vec(self).map_err(|error| BackupError::corrupted(format!("manifest serialization failed: {error}"))) + } + + /// Seal the manifest: mark it complete and stamp the canonical digest. + pub fn seal(mut self) -> Result<Self, BackupError> { + self.completeness = CompletenessState::Complete; + self.manifest_digest = self.compute_digest()?; + Ok(self) + } + + /// Compute the digest over the canonical manifest bytes. + /// + /// Canonical form: compact JSON serialization of this manifest with the + /// digest hex emptied. Field order is struct declaration order and is + /// frozen for format version 1, so the same manifest content always + /// hashes to the same value. + pub fn compute_digest(&self) -> Result<ContentDigest, BackupError> { + let mut unsealed = self.clone(); + unsealed.manifest_digest = ContentDigest::placeholder(self.manifest_digest.algorithm); + let canonical = serde_json::to_vec(&unsealed) + .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?; + match self.manifest_digest.algorithm { + DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)), + } + } + + /// Verify the sealed digest against the current manifest content. + pub fn verify_digest(&self) -> Result<(), BackupError> { + if !self.manifest_digest.is_well_formed() { + return Err(BackupError::corrupted("manifest digest is not a well-formed digest value")); + } + if self.compute_digest()? != self.manifest_digest { + return Err(BackupError::corrupted( + "manifest digest mismatch: content does not match the sealed digest", + )); + } + Ok(()) + } + + /// Look up a required artifact by kind, failing closed when absent. + pub fn require_artifact(&self, kind: ArtifactKind) -> Result<&ArtifactDescriptor, BackupError> { + self.artifacts + .iter() + .find(|artifact| artifact.kind == kind) + .ok_or_else(|| BackupError::missing_artifact(artifact_kind_name(kind))) + } + + /// Validate the full manifest contract. + /// + /// This is decode-side validation and also guards [`Self::encode`], so a + /// producer cannot publish a manifest a decoder would reject. + pub fn validate(&self) -> Result<(), BackupError> { + if self.format_version != Self::FORMAT_VERSION { + return Err(BackupError::UnknownVersion { + found: self.format_version, + supported: Self::FORMAT_VERSION, + }); + } + if self.completeness != CompletenessState::Complete { + return Err(BackupError::incomplete_bundle("completeness marker records an in-progress bundle")); + } + self.verify_digest()?; + require_non_empty("backup_id", &self.backup_id)?; + require_non_empty("rustfs_version", &self.rustfs_version)?; + require_non_empty("deployment_identity", &self.deployment_identity)?; + require_non_empty("backup_kek.kek_id", &self.backup_kek.kek_id)?; + if self.key_versions.is_some() { + return Err(BackupError::corrupted("reserved field key_versions must not carry data")); + } + if self.capability_discovery.is_some() { + return Err(BackupError::corrupted("reserved field capability_discovery must not carry data")); + } + self.validate_responsibility()?; + self.validate_local_kdf()?; + self.validate_artifacts()?; + Ok(()) + } + + fn validate_responsibility(&self) -> Result<(), BackupError> { + match BackupResponsibility::for_backend(self.backend, self.at_rest_protection) { + None => Err(BackupError::corrupted(format!( + "at-rest protection {:?} is not a defined state for backend {:?}", + self.at_rest_protection, self.backend + ))), + Some(expected) if expected != self.responsibility => Err(BackupError::corrupted(format!( + "declared responsibility {:?} contradicts the matrix row {expected:?} for ({:?}, {:?})", + self.responsibility, self.backend, self.at_rest_protection + ))), + Some(_) => Ok(()), + } + } + + fn validate_local_kdf(&self) -> Result<(), BackupError> { + match (self.backend, &self.local_kdf) { + (BackupBackendKind::Local, None) => { + Err(BackupError::corrupted("local backend manifest must carry a local_kdf descriptor")) + } + (BackupBackendKind::Local, Some(descriptor)) => descriptor.validate(), + (_, Some(_)) => Err(BackupError::corrupted("local_kdf descriptor is only valid for the Local backend")), + (_, None) => Ok(()), + } + } + + fn validate_artifacts(&self) -> Result<(), BackupError> { + let mut paths = BTreeSet::new(); + for artifact in &self.artifacts { + if artifact.kind.is_reserved() { + return Err(BackupError::corrupted(format!( + "artifact kind {:?} is reserved and not valid in format version 1", + artifact.kind + ))); + } + validate_bundle_relative_path(&artifact.path)?; + if artifact.len == 0 { + return Err(BackupError::corrupted(format!( + "artifact '{}' declares a zero-length payload; an AEAD payload is never empty", + artifact.path + ))); + } + if !artifact.encrypted_digest.is_well_formed() { + return Err(BackupError::corrupted(format!( + "artifact '{}' has a malformed encrypted digest", + artifact.path + ))); + } + if !paths.insert(artifact.path.as_str()) { + return Err(BackupError::corrupted(format!( + "artifact path '{}' appears more than once", + artifact.path + ))); + } + } + + let has_key_material = self + .artifacts + .iter() + .any(|artifact| artifact.kind == ArtifactKind::KeyMaterial); + match self.responsibility { + BackupResponsibility::FullMaterial => { + if !has_key_material { + return Err(BackupError::missing_artifact(artifact_kind_name(ArtifactKind::KeyMaterial))); + } + } + BackupResponsibility::ReferenceOnly | BackupResponsibility::MetadataPlusExternalRoot => { + if has_key_material { + return Err(BackupError::corrupted(format!( + "a {:?} bundle must not embed key material artifacts", + self.responsibility + ))); + } + } + } + Ok(()) + } +} + +fn require_non_empty(field: &str, value: &str) -> Result<(), BackupError> { + if value.is_empty() { + return Err(BackupError::corrupted(format!("security-critical field {field} must not be empty"))); + } + Ok(()) +} + +fn artifact_kind_name(kind: ArtifactKind) -> String { + // Wire name of the kind, reusing the serde rename rules so error text and + // schema never diverge. + serde_json::to_value(kind) + .ok() + .and_then(|value| value.as_str().map(str::to_string)) + .unwrap_or_else(|| format!("{kind:?}")) +} + +fn validate_bundle_relative_path(path: &str) -> Result<(), BackupError> { + if path.is_empty() { + return Err(BackupError::corrupted("artifact path must not be empty")); + } + if path.contains('\\') || path.contains('\0') { + return Err(BackupError::corrupted(format!( + "artifact path {path:?} must not contain backslashes or NUL" + ))); + } + let parsed = Path::new(path); + if parsed.is_absolute() { + return Err(BackupError::corrupted(format!("artifact path {path:?} must be bundle-relative"))); + } + for component in parsed.components() { + match component { + Component::Normal(_) => {} + _ => { + return Err(BackupError::corrupted(format!( + "artifact path {path:?} must not contain traversal or special components" + ))); + } + } + } + Ok(()) +} + +fn map_serde_error(error: serde_json::Error) -> BackupError { + if error.classify() == serde_json::error::Category::Eof { + BackupError::truncated(error.to_string()) + } else { + BackupError::corrupted(error.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const DUMMY_ARTIFACT_DIGEST: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + /// Handwritten independently of the serializer so a silent serde-side + /// change cannot keep the round-trip test green while breaking the wire + /// format. The digest hex is the sealed digest of exactly this content. + const FIXTURE: &str = r#"{ + "format_version": 1, + "backup_id": "backup-0001", + "created_at": "2026-07-30T00:00:00+00:00[UTC]", + "rustfs_version": "1.0.0-test", + "deployment_identity": "deployment-a", + "backend": "Local", + "at_rest_protection": "encrypted-master-key", + "responsibility": "full-material", + "snapshot_generation": 7, + "backup_kek": { + "kek_id": "backup-kek-1", + "kek_version": 1, + "aead_algorithm": "aes-256-gcm" + }, + "artifacts": [ + { + "kind": "key-material", + "path": "keys/key-material.enc", + "len": 256, + "aead_algorithm": "aes-256-gcm", + "encrypted_digest": { + "algorithm": "sha-256", + "hex": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + }, + { + "kind": "key-metadata", + "path": "keys/key-metadata.enc", + "len": 256, + "aead_algorithm": "aes-256-gcm", + "encrypted_digest": { + "algorithm": "sha-256", + "hex": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + }, + { + "kind": "master-key-salt", + "path": "master-key.salt.enc", + "len": 256, + "aead_algorithm": "aes-256-gcm", + "encrypted_digest": { + "algorithm": "sha-256", + "hex": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + }, + { + "kind": "kms-config", + "path": "config/kms-config.enc", + "len": 256, + "aead_algorithm": "aes-256-gcm", + "encrypted_digest": { + "algorithm": "sha-256", + "hex": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + } + } + ], + "local_kdf": { + "derivation": { + "argon2id": { + "version": 19, + "memory_kib": 19456, + "iterations": 2, + "parallelism": 1, + "output_len": 32, + "salt_len": 16 + } + }, + "protection_modes": ["encrypted-master-key"], + "master_key_verifier": "verifier-opaque-1" + }, + "completeness": "complete", + "manifest_digest": { + "algorithm": "sha-256", + "hex": "SEALED_DIGEST_HEX" + } + }"#; + + /// Sealed digest of the fixture content above; computed once from the + /// canonical form and frozen. If serialization layout or field order + /// changes, this value changes and the fixture test fails — which is the + /// point: that is a format-version bump, not a patch. + const FIXTURE_DIGEST_HEX: &str = "01accb3e2bc51e12d17d1efc52ad6ab9441c50bec52c3fb29e0a9f92b725cdaa"; + + fn fixture() -> String { + FIXTURE.replace("SEALED_DIGEST_HEX", FIXTURE_DIGEST_HEX) + } + + fn artifact(kind: ArtifactKind, path: &str) -> ArtifactDescriptor { + ArtifactDescriptor { + kind, + path: path.to_string(), + len: 256, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + encrypted_digest: ContentDigest { + algorithm: DigestAlgorithm::Sha256, + hex: DUMMY_ARTIFACT_DIGEST.to_string(), + }, + } + } + + fn local_manifest_unsealed() -> BackupManifest { + BackupManifest { + format_version: BackupManifest::FORMAT_VERSION, + backup_id: "backup-0001".to_string(), + created_at: "2026-07-30T00:00:00+00:00[UTC]" + .parse() + .expect("fixture timestamp should parse"), + rustfs_version: "1.0.0-test".to_string(), + deployment_identity: "deployment-a".to_string(), + backend: BackupBackendKind::Local, + at_rest_protection: AtRestProtection::EncryptedMasterKey, + responsibility: BackupResponsibility::FullMaterial, + snapshot_generation: 7, + backup_kek: BackupKekDescriptor { + kek_id: "backup-kek-1".to_string(), + kek_version: 1, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + }, + artifacts: vec![ + artifact(ArtifactKind::KeyMaterial, "keys/key-material.enc"), + artifact(ArtifactKind::KeyMetadata, "keys/key-metadata.enc"), + artifact(ArtifactKind::MasterKeySalt, "master-key.salt.enc"), + artifact(ArtifactKind::KmsConfig, "config/kms-config.enc"), + ], + local_kdf: Some(LocalKdfDescriptor::current( + vec![AtRestProtection::EncryptedMasterKey], + Some("verifier-opaque-1".to_string()), + )), + key_versions: None, + capability_discovery: None, + completeness: CompletenessState::InProgress, + manifest_digest: ContentDigest { + algorithm: DigestAlgorithm::Sha256, + hex: String::new(), + }, + } + } + + fn static_manifest_unsealed() -> BackupManifest { + BackupManifest { + backend: BackupBackendKind::Static, + at_rest_protection: AtRestProtection::ExternalSecretDelivery, + responsibility: BackupResponsibility::ReferenceOnly, + artifacts: vec![artifact(ArtifactKind::KmsConfig, "config/kms-config.enc")], + local_kdf: None, + ..local_manifest_unsealed() + } + } + + fn seal(manifest: BackupManifest) -> BackupManifest { + manifest.seal().expect("sealing should succeed") + } + + fn expect_corrupted(result: Result<BackupManifest, BackupError>, needle: &str) { + match result { + Err(BackupError::Corrupted { reason }) => { + assert!( + reason.contains(needle), + "expected corruption reason containing {needle:?}, got {reason:?}" + ); + } + other => panic!("expected Corrupted error containing {needle:?}, got {other:?}"), + } + } + + #[test] + fn sealed_manifest_round_trips() { + let sealed = seal(local_manifest_unsealed()); + let bytes = sealed.encode().expect("encoding should succeed"); + let decoded = BackupManifest::decode(&bytes).expect("decoding should succeed"); + assert_eq!(decoded, sealed); + assert_eq!(decoded.completeness, CompletenessState::Complete); + } + + #[test] + fn handwritten_fixture_decodes_and_matches_serialization() { + let decoded = BackupManifest::decode(fixture().as_bytes()).expect("fixture should decode"); + let expected = seal(local_manifest_unsealed()); + assert_eq!(decoded, expected); + + // The serializer must produce exactly the handwritten wire content. + let serialized: serde_json::Value = + serde_json::from_slice(&expected.encode().expect("encoding should succeed")).expect("re-parse should succeed"); + let fixture_value: serde_json::Value = serde_json::from_str(&fixture()).expect("fixture should parse"); + assert_eq!(serialized, fixture_value); + } + + #[test] + fn unknown_format_version_is_rejected() { + let tampered = fixture().replace("\"format_version\": 1", "\"format_version\": 99"); + match BackupManifest::decode(tampered.as_bytes()) { + Err(BackupError::UnknownVersion { found, supported }) => { + assert_eq!(found, 99); + assert_eq!(supported, BackupManifest::FORMAT_VERSION); + } + other => panic!("expected UnknownVersion, got {other:?}"), + } + } + + #[test] + fn missing_completeness_marker_fails_closed() { + let mut value: serde_json::Value = serde_json::from_str(&fixture()).expect("fixture should parse"); + value + .as_object_mut() + .expect("fixture should be an object") + .remove("completeness"); + let bytes = serde_json::to_vec(&value).expect("serialization should succeed"); + match BackupManifest::decode(&bytes) { + Err(BackupError::IncompleteBundle { reason }) => { + assert!(reason.contains("no completeness marker"), "unexpected reason {reason:?}"); + } + other => panic!("expected IncompleteBundle, got {other:?}"), + } + } + + #[test] + fn in_progress_completeness_fails_closed() { + let tampered = fixture().replace("\"complete\"", "\"in-progress\""); + match BackupManifest::decode(tampered.as_bytes()) { + Err(BackupError::IncompleteBundle { reason }) => { + assert!(reason.contains("in-progress"), "unexpected reason {reason:?}"); + } + other => panic!("expected IncompleteBundle, got {other:?}"), + } + } + + #[test] + fn missing_security_critical_field_fails_closed() { + let mut value: serde_json::Value = serde_json::from_str(&fixture()).expect("fixture should parse"); + value + .as_object_mut() + .expect("fixture should be an object") + .remove("backup_kek"); + let bytes = serde_json::to_vec(&value).expect("serialization should succeed"); + expect_corrupted(BackupManifest::decode(&bytes), "missing field"); + } + + #[test] + fn duplicate_field_fails_closed() { + let tampered = fixture().replace("\"snapshot_generation\": 7", "\"snapshot_generation\": 7, \"snapshot_generation\": 7"); + expect_corrupted(BackupManifest::decode(tampered.as_bytes()), "duplicate field"); + } + + #[test] + fn unknown_field_fails_closed() { + let tampered = fixture().replace("\"snapshot_generation\": 7", "\"surprise\": true, \"snapshot_generation\": 7"); + expect_corrupted(BackupManifest::decode(tampered.as_bytes()), "unknown field"); + } + + #[test] + fn truncated_manifest_fails_closed() { + let full = fixture(); + let truncated = &full.as_bytes()[..full.len() / 2]; + match BackupManifest::decode(truncated) { + Err(BackupError::Truncated { .. }) => {} + other => panic!("expected Truncated, got {other:?}"), + } + } + + #[test] + fn tampered_generation_is_rejected() { + let tampered = fixture().replace("\"snapshot_generation\": 7", "\"snapshot_generation\": 8"); + expect_corrupted(BackupManifest::decode(tampered.as_bytes()), "digest mismatch"); + } + + #[test] + fn tampered_digest_is_rejected() { + let last_char = FIXTURE_DIGEST_HEX.as_bytes()[FIXTURE_DIGEST_HEX.len() - 1]; + let flipped = if last_char == b'0' { '1' } else { '0' }; + let mut tampered_digest = FIXTURE_DIGEST_HEX[..FIXTURE_DIGEST_HEX.len() - 1].to_string(); + tampered_digest.push(flipped); + let tampered = fixture().replace(FIXTURE_DIGEST_HEX, &tampered_digest); + expect_corrupted(BackupManifest::decode(tampered.as_bytes()), "digest mismatch"); + } + + #[test] + fn malformed_digest_value_is_rejected() { + let tampered = fixture().replace(FIXTURE_DIGEST_HEX, "not-hex"); + expect_corrupted(BackupManifest::decode(tampered.as_bytes()), "well-formed"); + } + + #[test] + fn wrong_kek_fails_closed() { + let sealed = seal(local_manifest_unsealed()); + sealed + .backup_kek + .ensure_matches("backup-kek-1", 1) + .expect("matching KEK should pass"); + + match sealed.backup_kek.ensure_matches("backup-kek-2", 1) { + Err(BackupError::WrongKek { + required_kek_id, + supplied_kek_id, + .. + }) => { + assert_eq!(required_kek_id, "backup-kek-1"); + assert_eq!(supplied_kek_id, "backup-kek-2"); + } + other => panic!("expected WrongKek, got {other:?}"), + } + assert!(matches!( + sealed.backup_kek.ensure_matches("backup-kek-1", 2), + Err(BackupError::WrongKek { .. }) + )); + } + + #[test] + fn full_material_bundle_requires_key_material_artifact() { + let mut manifest = local_manifest_unsealed(); + manifest + .artifacts + .retain(|artifact| artifact.kind != ArtifactKind::KeyMaterial); + match seal(manifest).validate() { + Err(BackupError::MissingArtifact { artifact }) => assert_eq!(artifact, "key-material"), + other => panic!("expected MissingArtifact, got {other:?}"), + } + } + + #[test] + fn reference_only_bundle_must_not_embed_key_material() { + let mut manifest = static_manifest_unsealed(); + manifest + .artifacts + .push(artifact(ArtifactKind::KeyMaterial, "keys/forbidden.enc")); + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("must not embed key material"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + } + + #[test] + fn reserved_artifact_kinds_are_rejected() { + for kind in [ArtifactKind::Alias, ArtifactKind::Policy] { + assert!(kind.is_reserved()); + let mut manifest = local_manifest_unsealed(); + manifest.artifacts.push(artifact(kind, "reserved/artifact.enc")); + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("reserved"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted for {kind:?}, got {other:?}"), + } + } + } + + #[test] + fn reserved_fields_reject_data() { + let with_data = fixture().replace("\"completeness\"", "\"key_versions\": {}, \"completeness\""); + expect_corrupted(BackupManifest::decode(with_data.as_bytes()), "reserved"); + + let with_discovery = fixture().replace("\"completeness\"", "\"capability_discovery\": [1], \"completeness\""); + expect_corrupted(BackupManifest::decode(with_discovery.as_bytes()), "reserved"); + + // Explicit null carries no data and is tolerated as absence; digest + // verification still passes because null slots are skipped on + // serialization. + let with_null = fixture().replace("\"completeness\"", "\"key_versions\": null, \"completeness\""); + let decoded = BackupManifest::decode(with_null.as_bytes()).expect("null reserved slot should decode"); + assert_eq!(decoded.key_versions, None); + } + + #[test] + fn responsibility_contradicting_matrix_is_rejected() { + let mut manifest = local_manifest_unsealed(); + manifest.responsibility = BackupResponsibility::ReferenceOnly; + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("contradicts the matrix"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + + let mut manifest = local_manifest_unsealed(); + manifest.at_rest_protection = AtRestProtection::StorageOnly; + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("not a defined state"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + } + + #[test] + fn local_kdf_descriptor_presence_is_tied_to_backend() { + let mut manifest = local_manifest_unsealed(); + manifest.local_kdf = None; + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("local_kdf"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + + let mut manifest = static_manifest_unsealed(); + manifest.local_kdf = Some(LocalKdfDescriptor::current(vec![AtRestProtection::EncryptedMasterKey], None)); + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("only valid for the Local backend"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + } + + #[test] + fn local_kdf_descriptor_rejects_non_local_and_duplicate_modes() { + let mut manifest = local_manifest_unsealed(); + manifest.local_kdf = Some(LocalKdfDescriptor::current(vec![AtRestProtection::StorageOnly], None)); + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("non-local protection mode"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + + let mut manifest = local_manifest_unsealed(); + manifest.local_kdf = Some(LocalKdfDescriptor::current( + vec![AtRestProtection::EncryptedMasterKey, AtRestProtection::EncryptedMasterKey], + None, + )); + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("more than once"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + } + + #[test] + fn artifact_paths_must_be_bundle_relative_and_unique() { + for bad_path in ["/etc/keys.enc", "../escape.enc", "a/../b.enc", ""] { + let mut manifest = local_manifest_unsealed(); + manifest.artifacts.push(artifact(ArtifactKind::KeyMetadata, bad_path)); + assert!( + matches!(seal(manifest).validate(), Err(BackupError::Corrupted { .. })), + "path {bad_path:?} should be rejected" + ); + } + + let mut manifest = local_manifest_unsealed(); + let duplicate = manifest.artifacts[0].clone(); + manifest.artifacts.push(duplicate); + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("more than once"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + } + + #[test] + fn zero_length_artifacts_are_rejected() { + let mut manifest = local_manifest_unsealed(); + manifest.artifacts[0].len = 0; + match seal(manifest).validate() { + Err(BackupError::Corrupted { reason }) => { + assert!(reason.contains("zero-length"), "unexpected reason {reason:?}"); + } + other => panic!("expected Corrupted, got {other:?}"), + } + } + + #[test] + fn unsealed_manifest_cannot_be_encoded() { + let manifest = local_manifest_unsealed(); + assert!(matches!(manifest.encode(), Err(BackupError::IncompleteBundle { .. }))); + } + + #[test] + fn require_artifact_reports_missing_kind() { + let sealed = seal(static_manifest_unsealed()); + sealed + .require_artifact(ArtifactKind::KmsConfig) + .expect("config artifact should be present"); + match sealed.require_artifact(ArtifactKind::MasterKeySalt) { + Err(BackupError::MissingArtifact { artifact }) => assert_eq!(artifact, "master-key-salt"), + other => panic!("expected MissingArtifact, got {other:?}"), + } + } + + #[test] + fn current_local_kdf_matches_backend_constants() { + // The descriptor is defined in terms of the local.rs constants, so + // this pins the concrete values: changing a KDF parameter must be a + // conscious contract decision, not a drive-by edit. + assert_eq!( + LocalKeyDerivation::current_argon2id(), + LocalKeyDerivation::Argon2id { + version: 0x13, + memory_kib: 19 * 1024, + iterations: 2, + parallelism: 1, + output_len: 32, + salt_len: 16, + } + ); + } + + #[test] + fn static_bundle_round_trips() { + let sealed = seal(static_manifest_unsealed()); + let bytes = sealed.encode().expect("encoding should succeed"); + let decoded = BackupManifest::decode(&bytes).expect("decoding should succeed"); + assert_eq!(decoded, sealed); + assert_eq!(decoded.responsibility, BackupResponsibility::ReferenceOnly); + } +} diff --git a/crates/kms/src/backup/mod.rs b/crates/kms/src/backup/mod.rs new file mode 100644 index 000000000..59cdba350 --- /dev/null +++ b/crates/kms/src/backup/mod.rs @@ -0,0 +1,63 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Backup/restore contract types for KMS state. +//! +//! This module is contract-only: it defines the versioned backup manifest, +//! the per-backend responsibility matrix, typed failure modes, and the +//! restore dry-run report. Nothing here is wired into handlers or backends; +//! backup export, restore orchestration, and the admin API build on these +//! types in follow-up changes. +//! +//! # Bundle model +//! +//! A backup bundle is a set of AEAD-encrypted artifacts described by a single +//! [`BackupManifest`]. All state in a bundle belongs to one snapshot +//! generation — there is no partially consistent bundle. The bundle is +//! protected by a backup KEK that is deliberately outside the business KMS +//! trust hierarchy, and the manifest is sealed with a completeness marker and +//! a final digest; a bundle that never reached its marker is permanently +//! non-restorable. +//! +//! # Restore ordering +//! +//! Restore implementations must follow this order: re-establish the external +//! trust root first (Vault/HSM native restore where one exists), then +//! material and version records into staging, then metadata and +//! configuration, then verification, and only then an explicit atomic +//! cutover. A dry-run ([`RestoreDryRunReport`]) performs zero writes. +//! +//! # Deliberately unfrozen +//! +//! Fields whose shape depends on contracts still in flight are reserved +//! rather than guessed (see [`ReservedSlot`]): the per-key version inventory +//! (backlog#1565) and capability discovery (backlog#1571). Alias and policy +//! artifacts are reserved names for features that do not exist yet. Reserved +//! slots reject data in format version 1 and become real types in a later +//! format version. + +mod capability; +mod dry_run; +mod error; +mod manifest; + +pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; +pub use dry_run::{ + ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, +}; +pub use error::BackupError; +pub use manifest::{ + AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, + DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, +}; diff --git a/crates/kms/src/error.rs b/crates/kms/src/error.rs index 8a7ef6563..98fbb5008 100644 --- a/crates/kms/src/error.rs +++ b/crates/kms/src/error.rs @@ -124,6 +124,10 @@ pub enum KmsError { /// Requested master key version has no persisted material for the key #[error("Key version {version} not found for key {key_id}")] KeyVersionNotFound { key_id: String, version: u32 }, + + /// Backup/restore bundle contract violation; see [`crate::backup::BackupError`] + #[error(transparent)] + Backup(#[from] crate::backup::BackupError), } impl KmsError { diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index 3867ef7b0..f05d45e16 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -66,6 +66,7 @@ // Core modules pub mod api_types; pub mod backends; +pub mod backup; mod cache; pub mod config; mod encryption; From 40ef0db9ccb9be2a2d78a1841f1b762fcf2ffebd Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 01:49:55 +0800 Subject: [PATCH 26/52] test(kms): pin rotation contracts across backends and document retention (#5486) * feat(kms): retain historical master key versions for Vault KV2 rotation Rotation previously had to be rejected outright because replacing the stored material would orphan every DEK wrapped by earlier versions. Vault KV2 now keeps each version's material in an immutable, create-only record at {prefix}/{key_id}/versions/{N} and treats the top-level record as the current-version pointer plus fast-path material copy: - decrypt resolves the envelope's master_key_version to its version record; a missing version fails closed with KeyVersionNotFound and never falls back to the current material - envelopes without a version (pre-versioning writers) resolve to the baseline_version frozen at the key's first rotation, so never-rotated keys behave exactly as before - generate_data_key stamps the wrapping version from the same key record snapshot that supplied the material - rotate_key commits in check-and-set order: freeze baseline, persist the next version's material, then switch the current pointer; any failure leaves the current pointer untouched, and concurrent rotations serialize on the CAS writes with monotonically unique versions - key listings drop the versions/ directory entries and physical key deletion purges version records before the key record Refs rustfs/backlog#1565 * test(kms): pin rotation contracts across backends and document retention Completes the backlog#1565 series with the cross-backend regression net and operator documentation: - Vault Transit: ignored integration test proving version-prefixed historical ciphertext still decrypts after rotation, with no RustFS-side version bookkeeping in the envelope - Local: offline mixed-format test interleaving pre-versioning and versioned envelopes through a rejected rotation; the existing rotation-rejection pinning test already covers material immutability - docs: kms-backend-security.md gains the KV2 versioned retention model, version record retention/destruction preconditions, and the upgrade-before-first-rotation cluster constraint Refs rustfs/backlog#1565 --- crates/kms/src/backends/local.rs | 58 +++++++++++++++++++++ crates/kms/src/backends/vault_transit.rs | 66 ++++++++++++++++++++++++ docs/operations/kms-backend-security.md | 28 ++++++++-- 3 files changed, 149 insertions(+), 3 deletions(-) diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 1cbad8a1f..2975a2999 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -1910,6 +1910,64 @@ mod tests { ); } + /// Mixed-format regression for rustfs/backlog#1565: a batch interleaving + /// pre-versioning envelopes (no master_key_version field) with versioned ones + /// must route and decrypt in full, and a rejected rotation in the middle must + /// not disturb either format. + #[tokio::test] + async fn mixed_format_envelopes_decrypt_across_rejected_rotation() { + let (client, _temp_dir) = create_test_client().await; + let key_id = "mixed-format-key"; + client.create_key(key_id, "AES_256", None).await.expect("create key"); + + let request = GenerateKeyRequest::new(key_id.to_string(), "AES_256".to_string()); + let mut batch = Vec::new(); + for index in 0..4 { + let data_key = client.generate_data_key(&request, None).await.expect("generate data key"); + let ciphertext = if index % 2 == 0 { + // Legacy shape: the local backend already omits master_key_version. + let envelope: serde_json::Value = serde_json::from_slice(&data_key.ciphertext).expect("parse envelope"); + assert!( + !envelope + .as_object() + .expect("envelope is an object") + .contains_key("master_key_version"), + "local envelopes must keep the pre-versioning shape" + ); + data_key.ciphertext.clone() + } else { + // Versioned shape, as a rotation-aware writer would emit it. + let mut envelope: serde_json::Value = serde_json::from_slice(&data_key.ciphertext).expect("parse envelope"); + envelope + .as_object_mut() + .expect("envelope is an object") + .insert("master_key_version".to_string(), serde_json::json!(1)); + serde_json::to_vec(&envelope).expect("serialize versioned envelope") + }; + assert!( + crate::encryption::is_data_key_envelope(&ciphertext), + "batch member {index} must still route as a KMS envelope" + ); + batch.push((ciphertext, data_key.plaintext.clone().expect("plaintext"))); + } + + // A rejected rotation in the middle of the batch's lifetime must leave + // every already-issued envelope decryptable. + let error = client + .rotate_key(key_id, None) + .await + .expect_err("local rotation must stay rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. })); + + for (index, (ciphertext, plaintext)) in batch.iter().enumerate() { + let decrypted = client + .decrypt(&DecryptRequest::new(ciphertext.clone()), None) + .await + .unwrap_or_else(|error| panic!("batch member {index} must decrypt: {error}")); + assert_eq!(&decrypted, plaintext, "batch member {index} plaintext must round-trip"); + } + } + #[tokio::test] async fn startup_rejects_key_file_with_mismatched_embedded_id() { let (client, temp_dir) = create_dev_mode_client().await; diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 8ba3535a9..a291f85b1 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -907,4 +907,70 @@ mod tests { "after restart, a pending-deletion key must not be usable for new data keys" ); } + + /// Contract regression for rustfs/backlog#1565. + /// + /// Transit rotation is delegated entirely to Vault's own key versioning: the + /// ciphertext self-describes the wrapping version ("vault:vN:..."), so historical + /// ciphertext must keep decrypting after rotation without any RustFS-side + /// version bookkeeping in the envelope. + #[tokio::test] + #[ignore] // Requires a running Vault instance with transit engine enabled + async fn test_transit_old_ciphertext_decrypts_after_rotate() { + let client = VaultTransitKmsClient::new(test_vault_transit_config(), Duration::from_secs(30)) + .await + .expect("Failed to create VaultTransit client"); + + let key_id = format!("regression-1565-rotate-{}", uuid::Uuid::new_v4()); + client.create_key(&key_id, "AES_256", None).await.expect("create_key"); + + let request = GenerateKeyRequest { + master_key_id: key_id.clone(), + key_spec: "AES_256".to_string(), + key_length: Some(32), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }; + + let dk_v1 = client.generate_data_key(&request, None).await.expect("generate under v1"); + let env_v1: DataKeyEnvelope = serde_json::from_slice(&dk_v1.ciphertext).expect("parse v1 envelope"); + assert!( + env_v1.encrypted_key.starts_with(b"vault:v1:"), + "first-version Transit ciphertext must carry the vault:v1: prefix" + ); + assert_eq!( + env_v1.master_key_version, None, + "Transit envelopes must not carry a RustFS-side master key version" + ); + + let rotated = client.rotate_key(&key_id, None).await.expect("rotate_key"); + assert_eq!(rotated.version, 2, "rotation must advance the Transit key version"); + + let dk_v2 = client.generate_data_key(&request, None).await.expect("generate under v2"); + let env_v2: DataKeyEnvelope = serde_json::from_slice(&dk_v2.ciphertext).expect("parse v2 envelope"); + assert!( + env_v2.encrypted_key.starts_with(b"vault:v2:"), + "post-rotation Transit ciphertext must carry the vault:v2: prefix" + ); + + // Historical ciphertext keeps decrypting per Vault's version semantics, + // interleaved with post-rotation ciphertext. + for (data_key, label) in [(&dk_v1, "v1"), (&dk_v2, "v2"), (&dk_v1, "v1 again")] { + let plaintext = client + .decrypt( + &DecryptRequest { + ciphertext: data_key.ciphertext.clone(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .unwrap_or_else(|error| panic!("{label} ciphertext must stay decryptable after rotation: {error}")); + assert_eq!(Some(plaintext), data_key.plaintext, "{label} plaintext must round-trip"); + } + + // Cleanup so repeated runs against the same Vault do not accumulate keys. + let _ = client.schedule_key_deletion(&key_id, 7, None).await; + } } diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 08ad04b75..e8063f5b3 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -6,9 +6,9 @@ RustFS ships several KMS backends. They differ not only in deployment effort but | Backend | Config tag | Master key material location | At-rest protection of key material | Durability | Rotation | Intended use | | --- | --- | --- | --- | --- | --- | --- | -| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Crash-durable commits on local filesystems only; see [Local backend durability and deployment support matrix](#local-backend-durability-and-deployment-support-matrix) | Rejected (no versioned retention yet) | Development; single-node setups that accept host-level trust | +| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Crash-durable commits on local filesystems only; see [Local backend durability and deployment support matrix](#local-backend-durability-and-deployment-support-matrix) | Rejected by design (single material, development backend) | Development; single-node setups that accept host-level trust | | Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | No state persisted by RustFS | Rejected (read-only backend) | Simple deployments with an external secret manager | -| Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Delegated to Vault storage | Rejected (no versioned retention yet) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary | +| Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Delegated to Vault storage | Versioned retention (immutable per-version records + current pointer) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary | | Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | ## Vault KV2: what the backend does and does not do @@ -19,7 +19,7 @@ The Vault KV2 backend uses Vault purely as a **secure storage** service: - The backend never calls the Vault Transit engine. The `mount_path` configuration field and the `RUSTFS_KMS_VAULT_MOUNT_PATH` environment variable are deprecated leftovers: they are accepted for compatibility and ignored. - Data-encryption keys (DEKs) handed to the object-encryption path are still wrapped with AES-256-GCM under the master key; the statement above concerns the master key's storage in Vault, not the DEK envelope. - The backend reports this boundary in its `backend_info` metadata as `at_rest_protection: vault-kv2-acl`. -- Key rotation is rejected (`InvalidOperation`) until versioned key material retention lands; rotating by overwriting the stored key would orphan every DEK wrapped by the previous version. +- Key rotation retains every historical master key version as an immutable record under `{prefix}/{key_id}/versions/{N}` and only then moves the current-version pointer; see [Master key rotation](#master-key-rotation-retention-destruction-and-upgrade-ordering) for the retention preconditions and the cluster-upgrade ordering constraint. > **Warning: KV read access is equivalent to holding the master keys.** > Any Vault identity (token, AppRole, or policy) that can `read` the RustFS key path in KV v2 can recover the plaintext master key material and decrypt every object protected by those keys. Treat KV read grants on that path with the same care as handing out the keys themselves. If this is not acceptable, use the Vault Transit backend instead. @@ -41,10 +41,32 @@ path "secret/metadata/rustfs/kms/keys/*" { Notes: +- The trailing wildcards also cover the per-version material records that rotation creates under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed. - `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. - Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them. - Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure. +## Master key rotation: retention, destruction, and upgrade ordering + +Rotation support differs per backend. Local and Static reject rotation outright (`InvalidOperation`); their single key material is never overwritten. Vault Transit delegates rotation to the Transit engine's own key versioning (ciphertext is version-prefixed, e.g. `vault:v1:...`). Vault KV2 rotates by retaining every historical version, as described below. Rotation is currently only reachable through the backend-level `KmsClient::rotate_key` API; it is not exposed through the admin or S3 surface. + +### Vault KV2 versioned retention model + +Each rotation writes the new version's material to `{prefix}/{key_id}/versions/{N}` as an immutable, create-only record, and only after that material is durably persisted does a check-and-set write move the top-level record (the current-version pointer, which also mirrors the current material as a fast path). The first rotation additionally freezes the pre-rotation material as a version record and pins it as the key's `baseline_version`; DEK envelopes written before versioning existed (no `master_key_version` field) always resolve to that baseline, never to whatever version is current. + +Decryption loads exactly the version recorded in the envelope and fails closed with a typed `KeyVersionNotFound` error when that version's record is missing. There is deliberately no fallback to the current material: falling back would silently feed the wrong key to AEAD and mask tampered envelopes. + +### Retention and destruction preconditions + +- Every version record that any stored DEK envelope references must remain readable. Until an object rewrap/migration capability exists, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped. +- Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely. +- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. +- For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext. + +### Upgrade before first rotation (hard constraint) + +Do not rotate any key until **every** RustFS node in the cluster runs a build that understands the `master_key_version` envelope field. Older binaries ignore the field and always decrypt with the current material: harmless while nothing has been rotated, but after a rotation they will fail to decrypt every object wrapped by an earlier key version. Complete the rolling upgrade of the entire cluster first, then rotate. + ## Choosing between Vault KV2 and Vault Transit Use **Vault Transit** (`VaultTransit`) when key material must be cryptographically isolated from anyone holding storage-level read access: Transit keeps key-encryption keys inside Vault and only ever returns ciphertext, and supports server-side key versioning/rotation. From 342ee1df78342551202c869ea13913aee833406f Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 06:09:43 +0800 Subject: [PATCH 27/52] feat(kms): add backend capability discovery (#5485) --- crates/kms/src/backends/local.rs | 13 +- crates/kms/src/backends/mod.rs | 233 ++++++++++++++++++ ...ds__tests__local_backend_capabilities.snap | 14 ++ ...s__tests__static_backend_capabilities.snap | 14 ++ ...tests__vault_kv2_backend_capabilities.snap | 14 ++ ...s__vault_transit_backend_capabilities.snap | 14 ++ crates/kms/src/backends/static_kms.rs | 8 +- crates/kms/src/backends/vault.rs | 12 +- crates/kms/src/backends/vault_transit.rs | 14 +- crates/kms/src/error.rs | 12 + crates/kms/src/manager.rs | 5 + crates/kms/src/service.rs | 9 + rustfs/src/admin/handlers/kms_management.rs | 35 +++ 13 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap create mode 100644 crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap create mode 100644 crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap create mode 100644 crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 2975a2999..f6e71ac97 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -14,7 +14,7 @@ //! Local file-based KMS backend implementation -use crate::backends::{BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; use crate::config::KmsConfig; use crate::config::LocalConfig; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -1492,6 +1492,17 @@ impl KmsBackend for LocalKmsBackend { async fn health_check(&self) -> Result<bool> { self.client.health_check().await.map(|_| true) } + + fn capabilities(&self) -> BackendCapabilities { + // Rotation stays unadvertised until historical key versions can be + // retained (see LocalKmsClient::rotate_key); without version history + // there is also no versioning capability. Deletion deadlines are not + // yet persisted across restarts, but scheduling itself is supported. + BackendCapabilities::minimal() + .with_enable_disable(true) + .with_schedule_deletion(true) + .with_physical_delete(true) + } } #[cfg(test)] diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index 76da244f0..72d4df7c5 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -17,6 +17,7 @@ use crate::error::Result; use crate::types::*; use async_trait::async_trait; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; pub mod local; @@ -184,6 +185,16 @@ pub trait KmsBackend: Send + Sync { /// Health check async fn health_check(&self) -> Result<bool>; + + /// Report which operations this backend actually supports. + /// + /// The default is conservative: only the operations every backend is + /// required to implement by this trait are advertised. Optional lifecycle + /// operations (rotation, enable/disable, deletion scheduling, ...) must be + /// opted in by overriding this method. + fn capabilities(&self) -> BackendCapabilities { + BackendCapabilities::minimal() + } } /// Information about a KMS backend @@ -237,3 +248,225 @@ impl BackendInfo { self } } + +/// Set of operations a KMS backend supports. +/// +/// Reported by [`KmsBackend::capabilities`] so callers (manager, admin API) +/// can discover what the active backend can do without probing individual +/// operations. Marked `#[non_exhaustive]` so new capability flags can be +/// added without breaking downstream code; construct values through +/// [`BackendCapabilities::minimal`] and the `with_*` builders. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct BackendCapabilities { + /// Direct encryption of caller-provided plaintext with a master key + pub encrypt: bool, + /// Decryption of previously produced ciphertext + pub decrypt: bool, + /// Data encryption key (DEK) generation + pub generate_data_key: bool, + /// Key rotation that retains prior versions for decryption + pub rotate: bool, + /// Enabling and disabling keys + pub enable_disable: bool, + /// Scheduling key deletion with a pending window + pub schedule_deletion: bool, + /// Multiple key versions addressable after rotation + pub versioning: bool, + /// Irreversible physical deletion of key material + pub physical_delete: bool, +} + +impl BackendCapabilities { + /// Conservative baseline: only the operations that every [`KmsBackend`] + /// implementation is required to provide by the trait. All optional + /// lifecycle capabilities default to unsupported. + pub const fn minimal() -> Self { + Self { + encrypt: true, + decrypt: true, + generate_data_key: true, + rotate: false, + enable_disable: false, + schedule_deletion: false, + versioning: false, + physical_delete: false, + } + } + + /// Set whether direct encryption is supported + pub const fn with_encrypt(mut self, encrypt: bool) -> Self { + self.encrypt = encrypt; + self + } + + /// Set whether decryption is supported + pub const fn with_decrypt(mut self, decrypt: bool) -> Self { + self.decrypt = decrypt; + self + } + + /// Set whether data key generation is supported + pub const fn with_generate_data_key(mut self, generate_data_key: bool) -> Self { + self.generate_data_key = generate_data_key; + self + } + + /// Set whether version-retaining key rotation is supported + pub const fn with_rotate(mut self, rotate: bool) -> Self { + self.rotate = rotate; + self + } + + /// Set whether enabling/disabling keys is supported + pub const fn with_enable_disable(mut self, enable_disable: bool) -> Self { + self.enable_disable = enable_disable; + self + } + + /// Set whether scheduled deletion with a pending window is supported + pub const fn with_schedule_deletion(mut self, schedule_deletion: bool) -> Self { + self.schedule_deletion = schedule_deletion; + self + } + + /// Set whether multiple key versions are supported + pub const fn with_versioning(mut self, versioning: bool) -> Self { + self.versioning = versioning; + self + } + + /// Set whether physical deletion of key material is supported + pub const fn with_physical_delete(mut self, physical_delete: bool) -> Self { + self.physical_delete = physical_delete; + self + } +} + +impl Default for BackendCapabilities { + fn default() -> Self { + Self::minimal() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::KmsConfig; + use base64::Engine as _; + use base64::engine::general_purpose::STANDARD as BASE64; + + /// Backend that implements only the trait-mandated operations and relies + /// on the default `capabilities` implementation. + struct MinimalBackend; + + #[async_trait] + impl KmsBackend for MinimalBackend { + async fn create_key(&self, _request: CreateKeyRequest) -> Result<CreateKeyResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn encrypt(&self, _request: EncryptRequest) -> Result<EncryptResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn decrypt(&self, _request: DecryptRequest) -> Result<DecryptResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn describe_key(&self, _request: DescribeKeyRequest) -> Result<DescribeKeyResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn list_keys(&self, _request: ListKeysRequest) -> Result<ListKeysResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn delete_key(&self, _request: DeleteKeyRequest) -> Result<DeleteKeyResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn cancel_key_deletion(&self, _request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> { + unimplemented!("not exercised by capability tests") + } + + async fn health_check(&self) -> Result<bool> { + Ok(true) + } + } + + fn capabilities_snapshot(capabilities: BackendCapabilities) -> std::collections::BTreeMap<String, bool> { + serde_json::from_value(serde_json::to_value(capabilities).expect("capabilities should serialize")) + .expect("capabilities should deserialize into a flat bool map") + } + + #[test] + fn default_capabilities_are_conservative() { + let capabilities = MinimalBackend.capabilities(); + assert_eq!(capabilities, BackendCapabilities::minimal()); + assert_eq!(capabilities, BackendCapabilities::default()); + + // The conservative baseline advertises only trait-mandated operations. + assert!(capabilities.encrypt); + assert!(capabilities.decrypt); + assert!(capabilities.generate_data_key); + assert!(!capabilities.rotate); + assert!(!capabilities.enable_disable); + assert!(!capabilities.schedule_deletion); + assert!(!capabilities.versioning); + assert!(!capabilities.physical_delete); + } + + #[tokio::test] + async fn local_backend_capabilities_golden() { + let temp_dir = tempfile::tempdir().expect("temp dir should be created"); + let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + let backend = local::LocalKmsBackend::new(config).await.expect("local backend should build"); + + insta::assert_json_snapshot!("local_backend_capabilities", capabilities_snapshot(backend.capabilities())); + } + + #[tokio::test] + async fn vault_kv2_backend_capabilities_golden() { + let config = KmsConfig::vault( + url::Url::parse("http://127.0.0.1:8200").expect("vault URL should parse"), + "dev-token".to_string(), + ) + .with_insecure_development_defaults(); + // Constructing the client performs no network I/O with token auth. + let backend = vault::VaultKmsBackend::new(config) + .await + .expect("vault kv2 backend should build"); + + insta::assert_json_snapshot!("vault_kv2_backend_capabilities", capabilities_snapshot(backend.capabilities())); + } + + #[tokio::test] + async fn vault_transit_backend_capabilities_golden() { + let config = KmsConfig::vault_transit( + url::Url::parse("http://127.0.0.1:8200").expect("vault URL should parse"), + "dev-token".to_string(), + ) + .with_insecure_development_defaults(); + // Constructing the client performs no network I/O with token auth. + let backend = vault_transit::VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + insta::assert_json_snapshot!("vault_transit_backend_capabilities", capabilities_snapshot(backend.capabilities())); + } + + #[tokio::test] + async fn static_backend_capabilities_golden() { + let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode([0u8; 32])); + let backend = static_kms::StaticKmsBackend::new(config) + .await + .expect("static backend should build"); + + insta::assert_json_snapshot!("static_backend_capabilities", capabilities_snapshot(backend.capabilities())); + } +} diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap new file mode 100644 index 000000000..57be08b27 --- /dev/null +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap @@ -0,0 +1,14 @@ +--- +source: crates/kms/src/backends/mod.rs +expression: capabilities_snapshot(backend.capabilities()) +--- +{ + "decrypt": true, + "enable_disable": true, + "encrypt": true, + "generate_data_key": true, + "physical_delete": true, + "rotate": false, + "schedule_deletion": true, + "versioning": false +} diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap new file mode 100644 index 000000000..27c6e87a4 --- /dev/null +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap @@ -0,0 +1,14 @@ +--- +source: crates/kms/src/backends/mod.rs +expression: capabilities_snapshot(backend.capabilities()) +--- +{ + "decrypt": true, + "enable_disable": false, + "encrypt": true, + "generate_data_key": true, + "physical_delete": false, + "rotate": false, + "schedule_deletion": false, + "versioning": false +} diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap new file mode 100644 index 000000000..57be08b27 --- /dev/null +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap @@ -0,0 +1,14 @@ +--- +source: crates/kms/src/backends/mod.rs +expression: capabilities_snapshot(backend.capabilities()) +--- +{ + "decrypt": true, + "enable_disable": true, + "encrypt": true, + "generate_data_key": true, + "physical_delete": true, + "rotate": false, + "schedule_deletion": true, + "versioning": false +} diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap new file mode 100644 index 000000000..2c0bd7fce --- /dev/null +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap @@ -0,0 +1,14 @@ +--- +source: crates/kms/src/backends/mod.rs +expression: capabilities_snapshot(backend.capabilities()) +--- +{ + "decrypt": true, + "enable_disable": true, + "encrypt": true, + "generate_data_key": true, + "physical_delete": true, + "rotate": true, + "schedule_deletion": true, + "versioning": true +} diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 8dd4af518..73b988eeb 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -21,7 +21,7 @@ //! //! encrypted_data(plaintext_len+16) || nonce (12 bytes) -use crate::backends::{BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; use crate::config::{BackendConfig, KmsConfig}; use crate::encryption::DataKeyEnvelope; use crate::error::{KmsError, Result}; @@ -435,6 +435,12 @@ impl KmsBackend for StaticKmsBackend { async fn health_check(&self) -> Result<bool> { Ok(true) } + + fn capabilities(&self) -> BackendCapabilities { + // Static KMS is a read-only single-key backend: it only performs + // cryptographic operations and rejects every lifecycle mutation. + BackendCapabilities::minimal() + } } #[cfg(test)] diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 01cc4f042..97e1bc613 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -15,7 +15,7 @@ //! Vault-based KMS backend implementation using vaultrs use crate::backends::vault_credentials::{VaultClientHandle, VaultConnectionSettings, VaultCredentialProvider, token_source_for}; -use crate::backends::{BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; use crate::error::{KmsError, Result}; @@ -1095,6 +1095,16 @@ impl KmsBackend for VaultKmsBackend { async fn health_check(&self) -> Result<bool> { self.client.health_check().await.map(|_| true) } + + fn capabilities(&self) -> BackendCapabilities { + // Rotation is unadvertised: the KV2 backend cannot rotate without + // replacing key material in place, and no historical versions are + // retained, so versioning is unsupported as well. + BackendCapabilities::minimal() + .with_enable_disable(true) + .with_schedule_deletion(true) + .with_physical_delete(true) + } } #[cfg(test)] diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index a291f85b1..a4153cdfc 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -15,7 +15,7 @@ //! Vault Transit-based KMS backend. use crate::backends::vault_credentials::{VaultClientHandle, VaultConnectionSettings, VaultCredentialProvider, token_source_for}; -use crate::backends::{BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; use crate::error::{KmsError, Result}; @@ -762,6 +762,18 @@ impl KmsBackend for VaultTransitKmsBackend { async fn health_check(&self) -> Result<bool> { self.client.health_check().await.map(|_| true) } + + fn capabilities(&self) -> BackendCapabilities { + // Vault Transit natively supports version-retaining rotation, keeps + // prior versions addressable for decryption, and allows physical + // deletion once a key is pending deletion. + BackendCapabilities::minimal() + .with_rotate(true) + .with_enable_disable(true) + .with_schedule_deletion(true) + .with_versioning(true) + .with_physical_delete(true) + } } #[cfg(test)] diff --git a/crates/kms/src/error.rs b/crates/kms/src/error.rs index 98fbb5008..409494b08 100644 --- a/crates/kms/src/error.rs +++ b/crates/kms/src/error.rs @@ -128,6 +128,10 @@ pub enum KmsError { /// Backup/restore bundle contract violation; see [`crate::backup::BackupError`] #[error(transparent)] Backup(#[from] crate::backup::BackupError), + + /// Operation is not supported by the active KMS backend + #[error("Operation '{operation}' is not supported by KMS backend '{backend}'")] + UnsupportedCapability { backend: String, operation: String }, } impl KmsError { @@ -269,6 +273,14 @@ impl KmsError { version, } } + + /// Create an unsupported capability error + pub fn unsupported_capability<S1: Into<String>, S2: Into<String>>(backend: S1, operation: S2) -> Self { + Self::UnsupportedCapability { + backend: backend.into(), + operation: operation.into(), + } + } } /// Convert from standard library errors diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 3cfc8612c..c8bf9e115 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -159,6 +159,11 @@ impl KmsManager { pub async fn health_check(&self) -> Result<bool> { self.backend.health_check().await } + + /// Report the capabilities of the configured backend + pub fn backend_capabilities(&self) -> crate::backends::BackendCapabilities { + self.backend.capabilities() + } } #[cfg(test)] diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index 9b6874dfb..01902541b 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -184,6 +184,15 @@ impl ObjectEncryptionService { self.kms_manager.health_check().await } + /// Report the capabilities of the configured backend + /// + /// # Returns + /// The capability matrix advertised by the active KMS backend + /// + pub fn backend_capabilities(&self) -> crate::backends::BackendCapabilities { + self.kms_manager.backend_capabilities() + } + /// Create a data encryption key for object encryption /// /// # Arguments diff --git a/rustfs/src/admin/handlers/kms_management.rs b/rustfs/src/admin/handlers/kms_management.rs index bbeca1d90..88ee3a6f9 100644 --- a/rustfs/src/admin/handlers/kms_management.rs +++ b/rustfs/src/admin/handlers/kms_management.rs @@ -72,6 +72,10 @@ pub struct KmsStatusResponse { pub cache_enabled: bool, pub cache_stats: Option<CacheStatsResponse>, pub default_key_id: Option<String>, + /// Capability matrix of the active backend. Additive field: omitted by + /// older servers, so it must stay optional for consumers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capabilities: Option<rustfs_kms::backends::BackendCapabilities>, } #[derive(Debug, Serialize, Deserialize)] @@ -204,6 +208,7 @@ impl Operation for KmsStatusHandler { cache_enabled: config.as_ref().is_some_and(|cfg| cfg.enable_cache), cache_stats, default_key_id: service.get_default_key_id().cloned(), + capabilities: Some(service.backend_capabilities()), }; let data = serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; @@ -339,4 +344,34 @@ mod tests { fn kms_clear_cache_rejects_server_info_fallback() { assert_lacks_action(&kms_clear_cache_actions(), Action::AdminAction(AdminAction::ServerInfoAdminAction)); } + + /// The `capabilities` field is additive: payloads produced by older + /// servers (without the field) must keep deserializing, and the field + /// must be omitted from JSON when unset so existing consumers see an + /// unchanged response shape. + #[test] + fn kms_status_response_capabilities_field_is_additive() { + let legacy_json = serde_json::json!({ + "backend_type": "local", + "backend_status": "healthy", + "cache_enabled": true, + "cache_stats": null, + "default_key_id": null, + }); + let legacy: super::KmsStatusResponse = + serde_json::from_value(legacy_json).expect("legacy status payload should deserialize"); + assert!(legacy.capabilities.is_none()); + + let serialized = serde_json::to_value(&legacy).expect("status response should serialize"); + assert!(serialized.get("capabilities").is_none(), "unset capabilities must be omitted"); + + let with_capabilities = super::KmsStatusResponse { + capabilities: Some(rustfs_kms::backends::BackendCapabilities::minimal()), + ..legacy + }; + let serialized = serde_json::to_value(&with_capabilities).expect("status response should serialize"); + let capabilities = serialized.get("capabilities").expect("capabilities must be present when set"); + assert_eq!(capabilities.get("encrypt"), Some(&serde_json::Value::Bool(true))); + assert_eq!(capabilities.get("rotate"), Some(&serde_json::Value::Bool(false))); + } } From 3921336b23da9d247e276e05b80298e58d4deab1 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 06:29:49 +0800 Subject: [PATCH 28/52] feat(kms): AppRole login with background token renewal and fail-closed expiry (#5487) * feat(kms): add AppRole configuration surface for Vault auth Extend VaultAuthMethod::AppRole with secret_id_file (re-read on every login so external rotation is picked up), a configurable auth mount (default "approle"), and an optional fail-closed safety window. All new fields are serde(default) so previously persisted configurations keep deserializing, and the strict admin-configure deserializer accepts them as optional. Environment selection: setting RUSTFS_KMS_VAULT_APPROLE_ROLE_ID switches both Vault backends to AppRole; the secret_id comes from RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE (path stored, file wins) or RUSTFS_KMS_VAULT_APPROLE_SECRET_ID, following the static secret-key file precedent. validate() rejects AppRole configs without a role_id, without any secret_id source, or with an empty mount. Also append the CredentialsUnavailable error variant used by the fail-closed credential gate. * feat(kms): implement AppRole login with background renewal and fail-closed expiry Implement the AppRoleLogin token source (vaultrs approle login + renew-self) and wire lease-bound credentials through the provider: - Each successful login/renewal installs a new client generation in the ArcSwap; in-flight requests finish on the generation they captured. - A background renewal task refreshes at half the lease TTL: renewable tokens are renewed in place, everything else (or a failed renewal) falls back to a fresh login. Auth exchanges run under the typed retry policy (OpClass::Auth) and failed cycles retry on a fixed cadence, so the provider recovers once Vault does. - Fail-closed: current() refuses to hand out a token inside the configured safety window of its expiry (default: one attempt timeout), returning CredentialsUnavailable instead of sending a request whose token may lapse mid-flight. - Refreshes are single-flight: concurrent triggers for the same generation coalesce into one login. - The renewal task's owner handle lives on the KMS service version: stop() shuts it down explicitly and reconfigure recycles it via cancel-on-drop when the old version is discarded. - The secret_id file is re-read on every login attempt; missing or empty files fail the attempt without contacting Vault. Crate-owned copies of tokens and secret_ids are zeroized on drop, and Debug output of every credential-carrying type stays redacted (leak regression tests). The renewal machinery is covered by paused-clock tests driving a scripted token source: renew-at-half-TTL timing, login fallback, fail-closed window entry and recovery, prompt task recycling, and coalesced concurrent refreshes. * feat(kms): add Vault Agent token file authentication Add the TokenFile source: the token is read from an agent-managed sink file (RUSTFS_KMS_VAULT_TOKEN_FILE or the TokenFile auth config) and re-read once per poll interval (default 30s) through the existing renewal loop, so a token rotated by the agent installs a new client generation within one poll of the atomic replace. Each successful read extends the token's observed validity to twice the poll interval; a file that disappears or turns empty keeps failing the refresh until the fail-closed window trips, and heals the provider as soon as it is restored. Reads are strict and never contact Vault on failure: the file must be non-empty after trimming, and on Unix group/other permission bits are a hard error (mirroring the SFTP host-key rule). Rotation detection uses a content digest; the token itself is never stored on the source and the crate-owned copy is zeroized. Configuring the token file together with AppRole or an explicit static token is rejected as a configuration error. All new config fields are serde(default) and the strict admin-configure deserializer accepts the new variant. Covered by paused-clock tests (atomic replacement installs a new generation next cycle, deletion fails closed and recovers, prompt task recycling) plus negatives for missing/empty/over-permissive files and a Debug leak regression. * docs(kms): add Vault authentication and credential lifecycle runbook Cover choosing between static token, AppRole, and Vault Agent token file auth; AppRole role setup with SecretID delivery and rotation; Agent sink deployment with the permission requirements; and the fail-closed window semantics with a troubleshooting table keyed on the renewal task's log lines. --- crates/kms/src/api_types.rs | 53 +- crates/kms/src/backends/vault.rs | 75 +- crates/kms/src/backends/vault_credentials.rs | 1201 ++++++++++++++++-- crates/kms/src/backends/vault_transit.rs | 68 +- crates/kms/src/config.rs | 409 +++++- crates/kms/src/error.rs | 10 + crates/kms/src/service_manager.rs | 20 +- docs/operations/kms-backend-security.md | 2 + docs/operations/vault-kms-authentication.md | 122 ++ 9 files changed, 1813 insertions(+), 147 deletions(-) create mode 100644 docs/operations/vault-kms-authentication.md diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index 221529f5c..7d96951c9 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -235,15 +235,55 @@ pub struct StartKmsRequest { #[derive(Deserialize)] #[serde(deny_unknown_fields)] enum StrictVaultAuthMethod { - Token { token: String }, - AppRole { role_id: String, secret_id: String }, + Token { + token: String, + }, + AppRole { + role_id: String, + #[serde(default)] + secret_id: String, + #[serde(default)] + secret_id_file: Option<std::path::PathBuf>, + #[serde(default)] + mount: Option<String>, + #[serde(default)] + refresh_safety_window_secs: Option<u64>, + }, + TokenFile { + path: std::path::PathBuf, + #[serde(default)] + poll_interval_secs: Option<u64>, + #[serde(default)] + refresh_safety_window_secs: Option<u64>, + }, } impl From<StrictVaultAuthMethod> for VaultAuthMethod { fn from(value: StrictVaultAuthMethod) -> Self { match value { StrictVaultAuthMethod::Token { token } => Self::Token { token }, - StrictVaultAuthMethod::AppRole { role_id, secret_id } => Self::AppRole { role_id, secret_id }, + StrictVaultAuthMethod::AppRole { + role_id, + secret_id, + secret_id_file, + mount, + refresh_safety_window_secs, + } => Self::AppRole { + role_id, + secret_id, + secret_id_file, + mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()), + refresh_safety_window_secs, + }, + StrictVaultAuthMethod::TokenFile { + path, + poll_interval_secs, + refresh_safety_window_secs, + } => Self::TokenFile { + path, + poll_interval_secs, + refresh_safety_window_secs, + }, } } } @@ -403,6 +443,7 @@ impl From<&KmsConfig> for KmsConfigSummary { auth_method_type: match &vault_config.auth_method { VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(), + VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), }, has_stored_credentials: true, namespace: vault_config.namespace.clone(), @@ -416,6 +457,7 @@ impl From<&KmsConfig> for KmsConfigSummary { auth_method_type: match &vault_config.auth_method { VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(), + VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), }, has_stored_credentials: true, namespace: vault_config.namespace.clone(), @@ -872,10 +914,7 @@ mod tests { }); let approle = ConfigureKmsRequest::VaultKv2(ConfigureVaultKmsRequest { address: "https://vault.example.com:8200".to_string(), - auth_method: VaultAuthMethod::AppRole { - role_id: "configure-role-id".to_string(), - secret_id: "configure-approle-secret-id".to_string(), - }, + auth_method: VaultAuthMethod::approle("configure-role-id".to_string(), "configure-approle-secret-id".to_string()), namespace: None, mount_path: Some("transit".to_string()), kv_mount: Some("secret".to_string()), diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 97e1bc613..0b77d6e39 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -14,7 +14,10 @@ //! Vault-based KMS backend implementation using vaultrs -use crate::backends::vault_credentials::{VaultClientHandle, VaultConnectionSettings, VaultCredentialProvider, token_source_for}; +use crate::backends::vault_credentials::{ + CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider, + token_source_for, +}; use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -32,7 +35,7 @@ use vaultrs::{api::kv2::requests::SetSecretRequestOptions, error::ClientError, k /// Vault KMS client implementation pub struct VaultKmsClient { - credentials: VaultCredentialProvider, + credentials: Arc<VaultCredentialProvider>, config: VaultConfig, /// Mount path for the KV engine (typically "kv" or "secret") kv_mount: String, @@ -161,15 +164,18 @@ fn decode_stored_key_material(key_id: &str, encrypted_material: &str) -> Result< impl VaultKmsClient { /// Create a new Vault KMS client /// - /// `attempt_timeout` caps every HTTP request issued through this client. - pub async fn new(config: VaultConfig, attempt_timeout: Duration) -> Result<Self> { - let source = token_source_for(&config.auth_method)?; + /// `kms_config` supplies the per-attempt timeout that caps every HTTP + /// request issued through this client, plus the retry and fail-closed + /// budgets for credential refresh. + pub async fn new(config: VaultConfig, kms_config: &KmsConfig) -> Result<Self> { let settings = VaultConnectionSettings { address: config.address.clone(), namespace: config.namespace.clone(), - attempt_timeout, + attempt_timeout: kms_config.effective_timeout(), }; - let credentials = VaultCredentialProvider::new(settings, source).await?; + let source = token_source_for(&config.auth_method, &settings)?; + let policy = VaultCredentialPolicy::from_kms_config(kms_config, &config.auth_method); + let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?); info!(address = %config.address, "Vault KMS backend connected"); @@ -185,8 +191,9 @@ impl VaultKmsClient { /// Snapshot the authenticated Vault client for a single request. /// /// Every Vault call takes its own snapshot so a credential rotation - /// applies to subsequent calls without interrupting in-flight ones. - fn vault(&self) -> Arc<VaultClientHandle> { + /// applies to subsequent calls without interrupting in-flight ones. Fails + /// closed when the credentials could not be refreshed in time. + fn vault(&self) -> Result<Arc<VaultClientHandle>> { self.credentials.current() } @@ -231,7 +238,7 @@ impl VaultKmsClient { let path = self.key_version_path(key_id, version); let record: VaultKeyVersionRecord = - kv2::read(&self.vault().client, &self.kv_mount, &path) + kv2::read(&self.vault()?.client, &self.kv_mount, &path) .await .map_err(|e| match e { ClientError::ResponseWrapError => KmsError::key_version_not_found(key_id, version), @@ -271,7 +278,7 @@ impl VaultKmsClient { async fn get_key_data_versioned(&self, key_id: &str) -> Result<(u32, VaultKeyData)> { let path = self.key_path(key_id); - let metadata = kv2::read_metadata(&self.vault().client, &self.kv_mount, &path) + let metadata = kv2::read_metadata(&self.vault()?.client, &self.kv_mount, &path) .await .map_err(|e| match e { ClientError::ResponseWrapError => KmsError::key_not_found(key_id), @@ -283,7 +290,7 @@ impl VaultKmsClient { // Read the exact secret version from the metadata to keep the (cas, data) // pair consistent even if another writer lands in between. - let key_data: VaultKeyData = kv2::read_version(&self.vault().client, &self.kv_mount, &path, metadata.current_version) + let key_data: VaultKeyData = kv2::read_version(&self.vault()?.client, &self.kv_mount, &path, metadata.current_version) .await .map_err(|e| match e { ClientError::ResponseWrapError => KmsError::key_not_found(key_id), @@ -303,7 +310,7 @@ impl VaultKmsClient { let path = self.key_path(key_id); let written = - kv2::set_with_options(&self.vault().client, &self.kv_mount, &path, key_data, SetSecretRequestOptions { cas }) + kv2::set_with_options(&self.vault()?.client, &self.kv_mount, &path, key_data, SetSecretRequestOptions { cas }) .await .map_err(|e| { if is_cas_conflict(&e) { @@ -327,7 +334,8 @@ impl VaultKmsClient { async fn try_create_key_version_record(&self, key_id: &str, record: &VaultKeyVersionRecord) -> Result<bool> { let path = self.key_version_path(key_id, record.version); - match kv2::set_with_options(&self.vault().client, &self.kv_mount, &path, record, SetSecretRequestOptions { cas: 0 }).await + match kv2::set_with_options(&self.vault()?.client, &self.kv_mount, &path, record, SetSecretRequestOptions { cas: 0 }) + .await { Ok(_) => Ok(true), Err(e) if is_cas_conflict(&e) => Ok(false), @@ -339,7 +347,7 @@ impl VaultKmsClient { async fn store_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> { let path = self.key_path(key_id); - kv2::set(&self.vault().client, &self.kv_mount, &path, key_data) + kv2::set(&self.vault()?.client, &self.kv_mount, &path, key_data) .await .map_err(|e| KmsError::backend_error(format!("Failed to store key in Vault: {e}")))?; @@ -389,7 +397,7 @@ impl VaultKmsClient { async fn get_key_data(&self, key_id: &str) -> Result<VaultKeyData> { let path = self.key_path(key_id); - let secret: VaultKeyData = kv2::read(&self.vault().client, &self.kv_mount, &path) + let secret: VaultKeyData = kv2::read(&self.vault()?.client, &self.kv_mount, &path) .await .map_err(|e| match e { vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id), @@ -404,7 +412,7 @@ impl VaultKmsClient { /// List all keys stored in Vault async fn list_vault_keys(&self) -> Result<Vec<String>> { // List keys under the prefix - match kv2::list(&self.vault().client, &self.kv_mount, &self.key_path_prefix).await { + match kv2::list(&self.vault()?.client, &self.kv_mount, &self.key_path_prefix).await { Ok(keys) => { let keys = filter_key_directory_entries(keys); debug!("Found {} keys in Vault", keys.len()); @@ -431,11 +439,11 @@ impl VaultKmsClient { // record still exists and the deletion can be retried. The reverse order // would leave orphaned master key material in Vault after the key vanished. let versions_dir = self.key_versions_dir(key_id); - match kv2::list(&self.vault().client, &self.kv_mount, &versions_dir).await { + match kv2::list(&self.vault()?.client, &self.kv_mount, &versions_dir).await { Ok(versions) => { for version in versions { let version_path = format!("{versions_dir}/{version}"); - kv2::delete_metadata(&self.vault().client, &self.kv_mount, &version_path) + kv2::delete_metadata(&self.vault()?.client, &self.kv_mount, &version_path) .await .map_err(|e| KmsError::backend_error(format!("Failed to delete key version record from Vault: {e}")))?; } @@ -447,7 +455,7 @@ impl VaultKmsClient { // For this specific key path, we can safely delete the metadata // since each key has its own unique path under the prefix - kv2::delete_metadata(&self.vault().client, &self.kv_mount, &path) + kv2::delete_metadata(&self.vault()?.client, &self.kv_mount, &path) .await .map_err(|e| match e { vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), @@ -865,10 +873,17 @@ impl VaultKmsBackend { } }; - let client = VaultKmsClient::new(vault_config, config.effective_timeout()).await?; + let client = VaultKmsClient::new(vault_config, &config).await?; Ok(Self { client }) } + /// Spawn the background credential renewal task for this backend, if its + /// auth method issues lease-bound tokens. The caller owns the returned + /// handle; dropping it cancels the task. + pub(crate) fn spawn_credential_renewal(&self) -> Option<CredentialTaskHandle> { + self.client.credentials.spawn_renewal_task() + } + /// Update key metadata in Vault storage async fn update_key_metadata_in_storage(&self, key_id: &str, metadata: &KeyMetadata) -> Result<()> { // Get the current key data from Vault @@ -1167,7 +1182,7 @@ mod tests { tls: None, }; - let client = VaultKmsClient::new(config, Duration::from_secs(30)) + let client = VaultKmsClient::new(config, &KmsConfig::default()) .await .expect("Failed to create Vault client"); @@ -1220,7 +1235,7 @@ mod tests { #[tokio::test] async fn test_key_version_paths_stay_under_the_key() { - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); @@ -1305,7 +1320,7 @@ mod tests { #[tokio::test] async fn test_vault_kv2_backend_info_reports_at_rest_protection() { - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); @@ -1337,7 +1352,7 @@ mod tests { #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) async fn test_vault_kv2_decrypt_after_rotate() { - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); @@ -1374,7 +1389,7 @@ mod tests { #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) async fn test_vault_kv2_rotate_does_not_orphan_legacy_envelopes() { - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); @@ -1413,7 +1428,7 @@ mod tests { #[tokio::test] #[ignore] // Requires a running Vault instance (dev mode) async fn test_vault_kv2_envelope_version_tampering_fails_closed() { - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); @@ -1456,7 +1471,7 @@ mod tests { use std::sync::Arc; let client = Arc::new( - VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"), ); @@ -1515,7 +1530,7 @@ mod tests { // Regression: get_key_material previously "self-healed" a decrypt/length failure by // minting a fresh random master key and overwriting the stored value — destroying the // original key and making every DEK wrapped by it permanently undecryptable. - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); @@ -1553,7 +1568,7 @@ mod tests { // bootstrap case and silently generated + persisted a fresh master key on the // read path. Empty material must instead fail closed as MaterialMissing and // leave the stored record untouched. - let client = VaultKmsClient::new(integration_vault_config(), Duration::from_secs(30)) + let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) .await .expect("client"); diff --git a/crates/kms/src/backends/vault_credentials.rs b/crates/kms/src/backends/vault_credentials.rs index 486100dcd..066e0bb3a 100644 --- a/crates/kms/src/backends/vault_credentials.rs +++ b/crates/kms/src/backends/vault_credentials.rs @@ -17,21 +17,74 @@ //! [`VaultCredentialProvider`] owns the authenticated [`VaultClient`] and hands //! out per-request snapshots. Backends take a fresh snapshot via //! [`VaultCredentialProvider::current`] for every Vault call instead of holding -//! a client for their own lifetime: a future credential rotation then applies -//! to the next call, while calls already in flight finish on the generation -//! they captured (their `Arc` keeps it alive). +//! a client for their own lifetime: a credential rotation applies to the next +//! call, while calls already in flight finish on the generation they captured +//! (their `Arc` keeps it alive). +//! +//! Lease-bound tokens (AppRole) are kept fresh by a background renewal task +//! (see [`VaultCredentialProvider::spawn_renewal_task`]): it renews at half the +//! lease TTL, falls back to a fresh login when renewal is not possible, and +//! keeps retrying after failures. If the token still reaches the configured +//! safety window before expiry, [`VaultCredentialProvider::current`] fails +//! closed rather than handing out a token that may lapse mid-request. use std::fmt; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use arc_swap::ArcSwap; use async_trait::async_trait; +use sha2::Digest; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use tracing::{info, warn}; use vaultrs::client::{VaultClient, VaultClientSettingsBuilder}; use zeroize::{Zeroize, ZeroizeOnDrop}; -use crate::config::{VaultAuthMethod, redacted_secret}; +use crate::config::{KmsConfig, VaultAuthMethod, redacted_secret}; use crate::error::{KmsError, Result}; +use crate::policy::{self, AttemptError, ErrorClass, OpClass, RetryPolicy}; + +/// Result of a single authentication attempt, classified for the Auth retry +/// policy. +type AttemptResult<T> = std::result::Result<T, AttemptError>; + +/// Cadence for refresh retries after a failed cycle (on top of the bounded +/// retries inside one [`policy::execute`] call). +const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5); + +/// Default seconds between token file re-reads for [`TokenFileSource`]. +const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30; + +/// A crate-owned secret value, zeroized on drop and redacted in Debug output. +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +pub(crate) struct SecretString(String); + +impl SecretString { + pub(crate) fn new(value: String) -> Self { + Self(value) + } + + pub(crate) fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for SecretString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(redacted_secret(&self.0)) + } +} + +/// Expiry attributes of a lease-bound token. +#[derive(Debug, Clone, Copy)] +pub(crate) struct LeaseInfo { + /// Time-to-live granted at issue or renewal. + pub(crate) ttl: Duration, + /// Whether `renew-self` can extend this token. + pub(crate) renewable: bool, +} /// A Vault token handed out by a [`TokenSource`]. /// @@ -39,52 +92,87 @@ use crate::error::{KmsError, Result}; /// copy `vaultrs` keeps inside its client settings (or the HTTP headers built /// from it); it bounds how long the token lingers in memory owned by this /// module. -/// -/// AppRole login (PR-2) will extend this with the lease metadata returned by -/// the login endpoint (`lease_duration`, `renewable`, accessor). #[derive(Clone, Zeroize, ZeroizeOnDrop)] pub(crate) struct TokenLease { token: String, + /// `None` for tokens without an expiry (static configuration tokens and + /// non-expiring root-like tokens). + #[zeroize(skip)] + lease: Option<LeaseInfo>, } impl TokenLease { - pub(crate) fn new(token: String) -> Self { - Self { token } + pub(crate) fn new(token: String, lease: Option<LeaseInfo>) -> Self { + Self { token, lease } + } + + /// Map a Vault auth response onto a lease. A `lease_duration` of zero + /// means the token never expires, so no lease is tracked and no renewal is + /// scheduled. + fn from_auth(auth: vaultrs::api::AuthInfo) -> Self { + let lease = (auth.lease_duration > 0).then_some(LeaseInfo { + ttl: Duration::from_secs(auth.lease_duration), + renewable: auth.renewable, + }); + Self { + token: auth.client_token, + lease, + } } /// Expose the raw token for handing to the Vault client builder. pub(crate) fn expose(&self) -> &str { &self.token } + + pub(crate) fn lease_info(&self) -> Option<LeaseInfo> { + self.lease + } } impl fmt::Debug for TokenLease { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TokenLease") .field("token", &redacted_secret(&self.token)) + .field("lease", &self.lease) .finish() } } +/// Wrap a `vaultrs` failure as a classified attempt failure. +fn attempt_error(operation: &str, error: vaultrs::error::ClientError) -> AttemptError { + AttemptError { + class: policy::classify_vaultrs(&error), + error: KmsError::backend_error(format!("Vault {operation} failed: {error}")), + } +} + /// Source of Vault authentication tokens. /// -/// Only [`StaticToken`] exists today. The trait is async and fallible so -/// future sources can perform I/O when acquiring a token without changing the -/// provider: -/// - `AppRoleLogin` (PR-2): performs an `auth/approle/login` round trip and -/// returns the issued token with its lease metadata; -/// - `TokenFile` (PR-3): re-reads an agent-managed token file. +/// Implementations perform one attempt per call; bounded retries and backoff +/// are owned by the caller through [`policy::execute`] with [`OpClass::Auth`]. +/// Current sources are [`StaticToken`], [`AppRoleLogin`], and the agent-managed +/// [`TokenFileSource`]. #[async_trait] pub(crate) trait TokenSource: fmt::Debug + Send + Sync { - /// Acquire a token for a new client generation. + /// One login attempt yielding a token for a new client generation. + async fn acquire(&self) -> AttemptResult<TokenLease>; + + /// One `renew-self` attempt for the token held by `client`. /// - /// Called once at provider construction today; rotation (PR-2) will call - /// it again for every re-authentication. - async fn acquire(&self) -> Result<TokenLease>; + /// Sources whose tokens cannot be renewed fail fatally; callers fall back + /// to [`TokenSource::acquire`]. + async fn renew(&self, _client: &VaultClient) -> AttemptResult<TokenLease> { + Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::invalid_operation("this token source does not support renewal"), + }) + } } /// Token source for [`VaultAuthMethod::Token`]: always yields the token fixed -/// at configuration time. +/// at configuration time. The token carries no lease, so it is never renewed +/// and never expires from the provider's point of view. pub(crate) struct StaticToken { token: TokenLease, } @@ -92,14 +180,14 @@ pub(crate) struct StaticToken { impl StaticToken { pub(crate) fn new(token: String) -> Self { Self { - token: TokenLease::new(token), + token: TokenLease::new(token, None), } } } #[async_trait] impl TokenSource for StaticToken { - async fn acquire(&self) -> Result<TokenLease> { + async fn acquire(&self) -> AttemptResult<TokenLease> { Ok(self.token.clone()) } } @@ -111,16 +199,247 @@ impl fmt::Debug for StaticToken { } } -/// Map the configured auth method onto a token source. +/// Token source for [`VaultAuthMethod::AppRole`]: exchanges `role_id` + +/// `secret_id` for a lease-bound token via the AppRole auth engine. +pub(crate) struct AppRoleLogin { + /// Unauthenticated client used only for the login exchange. + login_client: VaultClient, + mount: String, + role_id: String, + /// Inline secret_id fallback, used when no file is configured. + secret_id: SecretString, + /// Secret-id file, re-read on every login so external rotation of the + /// secret_id is picked up without a restart. Takes precedence over the + /// inline value. + secret_id_file: Option<PathBuf>, +} + +impl AppRoleLogin { + pub(crate) fn new( + settings: &VaultConnectionSettings, + mount: String, + role_id: String, + secret_id: String, + secret_id_file: Option<PathBuf>, + ) -> Result<Self> { + Ok(Self { + login_client: settings.build_login_client()?, + mount, + role_id, + secret_id: SecretString::new(secret_id), + secret_id_file, + }) + } + + /// Resolve the secret_id for one login attempt. + /// + /// File problems are fatal for the attempt (replaying the same read within + /// one retry cycle cannot help), but the renewal loop keeps retrying on + /// its cadence, so repairing the file heals the source without a restart. + async fn resolve_secret_id(&self) -> AttemptResult<SecretString> { + let Some(path) = &self.secret_id_file else { + return Ok(self.secret_id.clone()); + }; + + let mut raw = tokio::fs::read_to_string(path).await.map_err(|error| AttemptError { + class: ErrorClass::Fatal, + error: KmsError::configuration_error(format!("Failed to read AppRole secret_id file {}: {error}", path.display())), + })?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + raw.zeroize(); + return Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::configuration_error(format!("AppRole secret_id file {} is empty", path.display())), + }); + } + let secret_id = SecretString::new(trimmed.to_string()); + raw.zeroize(); + Ok(secret_id) + } +} + +#[async_trait] +impl TokenSource for AppRoleLogin { + async fn acquire(&self) -> AttemptResult<TokenLease> { + let secret_id = self.resolve_secret_id().await?; + let auth = vaultrs::auth::approle::login(&self.login_client, &self.mount, &self.role_id, secret_id.expose()) + .await + .map_err(|error| attempt_error("AppRole login", error))?; + Ok(TokenLease::from_auth(auth)) + } + + async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> { + let auth = vaultrs::token::renew_self(client, None) + .await + .map_err(|error| attempt_error("token renewal", error))?; + Ok(TokenLease::from_auth(auth)) + } +} + +impl fmt::Debug for AppRoleLogin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The login client embeds Vault client settings and must stay out of + // Debug output; role_id is not a secret in Vault's AppRole model. + f.debug_struct("AppRoleLogin") + .field("mount", &self.mount) + .field("role_id", &self.role_id) + .field("secret_id", &self.secret_id) + .field("secret_id_file", &self.secret_id_file) + .finish_non_exhaustive() + } +} + +/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed +/// token file (for example a Vault Agent auto-auth sink). /// -/// AppRole is still rejected at construction time; PR-2 replaces this arm with -/// an `AppRoleLogin` source. -pub(crate) fn token_source_for(auth_method: &VaultAuthMethod) -> Result<Box<dyn TokenSource>> { +/// The agent owns the token's lifecycle; this source only tracks the file. +/// Every read grants the token an observed validity of `validity`, so the +/// renewal loop re-reads the file at half that interval and installs a new +/// client generation from whatever the file holds. A file that disappears or +/// turns empty keeps failing the refresh until the fail-closed window trips, +/// and heals the provider as soon as it is restored. +pub(crate) struct TokenFileSource { + path: PathBuf, + /// Observed validity granted per successful read; the renewal loop + /// re-reads at half this value. + validity: Duration, + /// Digest of the last token read, to tell agent-driven rotations apart + /// from plain refreshes in the logs. Never holds the token itself. + last_digest: std::sync::Mutex<Option<[u8; 32]>>, +} + +impl TokenFileSource { + pub(crate) fn new(path: PathBuf, validity: Duration) -> Self { + Self { + path, + validity, + last_digest: std::sync::Mutex::new(None), + } + } + + fn fatal(error: KmsError) -> AttemptError { + AttemptError { + class: ErrorClass::Fatal, + error, + } + } + + /// Reject a token file readable by group or other, mirroring the SFTP + /// host-key rule: a Vault token grants use of the KMS keys, so a mode + /// wider than owner-only is a deployment error, not a warning. + #[cfg(unix)] + fn check_permissions(&self, metadata: &std::fs::Metadata) -> AttemptResult<()> { + use std::os::unix::fs::PermissionsExt; + let mode = metadata.permissions().mode() & 0o777; + if mode & 0o077 != 0 { + return Err(Self::fatal(KmsError::configuration_error(format!( + "Vault token file {} has insecure permissions {mode:#o} (group and other permission bits must be unset)", + self.path.display() + )))); + } + Ok(()) + } +} + +#[async_trait] +impl TokenSource for TokenFileSource { + async fn acquire(&self) -> AttemptResult<TokenLease> { + // Synchronous reads on purpose: the token file is tiny, this runs at + // the renewal cadence, and staying off the blocking pool keeps the + // paused-clock tests of the renewal timing deterministic. + let metadata = std::fs::metadata(&self.path).map_err(|error| { + Self::fatal(KmsError::configuration_error(format!( + "Failed to read Vault token file {}: {error}", + self.path.display() + ))) + })?; + #[cfg(unix)] + self.check_permissions(&metadata)?; + + let mut raw = std::fs::read_to_string(&self.path).map_err(|error| { + Self::fatal(KmsError::configuration_error(format!( + "Failed to read Vault token file {}: {error}", + self.path.display() + ))) + })?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + raw.zeroize(); + return Err(Self::fatal(KmsError::configuration_error(format!( + "Vault token file {} is empty", + self.path.display() + )))); + } + + let digest: [u8; 32] = sha2::Sha256::digest(trimmed.as_bytes()).into(); + let previous = self + .last_digest + .lock() + .expect("token file digest mutex poisoned") + .replace(digest); + if previous.is_some_and(|previous| previous != digest) { + info!( + path = %self.path.display(), + modified = ?metadata.modified().ok(), + "Vault token file rotated; installing a new client generation" + ); + } + + let token = trimmed.to_string(); + raw.zeroize(); + Ok(TokenLease::new( + token, + Some(LeaseInfo { + ttl: self.validity, + renewable: false, + }), + )) + } +} + +impl fmt::Debug for TokenFileSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // The token itself is never stored on the source; only its digest is. + f.debug_struct("TokenFileSource") + .field("path", &self.path) + .field("validity", &self.validity) + .finish_non_exhaustive() + } +} + +/// Map the configured auth method onto a token source. +pub(crate) fn token_source_for( + auth_method: &VaultAuthMethod, + settings: &VaultConnectionSettings, +) -> Result<Box<dyn TokenSource>> { match auth_method { VaultAuthMethod::Token { token } => Ok(Box::new(StaticToken::new(token.clone()))), - VaultAuthMethod::AppRole { .. } => Err(KmsError::backend_error( - "AppRole authentication not yet implemented. Please use token authentication.", - )), + VaultAuthMethod::AppRole { + role_id, + secret_id, + secret_id_file, + mount, + .. + } => Ok(Box::new(AppRoleLogin::new( + settings, + mount.clone(), + role_id.clone(), + secret_id.clone(), + secret_id_file.clone(), + )?)), + VaultAuthMethod::TokenFile { + path, + poll_interval_secs, + .. + } => { + let poll_interval = Duration::from_secs(poll_interval_secs.unwrap_or(DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS).max(1)); + // Validity is twice the poll interval because the renewal loop + // fires at half the lease TTL: the file is then re-read once per + // poll interval, and a file that stops being readable expires the + // token after roughly two missed polls minus the safety window. + Ok(Box::new(TokenFileSource::new(path.clone(), poll_interval * 2))) + } } } @@ -135,7 +454,7 @@ pub(crate) struct VaultConnectionSettings { impl VaultConnectionSettings { /// Build an authenticated client for one generation. - fn build_client(&self, token: &TokenLease) -> Result<VaultClient> { + fn build_client(&self, token: &str) -> Result<VaultClient> { let mut settings_builder = VaultClientSettingsBuilder::default(); settings_builder.address(&self.address); // Defense in depth against stalled connections: vaultrs leaves the @@ -143,7 +462,7 @@ impl VaultConnectionSettings { // request would otherwise wait forever regardless of the // operation-level retry policy. settings_builder.timeout(Some(self.attempt_timeout)); - settings_builder.token(token.expose()); + settings_builder.token(token); if let Some(namespace) = &self.namespace { settings_builder.namespace(Some(namespace.clone())); @@ -155,6 +474,51 @@ impl VaultConnectionSettings { VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}"))) } + + /// Build the tokenless client used for login exchanges. + fn build_login_client(&self) -> Result<VaultClient> { + self.build_client("") + } +} + +/// Refresh and fail-closed tuning for a [`VaultCredentialProvider`]. +#[derive(Debug, Clone)] +pub(crate) struct VaultCredentialPolicy { + /// Retry budget for one login/renewal cycle. + pub(crate) retry: RetryPolicy, + /// Fail-closed margin: once the current token is within this window of + /// expiry without a successful refresh, [`VaultCredentialProvider::current`] + /// refuses to hand it out. + pub(crate) safety_window: Duration, + /// Pause between refresh cycles after a failed one. + pub(crate) retry_interval: Duration, +} + +impl VaultCredentialPolicy { + /// Derive the policy from the KMS configuration. + /// + /// The default safety window equals the per-attempt timeout: a request + /// issued now can stay in flight for up to one attempt timeout, so the + /// token must outlive at least that. + pub(crate) fn from_kms_config(config: &KmsConfig, auth_method: &VaultAuthMethod) -> Self { + let retry = RetryPolicy::from_config(config); + let safety_window = match auth_method { + VaultAuthMethod::AppRole { + refresh_safety_window_secs: Some(secs), + .. + } + | VaultAuthMethod::TokenFile { + refresh_safety_window_secs: Some(secs), + .. + } => Duration::from_secs(*secs), + _ => retry.attempt_timeout, + }; + Self { + retry, + safety_window, + retry_interval: DEFAULT_REFRESH_RETRY_INTERVAL, + } + } } /// One authenticated client generation. @@ -164,10 +528,27 @@ impl VaultConnectionSettings { /// client out from under an in-flight request. pub(crate) struct VaultClientHandle { /// Monotonic counter identifying the credential generation this client was - /// built from. Static tokens never rotate, so only generation 0 exists - /// today; rotation (PR-2) bumps it on every re-authentication. + /// built from; bumped on every successful refresh. pub(crate) generation: u64, pub(crate) client: VaultClient, + /// When this generation's token was issued (or last renewed). + issued_at: Instant, + /// Lease of this generation's token; `None` when it never expires. + lease: Option<LeaseInfo>, +} + +impl VaultClientHandle { + /// Absolute expiry of this generation's token. + fn expires_at(&self) -> Option<Instant> { + self.lease.map(|lease| self.issued_at + lease.ttl) + } + + /// When the renewal task should refresh this generation: half the TTL, + /// leaving the second half as budget for retries before the fail-closed + /// window is reached. + fn renew_at(&self) -> Option<Instant> { + self.lease.map(|lease| self.issued_at + lease.ttl / 2) + } } impl fmt::Debug for VaultClientHandle { @@ -176,46 +557,215 @@ impl fmt::Debug for VaultClientHandle { // never appear in Debug output. f.debug_struct("VaultClientHandle") .field("generation", &self.generation) + .field("lease", &self.lease) .finish_non_exhaustive() } } /// Owns the authenticated Vault client for a backend and hands out /// per-request snapshots. -/// -/// The provider keeps neither the settings nor the source after construction -/// because a static token can never be refreshed. Rotation (PR-2) will retain -/// both and add a refresh path that acquires a fresh lease, rebuilds the -/// client, and stores it under a bumped generation. pub(crate) struct VaultCredentialProvider { + settings: VaultConnectionSettings, + source: Box<dyn TokenSource>, + policy: VaultCredentialPolicy, current: ArcSwap<VaultClientHandle>, + /// Serializes refreshes so concurrent triggers coalesce into one login. + refresh_lock: tokio::sync::Mutex<()>, } impl VaultCredentialProvider { /// Authenticate with `source` and build the initial client generation. - pub(crate) async fn new(settings: VaultConnectionSettings, source: Box<dyn TokenSource>) -> Result<Self> { - let lease = source.acquire().await?; - let client = settings.build_client(&lease)?; + pub(crate) async fn new( + settings: VaultConnectionSettings, + source: Box<dyn TokenSource>, + policy: VaultCredentialPolicy, + ) -> Result<Self> { + let startup_cancel = CancellationToken::new(); + let lease = policy::execute("vault_login", OpClass::Auth, &policy.retry, &startup_cancel, || source.acquire()).await?; + let client = settings.build_client(lease.expose())?; Ok(Self { - current: ArcSwap::from_pointee(VaultClientHandle { generation: 0, client }), + current: ArcSwap::from_pointee(VaultClientHandle { + generation: 0, + client, + issued_at: Instant::now(), + lease: lease.lease_info(), + }), + settings, + source, + policy, + refresh_lock: tokio::sync::Mutex::new(()), }) } - /// Snapshot the current client generation. + /// Snapshot the current generation without the expiry gate. Internal use + /// (renewal scheduling) and tests only; request paths go through + /// [`VaultCredentialProvider::current`]. + pub(crate) fn snapshot(&self) -> Arc<VaultClientHandle> { + self.current.load_full() + } + + /// Snapshot the current client generation for a single request. /// /// Take one snapshot per Vault call: the returned `Arc` pins the /// generation for exactly that call, so a concurrent rotation applies to /// the next call without interrupting this one. - pub(crate) fn current(&self) -> Arc<VaultClientHandle> { - self.current.load_full() + /// + /// Fails closed when the token is inside the safety window of its expiry: + /// a request signed with such a token could lapse mid-flight, so refusing + /// it locally is strictly safer than an unpredictable remote failure. + pub(crate) fn current(&self) -> Result<Arc<VaultClientHandle>> { + let handle = self.current.load_full(); + if let Some(expires_at) = handle.expires_at() { + let now = Instant::now(); + if now + self.policy.safety_window >= expires_at { + return Err(KmsError::credentials_unavailable(format!( + "Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it", + handle.generation, self.policy.safety_window + ))); + } + } + Ok(handle) + } + + /// Refresh the credentials if generation `observed` is still current. + /// + /// Single-flight: concurrent callers serialize on the refresh lock, and a + /// caller that finds a newer generation already installed returns without + /// touching Vault. Renewable tokens are renewed in place; anything else + /// (or a failed renewal) falls back to a fresh login. + pub(crate) async fn refresh(&self, observed: u64, cancel: &CancellationToken) -> Result<()> { + let _guard = self.refresh_lock.lock().await; + let current = self.snapshot(); + if current.generation != observed { + return Ok(()); + } + + let renewable = current.lease.map(|lease| lease.renewable).unwrap_or(false); + let renewed = if renewable { + match policy::execute("vault_token_renew", OpClass::Auth, &self.policy.retry, cancel, || { + self.source.renew(¤t.client) + }) + .await + { + Ok(lease) => Some(lease), + Err(error @ KmsError::OperationCancelled { .. }) => return Err(error), + Err(error) => { + warn!( + generation = current.generation, + error = %error, + "Vault token renewal failed; falling back to a fresh login" + ); + None + } + } + } else { + None + }; + + let lease = match renewed { + Some(lease) => lease, + None => policy::execute("vault_login", OpClass::Auth, &self.policy.retry, cancel, || self.source.acquire()).await?, + }; + + let client = self.settings.build_client(lease.expose())?; + self.current.store(Arc::new(VaultClientHandle { + generation: current.generation + 1, + client, + issued_at: Instant::now(), + lease: lease.lease_info(), + })); + Ok(()) + } + + /// Spawn the background renewal task for lease-bound credentials. + /// + /// Returns `None` when the current token never expires (static tokens): + /// there is nothing to renew. The returned handle cancels the task when + /// dropped, tying the task's lifetime to whoever owns the handle (the + /// service version that owns this backend). + pub(crate) fn spawn_renewal_task(self: &Arc<Self>) -> Option<CredentialTaskHandle> { + self.snapshot().lease?; + let cancel = CancellationToken::new(); + let join = tokio::spawn(renewal_loop(Arc::clone(self), cancel.clone())); + Some(CredentialTaskHandle { + cancel, + join: std::sync::Mutex::new(Some(join)), + }) } } impl fmt::Debug for VaultCredentialProvider { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("VaultCredentialProvider") + .field("source", &self.source) .field("current", &self.current.load()) - .finish() + .field("policy", &self.policy) + .finish_non_exhaustive() + } +} + +/// Drive credential refreshes until cancelled. +/// +/// Each cycle sleeps until the current generation's renewal point (half TTL), +/// then refreshes. A failed cycle logs, waits `retry_interval`, and tries +/// again immediately (the renewal point is already in the past), so the +/// provider keeps trying to recover even after the fail-closed window has +/// been reached. +async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: CancellationToken) { + loop { + let handle = provider.snapshot(); + let Some(renew_at) = handle.renew_at() else { + // The current generation never expires; nothing left to schedule. + return; + }; + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = tokio::time::sleep_until(renew_at) => {} + } + + match provider.refresh(handle.generation, &cancel).await { + Ok(()) => {} + Err(KmsError::OperationCancelled { .. }) => return, + Err(error) => { + warn!( + generation = handle.generation, + error = %error, + "Vault credential refresh failed; retrying until the credentials recover" + ); + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = tokio::time::sleep(provider.policy.retry_interval) => {} + } + } + } + } +} + +/// Owner handle for a spawned renewal task. +/// +/// Dropping the handle cancels the task, so hanging it off the service version +/// recycles the task on stop and reconfigure without explicit lifecycle calls. +pub(crate) struct CredentialTaskHandle { + cancel: CancellationToken, + join: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>, +} + +impl CredentialTaskHandle { + /// Cancel the renewal task and wait for it to exit. + pub(crate) async fn shutdown(&self) { + self.cancel.cancel(); + let join = self.join.lock().expect("credential task join mutex poisoned").take(); + if let Some(join) = join { + let _ = join.await; + } + } +} + +impl Drop for CredentialTaskHandle { + fn drop(&mut self) { + self.cancel.cancel(); } } @@ -223,8 +773,10 @@ impl fmt::Debug for VaultCredentialProvider { mod tests { use super::*; use crate::config::REDACTED_SECRET; + use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; const TEST_TOKEN: &str = "vault-token-debug-leak-canary"; + const TEST_SECRET_ID: &str = "approle-secret-id-leak-canary"; fn test_settings() -> VaultConnectionSettings { VaultConnectionSettings { @@ -234,19 +786,113 @@ mod tests { } } - /// Building a provider never contacts Vault, so these tests run offline. - async fn test_provider() -> VaultCredentialProvider { - VaultCredentialProvider::new(test_settings(), Box::new(StaticToken::new(TEST_TOKEN.to_string()))) + /// Tight retry budget so paused-clock tests stay deterministic: one + /// attempt per cycle, failed cycles spaced by `retry_interval`. + fn test_policy(safety_window: Duration, retry_interval: Duration) -> VaultCredentialPolicy { + VaultCredentialPolicy { + retry: RetryPolicy { + attempt_timeout: Duration::from_secs(1), + op_deadline: Duration::from_secs(1), + max_attempts: 1, + base_backoff: Duration::from_millis(10), + max_backoff: Duration::from_millis(10), + }, + safety_window, + retry_interval, + } + } + + /// Shared observable state of a [`ScriptedSource`]. + #[derive(Debug, Default)] + struct ScriptedState { + login_calls: AtomicU32, + renew_calls: AtomicU32, + fail_login: AtomicBool, + fail_renew: AtomicBool, + } + + /// Token source with scriptable outcomes for driving the provider without + /// a Vault server. + #[derive(Debug)] + struct ScriptedSource { + state: Arc<ScriptedState>, + ttl: Duration, + renewable: bool, + login_delay: Duration, + } + + impl ScriptedSource { + fn lease(&self) -> Option<LeaseInfo> { + (!self.ttl.is_zero()).then_some(LeaseInfo { + ttl: self.ttl, + renewable: self.renewable, + }) + } + + fn failure() -> AttemptError { + AttemptError { + class: ErrorClass::RetryableStatus, + error: KmsError::backend_error("scripted auth failure (503)"), + } + } + } + + #[async_trait] + impl TokenSource for ScriptedSource { + async fn acquire(&self) -> AttemptResult<TokenLease> { + if !self.login_delay.is_zero() { + tokio::time::sleep(self.login_delay).await; + } + let call = self.state.login_calls.fetch_add(1, Ordering::SeqCst); + if self.state.fail_login.load(Ordering::SeqCst) { + return Err(Self::failure()); + } + Ok(TokenLease::new(format!("scripted-login-{call}"), self.lease())) + } + + async fn renew(&self, _client: &VaultClient) -> AttemptResult<TokenLease> { + let call = self.state.renew_calls.fetch_add(1, Ordering::SeqCst); + if self.state.fail_renew.load(Ordering::SeqCst) { + return Err(Self::failure()); + } + Ok(TokenLease::new(format!("scripted-renew-{call}"), self.lease())) + } + } + + async fn scripted_provider( + ttl: Duration, + renewable: bool, + policy: VaultCredentialPolicy, + ) -> (Arc<VaultCredentialProvider>, Arc<ScriptedState>) { + let state = Arc::new(ScriptedState::default()); + let source = ScriptedSource { + state: state.clone(), + ttl, + renewable, + login_delay: Duration::ZERO, + }; + let provider = VaultCredentialProvider::new(test_settings(), Box::new(source), policy) .await - .expect("provider construction must not require a live Vault") + .expect("scripted provider must build without a live Vault"); + (Arc::new(provider), state) + } + + async fn static_provider() -> VaultCredentialProvider { + VaultCredentialProvider::new( + test_settings(), + Box::new(StaticToken::new(TEST_TOKEN.to_string())), + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await + .expect("static provider must build without a live Vault") } #[tokio::test] async fn test_static_token_snapshots_pin_one_generation() { - let provider = test_provider().await; + let provider = static_provider().await; - let first = provider.current(); - let second = provider.current(); + let first = provider.current().expect("static tokens never expire"); + let second = provider.current().expect("static tokens never expire"); assert_eq!(first.generation, 0); assert!( @@ -255,50 +901,457 @@ mod tests { ); } + #[tokio::test] + async fn test_static_token_spawns_no_renewal_task() { + let provider = Arc::new(static_provider().await); + assert!(provider.spawn_renewal_task().is_none(), "a token without a lease has nothing to renew"); + } + #[tokio::test] async fn test_static_token_source_yields_configured_token() { - let source = token_source_for(&VaultAuthMethod::Token { - token: TEST_TOKEN.to_string(), - }) + let settings = test_settings(); + let source = token_source_for( + &VaultAuthMethod::Token { + token: TEST_TOKEN.to_string(), + }, + &settings, + ) .expect("token auth must map to a source"); let lease = source.acquire().await.expect("static acquire cannot fail"); assert_eq!(lease.expose(), TEST_TOKEN); + assert!(lease.lease_info().is_none(), "static tokens must not carry a lease"); } - /// Behavior pin: AppRole keeps failing at construction with the same - /// user-visible message until the login source lands (PR-2). - #[test] - fn test_approle_auth_method_still_rejected() { - let error = token_source_for(&VaultAuthMethod::AppRole { - role_id: "role".to_string(), - secret_id: "approle-secret-canary".to_string(), - }) - .expect_err("approle must stay rejected until the login source lands"); + #[tokio::test] + async fn test_approle_auth_method_maps_to_login_source() { + let settings = test_settings(); + let source = token_source_for(&VaultAuthMethod::approle("role".to_string(), TEST_SECRET_ID.to_string()), &settings) + .expect("approle auth must map to a login source"); - let rendered = error.to_string(); - assert!(rendered.contains("AppRole authentication not yet implemented"), "got: {rendered}"); - assert!(!rendered.contains("approle-secret-canary"), "error must not echo the secret id"); + assert!(format!("{source:?}").contains("AppRoleLogin")); + } + + #[tokio::test(start_paused = true)] + async fn test_renewal_task_renews_at_half_ttl() { + let (provider, state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await; + let task = provider.spawn_renewal_task().expect("lease-bound tokens need renewal"); + + tokio::time::sleep(Duration::from_secs(29)).await; + assert_eq!(state.renew_calls.load(Ordering::SeqCst), 0, "renewal must not run before half TTL"); + assert_eq!(provider.snapshot().generation, 0); + + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(state.renew_calls.load(Ordering::SeqCst), 1, "renewal must run at half TTL"); + assert_eq!(state.login_calls.load(Ordering::SeqCst), 1, "renewable tokens must not re-login"); + assert_eq!(provider.snapshot().generation, 1, "a successful renewal must install a new generation"); + + task.shutdown().await; + } + + #[tokio::test(start_paused = true)] + async fn test_failed_renewal_falls_back_to_login() { + let (provider, state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await; + state.fail_renew.store(true, Ordering::SeqCst); + let task = provider.spawn_renewal_task().expect("renewal task"); + + tokio::time::sleep(Duration::from_secs(31)).await; + assert_eq!(state.renew_calls.load(Ordering::SeqCst), 1, "renewal must be attempted first"); + assert_eq!( + state.login_calls.load(Ordering::SeqCst), + 2, + "failed renewal must fall back to a fresh login" + ); + assert_eq!(provider.snapshot().generation, 1); + + task.shutdown().await; + } + + #[tokio::test(start_paused = true)] + async fn test_current_fails_closed_inside_safety_window_and_recovers() { + let (provider, state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await; + state.fail_renew.store(true, Ordering::SeqCst); + state.fail_login.store(true, Ordering::SeqCst); + let task = provider.spawn_renewal_task().expect("renewal task"); + + // Refresh cycles at 30s, 35s, ... keep failing; the token stays usable + // until 50s (60s TTL minus the 10s safety window). + tokio::time::sleep(Duration::from_secs(49)).await; + provider + .current() + .expect("token outside the safety window must still be served"); + + tokio::time::sleep(Duration::from_secs(2)).await; + let error = provider + .current() + .expect_err("token inside the safety window must be refused"); + assert!( + matches!(error, KmsError::CredentialsUnavailable { .. }), + "expected CredentialsUnavailable, got {error:?}" + ); + + // Recovery: the next retry cycle succeeds, installs a fresh + // generation, and the provider serves requests again. + state.fail_renew.store(false, Ordering::SeqCst); + state.fail_login.store(false, Ordering::SeqCst); + tokio::time::sleep(Duration::from_secs(6)).await; + let handle = provider.current().expect("provider must recover after a successful refresh"); + assert!(handle.generation >= 1); + + task.shutdown().await; + } + + #[tokio::test(start_paused = true)] + async fn test_shutdown_recycles_renewal_task_promptly() { + let (provider, _state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await; + let task = provider.spawn_renewal_task().expect("renewal task"); + + tokio::time::timeout(Duration::from_secs(1), task.shutdown()) + .await + .expect("cancelled renewal task must exit promptly"); + } + + #[tokio::test(start_paused = true)] + async fn test_dropping_task_handle_cancels_renewal_task() { + let (provider, _state) = scripted_provider( + Duration::from_secs(60), + true, + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await; + let task = provider.spawn_renewal_task().expect("renewal task"); + let cancel_probe = task.cancel.clone(); + + drop(task); + + tokio::time::timeout(Duration::from_secs(1), cancel_probe.cancelled()) + .await + .expect("dropping the handle must cancel the renewal task"); + } + + #[tokio::test(start_paused = true)] + async fn test_concurrent_refreshes_coalesce_into_one_login() { + let state = Arc::new(ScriptedState::default()); + let source = ScriptedSource { + state: state.clone(), + ttl: Duration::from_secs(60), + renewable: false, + login_delay: Duration::from_millis(100), + }; + let provider = Arc::new( + VaultCredentialProvider::new( + test_settings(), + Box::new(source), + test_policy(Duration::from_secs(10), Duration::from_secs(5)), + ) + .await + .expect("provider"), + ); + assert_eq!(state.login_calls.load(Ordering::SeqCst), 1, "initial login"); + + let cancel = CancellationToken::new(); + let first = { + let provider = provider.clone(); + let cancel = cancel.clone(); + tokio::spawn(async move { provider.refresh(0, &cancel).await }) + }; + let second = { + let provider = provider.clone(); + let cancel = cancel.clone(); + tokio::spawn(async move { provider.refresh(0, &cancel).await }) + }; + first.await.expect("join").expect("refresh"); + second.await.expect("join").expect("refresh"); + + assert_eq!( + state.login_calls.load(Ordering::SeqCst), + 2, + "concurrent refreshes of the same generation must coalesce into one login" + ); + assert_eq!(provider.snapshot().generation, 1); + } + + #[tokio::test] + async fn test_approle_secret_id_file_missing_fails_fatally() { + let dir = tempfile::tempdir().expect("tempdir"); + let source = AppRoleLogin::new( + &test_settings(), + "approle".to_string(), + "role".to_string(), + String::new(), + Some(dir.path().join("absent-secret-id")), + ) + .expect("source"); + + let failure = source + .resolve_secret_id() + .await + .expect_err("missing secret_id file must fail the attempt"); + assert_eq!(failure.class, ErrorClass::Fatal); + assert!(matches!(failure.error, KmsError::ConfigurationError { .. })); + } + + #[tokio::test] + async fn test_approle_secret_id_file_empty_fails_fatally() { + let file = tempfile::NamedTempFile::new().expect("tempfile"); + std::fs::write(file.path(), " \n\t\n").expect("write whitespace"); + let source = AppRoleLogin::new( + &test_settings(), + "approle".to_string(), + "role".to_string(), + String::new(), + Some(file.path().to_path_buf()), + ) + .expect("source"); + + let failure = source + .resolve_secret_id() + .await + .expect_err("effectively empty secret_id file must fail the attempt"); + assert_eq!(failure.class, ErrorClass::Fatal); + assert!(failure.error.to_string().contains("is empty")); + } + + #[tokio::test] + async fn test_approle_secret_id_file_takes_precedence_and_is_trimmed() { + let file = tempfile::NamedTempFile::new().expect("tempfile"); + std::fs::write(file.path(), format!(" {TEST_SECRET_ID}\n")).expect("write secret"); + let source = AppRoleLogin::new( + &test_settings(), + "approle".to_string(), + "role".to_string(), + "inline-secret-id-must-lose".to_string(), + Some(file.path().to_path_buf()), + ) + .expect("source"); + + let secret_id = source.resolve_secret_id().await.expect("readable file must resolve"); + assert_eq!(secret_id.expose(), TEST_SECRET_ID); } /// Leak regression: the Debug output of every credential-carrying type - /// must stay free of the token literal. + /// must stay free of token and secret_id literals. #[tokio::test] async fn test_credential_types_debug_redacts_token() { - let provider = test_provider().await; - let handle = provider.current(); - let lease = TokenLease::new(TEST_TOKEN.to_string()); - let source = StaticToken::new(TEST_TOKEN.to_string()); + let provider = static_provider().await; + let handle = provider.current().expect("static token"); + let lease = TokenLease::new( + TEST_TOKEN.to_string(), + Some(LeaseInfo { + ttl: Duration::from_secs(60), + renewable: true, + }), + ); + let static_source = StaticToken::new(TEST_TOKEN.to_string()); + let approle_source = AppRoleLogin::new( + &test_settings(), + "approle".to_string(), + "leak-test-role-id".to_string(), + TEST_SECRET_ID.to_string(), + None, + ) + .expect("approle source"); for rendered in [ format!("{provider:?}"), format!("{handle:?}"), format!("{lease:?}"), - format!("{source:?}"), + format!("{static_source:?}"), + format!("{approle_source:?}"), ] { assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}"); + assert!( + !rendered.contains(TEST_SECRET_ID), + "debug output must not leak the approle secret_id: {rendered}" + ); } assert!(format!("{lease:?}").contains(REDACTED_SECRET)); + let approle_rendered = format!("{approle_source:?}"); + assert!(approle_rendered.contains("leak-test-role-id"), "role_id is not a secret"); + assert!(approle_rendered.contains(REDACTED_SECRET)); + } + + /// Write a token file with owner-only permissions, as a Vault Agent sink + /// would. + fn write_owner_only(path: &std::path::Path, contents: &str) { + std::fs::write(path, contents).expect("write token file"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).expect("chmod token file"); + } + } + + async fn token_file_provider(path: std::path::PathBuf, policy: VaultCredentialPolicy) -> Arc<VaultCredentialProvider> { + Arc::new( + VaultCredentialProvider::new(test_settings(), Box::new(TokenFileSource::new(path, Duration::from_secs(60))), policy) + .await + .expect("token file provider must build without a live Vault"), + ) + } + + #[tokio::test] + async fn test_token_file_source_reads_and_trims_token() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("vault-token"); + write_owner_only(&path, &format!(" {TEST_TOKEN}\n")); + let source = TokenFileSource::new(path, Duration::from_secs(60)); + + let lease = source.acquire().await.expect("readable token file must resolve"); + assert_eq!(lease.expose(), TEST_TOKEN); + let lease_info = lease.lease_info().expect("token file leases carry an observed validity"); + assert_eq!(lease_info.ttl, Duration::from_secs(60)); + assert!(!lease_info.renewable, "agent-managed tokens are refreshed by re-reading, not renew-self"); + } + + #[tokio::test] + async fn test_token_file_missing_fails_fatally() { + let dir = tempfile::tempdir().expect("tempdir"); + let source = TokenFileSource::new(dir.path().join("absent-token"), Duration::from_secs(60)); + + let failure = source.acquire().await.expect_err("missing token file must fail the attempt"); + assert_eq!(failure.class, ErrorClass::Fatal); + assert!(matches!(failure.error, KmsError::ConfigurationError { .. })); + } + + #[tokio::test] + async fn test_token_file_empty_fails_fatally() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("vault-token"); + write_owner_only(&path, " \n\t\n"); + let source = TokenFileSource::new(path, Duration::from_secs(60)); + + let failure = source + .acquire() + .await + .expect_err("effectively empty token file must fail the attempt"); + assert_eq!(failure.class, ErrorClass::Fatal); + assert!(failure.error.to_string().contains("is empty")); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_token_file_rejects_group_or_other_permission_bits() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("vault-token"); + write_owner_only(&path, TEST_TOKEN); + let source = TokenFileSource::new(path.clone(), Duration::from_secs(60)); + + for mode in [0o640u32, 0o604, 0o622, 0o660] { + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).expect("chmod"); + let failure = source + .acquire() + .await + .expect_err("token file readable or writable beyond the owner must be rejected"); + assert_eq!(failure.class, ErrorClass::Fatal, "mode {mode:#o}"); + let rendered = failure.error.to_string(); + assert!(rendered.contains("insecure permissions"), "mode {mode:#o}: {rendered}"); + assert!(!rendered.contains(TEST_TOKEN), "permission errors must not echo the token"); + } + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("chmod"); + source.acquire().await.expect("owner-only token file must resolve"); + } + + #[tokio::test(start_paused = true)] + async fn test_token_file_replacement_installs_new_generation_next_cycle() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("vault-token"); + write_owner_only(&path, "agent-token-v1"); + let provider = token_file_provider(path.clone(), test_policy(Duration::from_secs(10), Duration::from_secs(5))).await; + let task = provider.spawn_renewal_task().expect("token file credentials are lease-bound"); + + tokio::time::sleep(Duration::from_secs(29)).await; + assert_eq!(provider.snapshot().generation, 0, "no re-read before half the observed validity"); + + // Atomic replacement, as a Vault Agent sink writes: temp file + rename. + let staged = dir.path().join("vault-token.tmp"); + write_owner_only(&staged, "agent-token-v2"); + std::fs::rename(&staged, &path).expect("atomic replace"); + + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!( + provider.snapshot().generation, + 1, + "the poll after a rotation must install a new generation" + ); + + // A poll without a rotation still installs a fresh generation: each + // successful read re-extends the token's observed validity. + tokio::time::sleep(Duration::from_secs(30)).await; + assert_eq!(provider.snapshot().generation, 2); + + task.shutdown().await; + } + + #[tokio::test(start_paused = true)] + async fn test_token_file_deletion_fails_closed_and_recovers() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("vault-token"); + write_owner_only(&path, "agent-token-v1"); + let provider = token_file_provider(path.clone(), test_policy(Duration::from_secs(10), Duration::from_secs(5))).await; + let task = provider.spawn_renewal_task().expect("renewal task"); + + std::fs::remove_file(&path).expect("remove token file"); + + // Polls at 30s, 35s, ... keep failing; the last read keeps the token + // usable until 50s (60s observed validity minus the 10s safety window). + tokio::time::sleep(Duration::from_secs(49)).await; + provider + .current() + .expect("token outside the safety window must still be served"); + + tokio::time::sleep(Duration::from_secs(2)).await; + let error = provider + .current() + .expect_err("token inside the safety window must be refused"); + assert!( + matches!(error, KmsError::CredentialsUnavailable { .. }), + "expected CredentialsUnavailable, got {error:?}" + ); + + // Restoring the file heals the provider on the next retry cycle. + write_owner_only(&path, "agent-token-v2"); + tokio::time::sleep(Duration::from_secs(6)).await; + let handle = provider.current().expect("provider must recover once the token file is back"); + assert!(handle.generation >= 1); + + task.shutdown().await; + } + + #[tokio::test] + async fn test_token_file_debug_redacts_token() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("vault-token"); + write_owner_only(&path, TEST_TOKEN); + let source = TokenFileSource::new(path.clone(), Duration::from_secs(60)); + source.acquire().await.expect("token file must resolve"); + + let rendered = format!("{source:?}"); + assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}"); + assert!(rendered.contains("vault-token"), "the file path is not a secret"); } } diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index a4153cdfc..a1355af75 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -14,7 +14,10 @@ //! Vault Transit-based KMS backend. -use crate::backends::vault_credentials::{VaultClientHandle, VaultConnectionSettings, VaultCredentialProvider, token_source_for}; +use crate::backends::vault_credentials::{ + CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider, + token_source_for, +}; use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; @@ -129,7 +132,7 @@ impl From<TransitKeyMetadataPersisted> for TransitKeyMetadata { } pub struct VaultTransitKmsClient { - credentials: VaultCredentialProvider, + credentials: Arc<VaultCredentialProvider>, config: VaultTransitConfig, /// KV v2 mount path for persisting transit key metadata metadata_kv_mount: String, @@ -141,15 +144,18 @@ pub struct VaultTransitKmsClient { impl VaultTransitKmsClient { /// Create a new Vault Transit KMS client /// - /// `attempt_timeout` caps every HTTP request issued through this client. - pub async fn new(config: VaultTransitConfig, attempt_timeout: Duration) -> Result<Self> { - let source = token_source_for(&config.auth_method)?; + /// `kms_config` supplies the per-attempt timeout that caps every HTTP + /// request issued through this client, plus the retry and fail-closed + /// budgets for credential refresh. + pub async fn new(config: VaultTransitConfig, kms_config: &KmsConfig) -> Result<Self> { let settings = VaultConnectionSettings { address: config.address.clone(), namespace: config.namespace.clone(), - attempt_timeout, + attempt_timeout: kms_config.effective_timeout(), }; - let credentials = VaultCredentialProvider::new(settings, source).await?; + let source = token_source_for(&config.auth_method, &settings)?; + let policy = VaultCredentialPolicy::from_kms_config(kms_config, &config.auth_method); + let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?); Ok(Self { credentials, @@ -163,8 +169,9 @@ impl VaultTransitKmsClient { /// Snapshot the authenticated Vault client for a single request. /// /// Every Vault call takes its own snapshot so a credential rotation - /// applies to subsequent calls without interrupting in-flight ones. - fn vault(&self) -> Arc<VaultClientHandle> { + /// applies to subsequent calls without interrupting in-flight ones. Fails + /// closed when the credentials could not be refreshed in time. + fn vault(&self) -> Result<Arc<VaultClientHandle>> { self.credentials.current() } @@ -192,7 +199,7 @@ impl VaultTransitKmsClient { } async fn read_transit_key(&self, key_id: &str) -> Result<vaultrs::api::transit::responses::ReadKeyResponse> { - key::read(&self.vault().client, &self.config.mount_path, key_id) + key::read(&self.vault()?.client, &self.config.mount_path, key_id) .await .or_else(|e| Self::map_vault_error(key_id, e, "read")) } @@ -200,7 +207,7 @@ impl VaultTransitKmsClient { async fn create_transit_key(&self, key_id: &str) -> Result<()> { let mut builder = CreateKeyRequestBuilder::default(); builder.key_type(KeyType::Aes256Gcm96); - key::create(&self.vault().client, &self.config.mount_path, key_id, Some(&mut builder)) + key::create(&self.vault()?.client, &self.config.mount_path, key_id, Some(&mut builder)) .await .map_err(|e| KmsError::backend_error(format!("Failed to create Vault Transit key {key_id}: {e}"))) } @@ -217,7 +224,7 @@ impl VaultTransitKmsClient { builder.associated_data(aad); } - let response = data::encrypt(&self.vault().client, &self.config.mount_path, key_id, &plaintext_b64, Some(&mut builder)) + let response = data::encrypt(&self.vault()?.client, &self.config.mount_path, key_id, &plaintext_b64, Some(&mut builder)) .await .map_err(|e| KmsError::backend_error(format!("Failed to encrypt data with Vault Transit key {key_id}: {e}")))?; @@ -235,7 +242,7 @@ impl VaultTransitKmsClient { builder.associated_data(aad); } - let response = data::decrypt(&self.vault().client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) + let response = data::decrypt(&self.vault()?.client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) .await .map_err(|e| KmsError::backend_error(format!("Failed to decrypt data with Vault Transit key {key_id}: {e}")))?; @@ -250,7 +257,7 @@ impl VaultTransitKmsClient { async fn read_metadata_from_kv(&self, key_id: &str) -> Result<Option<TransitKeyMetadata>> { let path = self.metadata_key_path(key_id); - match kv2::read::<TransitKeyMetadataPersisted>(&self.vault().client, &self.metadata_kv_mount, &path).await { + match kv2::read::<TransitKeyMetadataPersisted>(&self.vault()?.client, &self.metadata_kv_mount, &path).await { Ok(persisted) => Ok(Some(persisted.into())), Err(vaultrs::error::ClientError::ResponseWrapError) | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(None), @@ -261,7 +268,7 @@ impl VaultTransitKmsClient { async fn write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { let path = self.metadata_key_path(key_id); let persisted: TransitKeyMetadataPersisted = metadata.clone().into(); - kv2::set(&self.vault().client, &self.metadata_kv_mount, &path, &persisted) + kv2::set(&self.vault()?.client, &self.metadata_kv_mount, &path, &persisted) .await .map(|_| ()) .map_err(|e| KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}"))) @@ -269,7 +276,7 @@ impl VaultTransitKmsClient { async fn delete_metadata_from_kv(&self, key_id: &str) -> Result<()> { let path = self.metadata_key_path(key_id); - match kv2::delete_metadata(&self.vault().client, &self.metadata_kv_mount, &path).await { + match kv2::delete_metadata(&self.vault()?.client, &self.metadata_kv_mount, &path).await { Ok(_) => Ok(()), Err(vaultrs::error::ClientError::ResponseWrapError) | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(()), @@ -483,7 +490,7 @@ impl KmsClient for VaultTransitKmsClient { } async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { - let all_keys = key::list(&self.vault().client, &self.config.mount_path) + let all_keys = key::list(&self.vault()?.client, &self.config.mount_path) .await .map_err(|e| KmsError::backend_error(format!("Failed to list Vault Transit keys: {e}")))? .keys; @@ -553,7 +560,7 @@ impl KmsClient for VaultTransitKmsClient { } async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { - key::rotate(&self.vault().client, &self.config.mount_path, key_id) + key::rotate(&self.vault()?.client, &self.config.mount_path, key_id) .await .map_err(|e| KmsError::backend_error(format!("Failed to rotate Vault Transit key {key_id}: {e}")))?; @@ -576,7 +583,7 @@ impl KmsClient for VaultTransitKmsClient { } async fn health_check(&self) -> Result<()> { - key::list(&self.vault().client, &self.config.mount_path) + key::list(&self.vault()?.client, &self.config.mount_path) .await .map(|_| ()) .map_err(|e| KmsError::backend_error(format!("Vault Transit health check failed: {e}"))) @@ -612,9 +619,16 @@ impl VaultTransitKmsBackend { } }; - let client = VaultTransitKmsClient::new(vault_config, config.effective_timeout()).await?; + let client = VaultTransitKmsClient::new(vault_config, &config).await?; Ok(Self { client }) } + + /// Spawn the background credential renewal task for this backend, if its + /// auth method issues lease-bound tokens. The caller owns the returned + /// handle; dropping it cancels the task. + pub(crate) fn spawn_credential_renewal(&self) -> Option<CredentialTaskHandle> { + self.client.credentials.spawn_renewal_task() + } } #[async_trait] @@ -698,7 +712,7 @@ impl KmsBackend for VaultTransitKmsBackend { let mut update_builder = UpdateKeyConfigurationRequestBuilder::default(); update_builder.deletion_allowed(true); key::update( - &self.client.vault().client, + &self.client.vault()?.client, &self.client.config.mount_path, &key_id, Some(&mut update_builder), @@ -708,7 +722,7 @@ impl KmsBackend for VaultTransitKmsBackend { KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")) })?; } - key::delete(&self.client.vault().client, &self.client.config.mount_path, &key_id) + key::delete(&self.client.vault()?.client, &self.client.config.mount_path, &key_id) .await .map_err(|e| KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")))?; self.client.delete_key_metadata(&key_id).await?; @@ -810,7 +824,7 @@ mod tests { let config = test_vault_transit_config(); // --- First "process": create a key and disable it --- - let client1 = VaultTransitKmsClient::new(config.clone(), Duration::from_secs(30)) + let client1 = VaultTransitKmsClient::new(config.clone(), &KmsConfig::default()) .await .expect("Failed to create VaultTransit client"); @@ -833,7 +847,7 @@ mod tests { assert_eq!(info_after_disable.status, KeyStatus::Disabled, "key must be Disabled after disable_key"); // --- Simulate restart: create a brand new client with empty cache --- - let client2 = VaultTransitKmsClient::new(config, Duration::from_secs(30)) + let client2 = VaultTransitKmsClient::new(config, &KmsConfig::default()) .await .expect("Failed to create second VaultTransit client (restart simulation)"); @@ -863,7 +877,7 @@ mod tests { async fn test_transit_pending_deletion_survives_restart_simulation() { let config = test_vault_transit_config(); - let client1 = VaultTransitKmsClient::new(config.clone(), Duration::from_secs(30)) + let client1 = VaultTransitKmsClient::new(config.clone(), &KmsConfig::default()) .await .expect("Failed to create VaultTransit client"); @@ -887,7 +901,7 @@ mod tests { "key must be PendingDeletion after schedule_key_deletion" ); - let client2 = VaultTransitKmsClient::new(config, Duration::from_secs(30)) + let client2 = VaultTransitKmsClient::new(config, &KmsConfig::default()) .await .expect("Failed to create second VaultTransit client (restart simulation)"); @@ -929,7 +943,7 @@ mod tests { #[tokio::test] #[ignore] // Requires a running Vault instance with transit engine enabled async fn test_transit_old_ciphertext_decrypts_after_rotate() { - let client = VaultTransitKmsClient::new(test_vault_transit_config(), Duration::from_secs(30)) + let client = VaultTransitKmsClient::new(test_vault_transit_config(), &KmsConfig::default()) .await .expect("Failed to create VaultTransit client"); diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 2e2f5e80e..be4376ca5 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -29,8 +29,14 @@ pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRAN pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX"; pub const ENV_KMS_STATIC_SECRET_KEY: &str = "RUSTFS_KMS_STATIC_SECRET_KEY"; pub const ENV_KMS_STATIC_SECRET_KEY_FILE: &str = "RUSTFS_KMS_STATIC_SECRET_KEY_FILE"; +pub const ENV_KMS_VAULT_APPROLE_ROLE_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_ROLE_ID"; +pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID"; +pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE"; +pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT"; +pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata"; +pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle"; /// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior. /// @@ -53,6 +59,10 @@ fn default_vault_kv2_mount_path() -> String { "transit".to_string() } +fn default_vault_approle_mount() -> String { + DEFAULT_VAULT_APPROLE_MOUNT.to_string() +} + pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[ RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"), RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"), @@ -388,18 +398,94 @@ impl Default for VaultTransitConfig { pub enum VaultAuthMethod { /// Token authentication Token { token: String }, - /// AppRole authentication - AppRole { role_id: String, secret_id: String }, + /// AppRole authentication: login with `role_id` + `secret_id` for a + /// lease-bound token that is renewed in the background. + AppRole { + role_id: String, + /// Inline secret_id; used only when `secret_id_file` is unset. + secret_id: String, + /// Path to a file holding the secret_id. Re-read on every login so an + /// externally rotated secret_id is picked up; takes precedence over the + /// inline value. + #[serde(default)] + secret_id_file: Option<PathBuf>, + /// AppRole auth engine mount path. + #[serde(default = "default_vault_approle_mount")] + mount: String, + /// Fail-closed margin in seconds: once the current token is within this + /// window of expiry without a successful refresh, requests are refused + /// instead of sent with a token that may lapse mid-flight. Defaults to + /// the per-attempt timeout. + #[serde(default)] + refresh_safety_window_secs: Option<u64>, + }, + /// Agent-managed token file (for example a Vault Agent auto-auth sink): + /// the token is read from `path` and re-read periodically so a token + /// rotated by the agent is picked up without a restart. + TokenFile { + path: PathBuf, + /// Seconds between token file re-reads. Each successful read also + /// extends the token's observed validity to twice this value, so a + /// file that stops being readable eventually trips the fail-closed + /// window. Defaults to 30 seconds. + #[serde(default)] + poll_interval_secs: Option<u64>, + /// Fail-closed margin in seconds, as on `AppRole`. Defaults to the + /// per-attempt timeout. + #[serde(default)] + refresh_safety_window_secs: Option<u64>, + }, +} + +impl VaultAuthMethod { + /// AppRole authentication with the default mount and no secret-id file. + pub fn approle(role_id: String, secret_id: String) -> Self { + Self::AppRole { + role_id, + secret_id, + secret_id_file: None, + mount: default_vault_approle_mount(), + refresh_safety_window_secs: None, + } + } + + /// Agent-managed token file with the default poll interval. + pub fn token_file(path: PathBuf) -> Self { + Self::TokenFile { + path, + poll_interval_secs: None, + refresh_safety_window_secs: None, + } + } } impl fmt::Debug for VaultAuthMethod { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Token { token } => f.debug_struct("Token").field("token", &redacted_secret(token)).finish(), - Self::AppRole { role_id, secret_id } => f + Self::AppRole { + role_id, + secret_id, + secret_id_file, + mount, + refresh_safety_window_secs, + } => f .debug_struct("AppRole") .field("role_id", role_id) .field("secret_id", &redacted_secret(secret_id)) + .field("secret_id_file", secret_id_file) + .field("mount", mount) + .field("refresh_safety_window_secs", refresh_safety_window_secs) + .finish(), + Self::TokenFile { + path, + poll_interval_secs, + refresh_safety_window_secs, + } => f + .debug_struct("TokenFile") + .field("path", path) + .field("poll_interval_secs", poll_interval_secs) + .field("refresh_safety_window_secs", refresh_safety_window_secs) .finish(), } } @@ -479,7 +565,7 @@ impl KmsConfig { backend: KmsBackend::VaultKv2, backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig { address: address.to_string(), - auth_method: VaultAuthMethod::AppRole { role_id, secret_id }, + auth_method: VaultAuthMethod::approle(role_id, secret_id), ..Default::default() })), ..Default::default() @@ -637,6 +723,8 @@ impl KmsConfig { return Err(KmsError::configuration_error("Vault KV2 address must use http or https scheme")); } + validate_vault_auth_method("Vault KV2", &config.auth_method)?; + if !self.allow_insecure_dev_defaults { validate_vault_development_defaults("Vault KV2", &config.address, &config.auth_method, config.tls.as_ref())?; } @@ -660,6 +748,8 @@ impl KmsConfig { return Err(KmsError::configuration_error("Vault Transit address must use http or https scheme")); } + validate_vault_auth_method("Vault Transit", &config.auth_method)?; + if !self.allow_insecure_dev_defaults { validate_vault_development_defaults( "Vault Transit", @@ -764,7 +854,7 @@ impl KmsConfig { } KmsBackend::VaultKv2 => { let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); - let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"); + let auth_method = vault_auth_method_from_env()?; let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") { @@ -779,7 +869,7 @@ impl KmsConfig { config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig { address, - auth_method: VaultAuthMethod::Token { token }, + auth_method, namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), mount_path, kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"), @@ -789,12 +879,12 @@ impl KmsConfig { } KmsBackend::VaultTransit => { let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200"); - let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"); + let auth_method = vault_auth_method_from_env()?; let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig { address, - auth_method: VaultAuthMethod::Token { token }, + auth_method, namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"), mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"), metadata_kv_mount: get_env_str( @@ -873,6 +963,95 @@ fn is_under_temp_dir(path: &Path) -> bool { path.starts_with(std::env::temp_dir()) } +/// Resolve the Vault auth method from environment variables. +/// +/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication; +/// the secret_id then comes from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE` +/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE` +/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the +/// file taking precedence. Without a role id the legacy token flow applies. +fn vault_auth_method_from_env() -> Result<VaultAuthMethod> { + if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) { + // A token file names one authoritative credential source; combining it + // with another one would leave the effective identity ambiguous, so + // that is a configuration error rather than a precedence rule. + if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() { + return Err(KmsError::configuration_error(format!( + "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method" + ))); + } + if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() { + return Err(KmsError::configuration_error(format!( + "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method" + ))); + } + return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file))); + } + + let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else { + return Ok(VaultAuthMethod::Token { + token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"), + }); + }; + + let secret_id_file = get_env_opt_str(ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE).map(PathBuf::from); + let secret_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_SECRET_ID).unwrap_or_default(); + if secret_id.is_empty() && secret_id_file.is_none() { + return Err(KmsError::configuration_error(format!( + "Vault AppRole requires {ENV_KMS_VAULT_APPROLE_SECRET_ID} or {ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE} to be set" + ))); + } + + Ok(VaultAuthMethod::AppRole { + role_id, + secret_id, + secret_id_file, + mount: get_env_str(ENV_KMS_VAULT_APPROLE_MOUNT, DEFAULT_VAULT_APPROLE_MOUNT), + refresh_safety_window_secs: None, + }) +} + +fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod) -> Result<()> { + match auth_method { + VaultAuthMethod::Token { .. } => Ok(()), + VaultAuthMethod::AppRole { + role_id, + secret_id, + secret_id_file, + mount, + .. + } => { + if role_id.is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} AppRole role_id cannot be empty"))); + } + if secret_id.is_empty() && secret_id_file.is_none() { + return Err(KmsError::configuration_error(format!( + "{backend_name} AppRole requires a secret_id or a secret_id_file" + ))); + } + if mount.is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} AppRole mount cannot be empty"))); + } + Ok(()) + } + VaultAuthMethod::TokenFile { + path, + poll_interval_secs, + .. + } => { + if path.as_os_str().is_empty() { + return Err(KmsError::configuration_error(format!("{backend_name} token file path cannot be empty"))); + } + if poll_interval_secs == &Some(0) { + return Err(KmsError::configuration_error(format!( + "{backend_name} token file poll interval must be greater than 0" + ))); + } + Ok(()) + } + } +} + fn validate_vault_development_defaults( backend_name: &str, address: &str, @@ -1297,6 +1476,220 @@ mod tests { ); } + #[test] + fn test_from_env_selects_approle_when_role_id_is_set() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault")), + ("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")), + (ENV_KMS_VAULT_APPROLE_SECRET_ID, Some("env-approle-secret-id")), + (ENV_KMS_VAULT_APPROLE_MOUNT, Some("approle-alt")), + // A stale token env var must not override the AppRole selection. + ("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")), + ], + || { + let config = KmsConfig::from_env().expect("kms config should load from env"); + let vault = config.vault_config().expect("vault backend config"); + let VaultAuthMethod::AppRole { + role_id, + secret_id, + secret_id_file, + mount, + refresh_safety_window_secs, + } = &vault.auth_method + else { + panic!("role id in the environment must select AppRole auth, got {:?}", vault.auth_method); + }; + assert_eq!(role_id, "env-role-id"); + assert_eq!(secret_id, "env-approle-secret-id"); + assert_eq!(secret_id_file, &None); + assert_eq!(mount, "approle-alt"); + assert_eq!(refresh_safety_window_secs, &None); + }, + ); + } + + #[test] + fn test_from_env_approle_secret_id_file_is_stored_as_path() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault-transit")), + ("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")), + (ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE, Some("/etc/rustfs/approle-secret-id")), + ], + || { + let config = KmsConfig::from_env().expect("kms config should load from env"); + let vault = config.vault_transit_config().expect("vault transit backend config"); + let VaultAuthMethod::AppRole { + secret_id, + secret_id_file, + mount, + .. + } = &vault.auth_method + else { + panic!("role id in the environment must select AppRole auth"); + }; + // The path is stored, not read: the secret_id file is re-read on + // every login so external rotation is picked up. + assert_eq!(secret_id_file.as_deref(), Some(std::path::Path::new("/etc/rustfs/approle-secret-id"))); + assert!(secret_id.is_empty()); + assert_eq!(mount, DEFAULT_VAULT_APPROLE_MOUNT); + }, + ); + } + + #[test] + fn test_from_env_approle_requires_secret_id_or_file() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault")), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")), + (ENV_KMS_VAULT_APPROLE_SECRET_ID, None::<&str>), + (ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE, None::<&str>), + ], + || { + let error = KmsConfig::from_env().expect_err("approle without a secret_id source must be rejected"); + assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_SECRET_ID)); + }, + ); + } + + #[test] + fn test_from_env_selects_token_file() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault")), + ("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")), + (ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")), + ], + || { + let config = KmsConfig::from_env().expect("kms config should load from env"); + let vault = config.vault_config().expect("vault backend config"); + let VaultAuthMethod::TokenFile { + path, + poll_interval_secs, + refresh_safety_window_secs, + } = &vault.auth_method + else { + panic!("token file in the environment must select TokenFile auth, got {:?}", vault.auth_method); + }; + assert_eq!(path, std::path::Path::new("/run/vault-agent/token")); + assert_eq!(poll_interval_secs, &None); + assert_eq!(refresh_safety_window_secs, &None); + }, + ); + } + + #[test] + fn test_from_env_token_file_is_mutually_exclusive_with_other_auth() { + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault")), + (ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")), + (ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")), + ], + || { + let error = KmsConfig::from_env().expect_err("token file combined with approle must be rejected"); + assert!(error.to_string().contains(ENV_KMS_VAULT_TOKEN_FILE)); + assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID)); + }, + ); + + with_vars( + vec![ + ("RUSTFS_KMS_BACKEND", Some("vault-transit")), + (ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")), + ("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")), + ], + || { + let error = KmsConfig::from_env().expect_err("token file combined with a static token must be rejected"); + assert!(error.to_string().contains(ENV_KMS_VAULT_TOKEN_FILE)); + assert!(error.to_string().contains("RUSTFS_KMS_VAULT_TOKEN")); + }, + ); + } + + #[test] + fn test_validate_rejects_bad_token_file_settings() { + let vault_config = |auth_method: VaultAuthMethod| KmsConfig { + backend: KmsBackend::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig { + address: "https://vault.example.com:8200".to_string(), + auth_method, + ..Default::default() + })), + ..Default::default() + }; + + let error = vault_config(VaultAuthMethod::token_file(PathBuf::new())) + .validate() + .expect_err("empty token file path must be rejected"); + assert!(error.to_string().contains("path")); + + let error = vault_config(VaultAuthMethod::TokenFile { + path: PathBuf::from("/run/vault-agent/token"), + poll_interval_secs: Some(0), + refresh_safety_window_secs: None, + }) + .validate() + .expect_err("zero poll interval must be rejected"); + assert!(error.to_string().contains("poll interval")); + + vault_config(VaultAuthMethod::token_file(PathBuf::from("/run/vault-agent/token"))) + .validate() + .expect("well-formed token file auth must validate"); + } + + #[test] + fn test_approle_config_deserializes_legacy_shape_with_defaults() { + // Persisted configurations from before the AppRole implementation only + // carry role_id and secret_id; the new fields must fill with defaults. + let legacy = serde_json::json!({ + "AppRole": { + "role_id": "legacy-role", + "secret_id": "legacy-secret-id", + } + }); + let auth: VaultAuthMethod = serde_json::from_value(legacy).expect("legacy AppRole config must keep deserializing"); + let VaultAuthMethod::AppRole { + role_id, + secret_id_file, + mount, + refresh_safety_window_secs, + .. + } = auth + else { + panic!("expected AppRole"); + }; + assert_eq!(role_id, "legacy-role"); + assert_eq!(secret_id_file, None); + assert_eq!(mount, DEFAULT_VAULT_APPROLE_MOUNT); + assert_eq!(refresh_safety_window_secs, None); + } + + #[test] + fn test_validate_rejects_incomplete_approle() { + let mut config = KmsConfig::vault_approle( + Url::parse("https://vault.example.com:8200").expect("vault URL"), + String::new(), + "secret-id".to_string(), + ); + let error = config.validate().expect_err("empty role_id must be rejected"); + assert!(error.to_string().contains("role_id")); + + config = KmsConfig::vault_approle( + Url::parse("https://vault.example.com:8200").expect("vault URL"), + "role-id".to_string(), + String::new(), + ); + let error = config + .validate() + .expect_err("approle without secret_id or secret_id_file must be rejected"); + assert!(error.to_string().contains("secret_id")); + } + #[test] fn test_from_env_requires_vault_development_opt_in() { with_vars( diff --git a/crates/kms/src/error.rs b/crates/kms/src/error.rs index 409494b08..0eeff00e9 100644 --- a/crates/kms/src/error.rs +++ b/crates/kms/src/error.rs @@ -132,6 +132,11 @@ pub enum KmsError { /// Operation is not supported by the active KMS backend #[error("Operation '{operation}' is not supported by KMS backend '{backend}'")] UnsupportedCapability { backend: String, operation: String }, + + /// Backend credentials expired or could not be refreshed in time; requests + /// fail closed instead of being sent with credentials that may lapse mid-flight + #[error("KMS credentials unavailable: {message}")] + CredentialsUnavailable { message: String }, } impl KmsError { @@ -281,6 +286,11 @@ impl KmsError { operation: operation.into(), } } + + /// Create a credentials unavailable error + pub fn credentials_unavailable<S: Into<String>>(message: S) -> Self { + Self::CredentialsUnavailable { message: message.into() } + } } /// Convert from standard library errors diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index be463c42d..7ec4453e8 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -14,6 +14,7 @@ //! KMS service manager for dynamic configuration and runtime management +use crate::backends::vault_credentials::CredentialTaskHandle; use crate::backends::{KmsBackend, local::LocalKmsBackend}; use crate::config::{BackendConfig, KmsConfig}; use crate::error::{KmsError, Result}; @@ -110,6 +111,10 @@ struct ServiceVersion { service: Arc<ObjectEncryptionService>, /// The KMS manager instance manager: Arc<KmsManager>, + /// Owner of the backend's credential renewal task, if the backend needs + /// one. Stop shuts it down explicitly; reconfigure recycles it through + /// the handle's cancel-on-drop behavior when the old version is discarded. + credential_task: Option<Arc<CredentialTaskHandle>>, } #[derive(Clone)] @@ -331,6 +336,13 @@ impl KmsServiceManager { current_service: None, })); + // Shut down the stopped version's credential renewal task before + // reporting stopped, so stop deterministically recycles the background + // task even while in-flight operations still hold the old service Arc. + if let Some(task) = state.current_service.as_ref().and_then(|sv| sv.credential_task.clone()) { + task.shutdown().await; + } + debug!( event = EVENT_KMS_SERVICE_STATE, component = LOG_COMPONENT_KMS, @@ -488,7 +500,10 @@ impl KmsServiceManager { info!("Creating KMS service version {} with backend: {:?}", version, config.backend); - // Create backend + // Create backend. Vault backends may also spawn a background + // credential renewal task whose owner handle lives on the service + // version, so replacing the version recycles the task. + let mut credential_task = None; let backend = match &config.backend_config { BackendConfig::Local(_) => { info!("Creating Local KMS backend for version {}", version); @@ -498,11 +513,13 @@ impl KmsServiceManager { BackendConfig::VaultKv2(_) => { info!("Creating Vault KV2 KMS backend for version {}", version); let backend = crate::backends::vault::VaultKmsBackend::new(config.clone()).await?; + credential_task = backend.spawn_credential_renewal().map(Arc::new); Arc::new(backend) as Arc<dyn KmsBackend> } BackendConfig::VaultTransit(_) => { info!("Creating Vault Transit KMS backend for version {}", version); let backend = crate::backends::vault_transit::VaultTransitKmsBackend::new(config.clone()).await?; + credential_task = backend.spawn_credential_renewal().map(Arc::new); Arc::new(backend) as Arc<dyn KmsBackend> } BackendConfig::Static(_) => { @@ -522,6 +539,7 @@ impl KmsServiceManager { version, service: encryption_service, manager: kms_manager, + credential_task, }) } diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index e8063f5b3..badc78bae 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -2,6 +2,8 @@ RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone. +For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). + ## Backend comparison | Backend | Config tag | Master key material location | At-rest protection of key material | Durability | Rotation | Intended use | diff --git a/docs/operations/vault-kms-authentication.md b/docs/operations/vault-kms-authentication.md new file mode 100644 index 000000000..6ee0a0c2c --- /dev/null +++ b/docs/operations/vault-kms-authentication.md @@ -0,0 +1,122 @@ +# Vault KMS authentication runbook + +This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenticate to Vault, how to deploy AppRole and Vault Agent token-file authentication, and how the fail-closed credential window behaves in production. For what each backend stores in Vault and the KV2/Transit policy scopes, see [KMS backend security properties](kms-backend-security.md). + +## Choosing an authentication method + +| Method | Config tag | Credential lifetime | Background renewal | Recommended for | +| --- | --- | --- | --- | --- | +| Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments | +| AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar | +| Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth | + +Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous. + +The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification. + +## AppRole authentication + +### Vault-side setup + +Create a policy scoped to exactly what the backend needs (see the KV2 and Transit policy examples in [KMS backend security properties](kms-backend-security.md)), then an AppRole that issues tokens carrying it: + +```shell +vault policy write rustfs-kms rustfs-kms-policy.hcl + +vault auth enable approle + +vault write auth/approle/role/rustfs-kms \ + token_policies="rustfs-kms" \ + token_ttl=1h \ + token_max_ttl=24h \ + secret_id_ttl=90d \ + secret_id_num_uses=0 +``` + +Keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s): the fail-closed window defaults to one attempt timeout, so a token TTL close to it leaves almost no usable lifetime. + +### RustFS configuration + +```shell +RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend +RUSTFS_KMS_VAULT_ADDRESS=https://vault.example.com:8200 +RUSTFS_KMS_VAULT_APPROLE_ROLE_ID=<role-id> +RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE=/etc/rustfs/approle-secret-id +# Alternatively, inline (the file takes precedence when both are set): +# RUSTFS_KMS_VAULT_APPROLE_SECRET_ID=<secret-id> +# Optional, defaults to "approle": +# RUSTFS_KMS_VAULT_APPROLE_MOUNT=approle +``` + +RustFS logs in at startup, then renews the token at half its TTL in the background. If renewal fails (network, Vault sealed, token revoked), it falls back to a full re-login; if that also fails, it keeps retrying every few seconds until Vault recovers. + +### SecretID delivery and rotation + +Deliver the SecretID out of band — a secrets-manager-mounted file, an init-container writing `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE`, or Vault response wrapping unwrapped by your deployment tooling. Treat it like a password: owner-readable file permissions, never in logs or shell history. + +The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login. + +An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart. + +## Vault Agent token file + +In this mode a Vault Agent (or any equivalent process) owns authentication and token renewal, and RustFS only reads the token sink file. + +### Vault Agent example + +```hcl +auto_auth { + method "approle" { + config = { + role_id_file_path = "/etc/vault-agent/role-id" + secret_id_file_path = "/etc/vault-agent/secret-id" + } + } + + sink "file" { + config = { + path = "/run/vault-agent/token" + mode = 0600 + } + } +} +``` + +### RustFS configuration + +```shell +RUSTFS_KMS_BACKEND=vault-transit +RUSTFS_KMS_VAULT_ADDRESS=https://vault.example.com:8200 +RUSTFS_KMS_VAULT_TOKEN_FILE=/run/vault-agent/token +``` + +The poll interval (`poll_interval_secs` in the `TokenFile` auth configuration, default 30 seconds) controls how often the file is re-read. Each successful read grants the token an observed validity of twice the poll interval and installs a fresh client generation, so an agent-rotated token is picked up within one poll interval of the atomic replace. + +Requirements enforced at every read, each failing the refresh without contacting Vault: + +- The file must exist and be non-empty after trimming whitespace. +- On Unix, the file must not be readable or writable by group or other (mode `0600` or stricter). Wider permissions are a hard error naming the offending mode, mirroring the SFTP host-key rule. RustFS must run as the file's owner. + +If the agent stops refreshing the file that is fine — RustFS re-reads the same token and keeps going as long as the token itself is valid on the Vault side. If the file disappears or turns empty, RustFS keeps serving requests on the last-read token until the fail-closed window trips, and heals automatically once the file is restored. + +## Fail-closed window + +For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side. + +- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it. +- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration. +- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise. + +The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file). + +### Troubleshooting + +| Symptom | Log line to look for | Likely cause and fix | +| --- | --- | --- | +| Requests fail with `KMS credentials unavailable` | `Vault credential refresh failed; retrying until the credentials recover` (warn, repeated) | Vault unreachable/sealed, or the credential source is broken; the provider recovers on its own once refresh succeeds — fix the cause, no restart needed | +| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file | +| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider | +| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider | +| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file | + +When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault. From 2e29c330a9778fd9ddfc83682631ccc6639a5e87 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 07:24:39 +0800 Subject: [PATCH 29/52] feat(kms): enforce shared key state machine across backends (#5489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): enforce shared key state machine across backends Unify the key state x operation matrix behind a single gate in backends/mod.rs and wire it into the Local, Vault KV2 and Vault Transit backends: Disabled keys reject encryption, data key generation and rotation while still allowing decryption and lifecycle recovery; PendingDeletion keys reject everything except decryption and cancellation (including repeated deletion scheduling); cancellation now requires an actual pending deletion everywhere. This closes the missing gates on KV2 encrypt/generate and Local generate_data_key, and stops enable_key from silently reverting a pending deletion. Decryption is deliberately left ungated in Disabled/PendingDeletion — an explicit, documented and tested deviation from AWS KMS, since gating it would break reads of existing objects the moment a key is disabled. Add shared contract tests driving the full matrix offline for Local (and via ignored tests against a live Vault for KV2/Transit), a stateless contract for Static, an SSE-shaped regression proving existing envelopes stay decryptable after disable, and a pin on the known-risk Enabled default of Transit's synthesized metadata fallback. Refs rustfs/backlog#1571 (part of rustfs/backlog#1562) * feat(kms): persist deletion deadlines and run a restartable deletion worker (#5491) --- crates/kms/src/backends/contract_tests.rs | 330 ++++++++++++++++++ crates/kms/src/backends/local.rs | 190 ++++++++++- crates/kms/src/backends/mod.rs | 114 ++++++- crates/kms/src/backends/vault.rs | 128 ++++++- crates/kms/src/backends/vault_transit.rs | 118 ++++++- crates/kms/src/deletion_worker.rs | 399 ++++++++++++++++++++++ crates/kms/src/lib.rs | 2 + crates/kms/src/manager.rs | 6 + crates/kms/src/service_manager.rs | 140 +++++++- crates/kms/src/types.rs | 5 + 10 files changed, 1394 insertions(+), 38 deletions(-) create mode 100644 crates/kms/src/backends/contract_tests.rs create mode 100644 crates/kms/src/deletion_worker.rs diff --git a/crates/kms/src/backends/contract_tests.rs b/crates/kms/src/backends/contract_tests.rs new file mode 100644 index 000000000..a436a17e2 --- /dev/null +++ b/crates/kms/src/backends/contract_tests.rs @@ -0,0 +1,330 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared key state × operation contract tests for KMS backends. +//! +//! Every stateful backend must satisfy the same lifecycle matrix (see +//! `ensure_key_state_permits`): Enabled permits everything, Disabled permits +//! decryption and lifecycle recovery but rejects new cryptographic use, and +//! PendingDeletion rejects everything except decryption and cancellation. +//! Decryption staying available in Disabled/PendingDeletion is an explicit, +//! tested deviation from AWS KMS: disabling a key must not break reads of +//! objects already encrypted under it. +//! +//! The full matrix runs offline against the Local backend. The Vault KV2 and +//! Vault Transit runs exercise the same helper but need a live Vault dev +//! server, so they are `#[ignore]`d in CI. Static is covered by its own +//! stateless contract below. + +use super::local::LocalKmsBackend; +use super::static_kms::StaticKmsBackend; +use super::vault::VaultKmsBackend; +use super::vault_transit::VaultTransitKmsBackend; +use super::{KmsBackend, KmsClient}; +use crate::config::KmsConfig; +use crate::error::{KmsError, Result}; +use crate::manager::KmsManager; +use crate::service::ObjectEncryptionService; +use crate::types::{ + CancelKeyDeletionRequest, CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, EncryptRequest, + GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext, +}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use rand::RngExt as _; +use std::collections::HashMap; +use std::sync::Arc; + +fn expect_invalid_key_state<T: std::fmt::Debug>(result: Result<T>, expected_fragment: &str) { + match result { + Err(KmsError::InvalidOperation { message }) => assert!( + message.contains(expected_fragment), + "expected invalid-key-state message containing {expected_fragment:?}, got {message:?}" + ), + other => panic!("expected InvalidOperation (invalid key state), got {other:?}"), + } +} + +fn context() -> HashMap<String, String> { + HashMap::from([("bucket".to_string(), "contract".to_string())]) +} + +fn generate_request(key_id: &str) -> GenerateDataKeyRequest { + GenerateDataKeyRequest { + key_id: key_id.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + } +} + +fn encrypt_request(key_id: &str) -> EncryptRequest { + EncryptRequest { + key_id: key_id.to_string(), + plaintext: b"contract-plaintext".to_vec(), + encryption_context: context(), + grant_tokens: Vec::new(), + } +} + +fn decrypt_request(ciphertext: Vec<u8>) -> DecryptRequest { + DecryptRequest { + ciphertext, + encryption_context: context(), + grant_tokens: Vec::new(), + } +} + +fn schedule_request(key_id: &str) -> DeleteKeyRequest { + DeleteKeyRequest { + key_id: key_id.to_string(), + pending_window_in_days: Some(7), + force_immediate: None, + } +} + +fn cancel_request(key_id: &str) -> CancelKeyDeletionRequest { + CancelKeyDeletionRequest { + key_id: key_id.to_string(), + } +} + +fn create_request(key_name: String) -> CreateKeyRequest { + CreateKeyRequest { + key_name: Some(key_name), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + } +} + +async fn assert_key_state(backend: &dyn KmsBackend, key_id: &str, expected: KeyState) { + let described = backend + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .expect("describe_key must succeed for an existing key"); + assert_eq!(described.key_metadata.key_state, expected, "unexpected state for key {key_id}"); +} + +/// Drives one freshly created (Enabled) key through the full state matrix. +/// +/// `backend` is the product surface; `client` drives the lifecycle +/// transitions not yet exposed through `KmsBackend`. +async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn KmsClient, key_id: &str) { + // Enabled: cryptographic use is allowed. Keep an envelope around to prove + // decryption keeps working in later states. + let data_key = backend + .generate_data_key(generate_request(key_id)) + .await + .expect("Enabled key must generate data keys"); + backend + .encrypt(encrypt_request(key_id)) + .await + .expect("Enabled key must encrypt"); + + // Enabled -> Disabled. + client + .disable_key(key_id, None) + .await + .expect("disable from Enabled must succeed"); + assert_key_state(backend, key_id, KeyState::Disabled).await; + + // Disabled: new cryptographic use and rotation are rejected... + expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "disabled"); + expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "disabled"); + expect_invalid_key_state(client.rotate_key(key_id, None).await, ""); + // ...but decryption of existing data keeps working (explicit AWS deviation)... + let decrypted = backend + .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) + .await + .expect("decrypt with a disabled key must keep working"); + assert_eq!(decrypted.plaintext, data_key.plaintext_key, "decrypt must recover the original data key"); + // ...disable stays idempotent, cancel has nothing to cancel, and enable recovers. + client.disable_key(key_id, None).await.expect("disable must be idempotent"); + expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion"); + client + .enable_key(key_id, None) + .await + .expect("enable from Disabled must succeed"); + assert_key_state(backend, key_id, KeyState::Enabled).await; + + // Disabled keys may still be scheduled for deletion. + client + .disable_key(key_id, None) + .await + .expect("disable before scheduling must succeed"); + backend + .delete_key(schedule_request(key_id)) + .await + .expect("scheduling deletion of a disabled key must succeed"); + assert_key_state(backend, key_id, KeyState::PendingDeletion).await; + + // PendingDeletion: everything except decryption and cancellation is rejected. + expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "pending deletion"); + expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "pending deletion"); + expect_invalid_key_state(client.enable_key(key_id, None).await, "pending deletion"); + expect_invalid_key_state(client.disable_key(key_id, None).await, "pending deletion"); + expect_invalid_key_state(client.rotate_key(key_id, None).await, ""); + expect_invalid_key_state(client.schedule_key_deletion(key_id, 7, None).await, "pending deletion"); + expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "pending deletion"); + let decrypted = backend + .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) + .await + .expect("decrypt with a pending-deletion key must keep working"); + assert_eq!(decrypted.plaintext, data_key.plaintext_key); + + // PendingDeletion -> Enabled through cancellation. + backend + .cancel_key_deletion(cancel_request(key_id)) + .await + .expect("cancel from PendingDeletion must succeed"); + assert_key_state(backend, key_id, KeyState::Enabled).await; + backend + .generate_data_key(generate_request(key_id)) + .await + .expect("cancelled key must be usable again"); + + // Cancel without a pending deletion is an invalid state transition. + expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion"); +} + +async fn local_fixture() -> (tempfile::TempDir, KmsConfig, LocalKmsBackend, String) { + let temp_dir = tempfile::tempdir().expect("temp dir should be created"); + let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + let backend = LocalKmsBackend::new(config.clone()) + .await + .expect("local backend should build"); + let created = backend + .create_key(create_request("contract-key".to_string())) + .await + .expect("key should be created"); + (temp_dir, config, backend, created.key_id) +} + +#[tokio::test] +async fn local_backend_state_machine_contract() { + let (_temp_dir, _config, backend, key_id) = local_fixture().await; + assert_state_machine_contract(&backend, backend.lifecycle_client(), &key_id).await; +} + +/// SSE-shaped regression: disabling a key must not break decryption of data +/// keys created while it was enabled, while new data key creation must fail. +#[tokio::test] +async fn local_disabled_key_keeps_decrypting_existing_envelopes() { + let (_temp_dir, config, backend, key_id) = local_fixture().await; + let backend = Arc::new(backend); + let service = ObjectEncryptionService::new(KmsManager::new(backend.clone(), config)); + + let object_context = ObjectEncryptionContext::new("sse-bucket".to_string(), "dir/object.bin".to_string()); + let kms_key = Some(key_id.clone()); + let (_data_key, encrypted_blob) = service + .create_data_key(&kms_key, &object_context) + .await + .expect("data key creation must succeed while the key is enabled"); + + backend + .lifecycle_client() + .disable_key(&key_id, None) + .await + .expect("disable must succeed"); + + service + .decrypt_data_key(&encrypted_blob, &object_context) + .await + .expect("existing objects must stay readable after their KMS key is disabled"); + expect_invalid_key_state(service.create_data_key(&kms_key, &object_context).await, "disabled"); +} + +/// Static is a stateless read-only backend: cryptographic operations always +/// work against the single configured key and every lifecycle mutation is +/// rejected as an invalid operation. +#[tokio::test] +async fn static_backend_stateless_contract() { + let key_id = "static-contract-key"; + let mut raw_key = [0u8; 32]; + rand::rng().fill(&mut raw_key[..]); + let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key)); + let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build"); + // StaticKmsBackend implements both traits with overlapping method names, + // so pin each surface once instead of qualifying every call. + let backend: &dyn KmsBackend = &static_backend; + let client: &dyn KmsClient = &static_backend; + + let data_key = backend + .generate_data_key(generate_request(key_id)) + .await + .expect("static backend must generate data keys"); + let decrypted = backend + .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) + .await + .expect("static backend must decrypt its own envelopes"); + assert_eq!(decrypted.plaintext, data_key.plaintext_key); + assert_key_state(backend, key_id, KeyState::Enabled).await; + + expect_invalid_key_state(backend.create_key(create_request("another-key".to_string())).await, "read-only"); + expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "read-only"); + expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "read-only"); + expect_invalid_key_state(client.disable_key(key_id, None).await, "read-only"); + expect_invalid_key_state(client.schedule_key_deletion(key_id, 7, None).await, "read-only"); + expect_invalid_key_state(client.rotate_key(key_id, None).await, "read-only"); +} + +fn vault_dev_config(constructor: fn(url::Url, String) -> KmsConfig) -> KmsConfig { + let address = std::env::var("RUSTFS_KMS_VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()); + let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-token".to_string()); + let mut config = constructor(url::Url::parse(&address).expect("vault address should parse"), token); + config.allow_insecure_dev_defaults = true; + config +} + +#[tokio::test] +#[ignore] // Requires a running Vault instance (dev mode) with a KV2 mount +async fn vault_kv2_backend_state_machine_contract() { + let config = vault_dev_config(KmsConfig::vault); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + let created = backend + .create_key(create_request(format!("contract-{}", uuid::Uuid::new_v4()))) + .await + .expect("key should be created"); + + assert_state_machine_contract(&backend, backend.lifecycle_client(), &created.key_id).await; + + // Cleanup: leave the key pending deletion so repeated runs stay tidy. + let _ = backend.delete_key(schedule_request(&created.key_id)).await; +} + +#[tokio::test] +#[ignore] // Requires a running Vault instance (dev mode) with the transit engine enabled +async fn vault_transit_backend_state_machine_contract() { + let config = vault_dev_config(KmsConfig::vault_transit); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + let created = backend + .create_key(create_request(format!("contract-{}", uuid::Uuid::new_v4()))) + .await + .expect("key should be created"); + + assert_state_machine_contract(&backend, backend.lifecycle_client(), &created.key_id).await; + + // Transit additionally supports rotation, which must only work while the + // key is Enabled (the shared matrix already covered the rejections). + backend + .lifecycle_client() + .rotate_key(&created.key_id, None) + .await + .expect("rotation of an Enabled transit key must succeed"); + + let _ = backend.delete_key(schedule_request(&created.key_id)).await; +} diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index f6e71ac97..3c48ff2c7 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -14,7 +14,9 @@ //! Local file-based KMS backend implementation -use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{ + BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_status_permits, +}; use crate::config::KmsConfig; use crate::config::LocalConfig; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -425,6 +427,10 @@ struct StoredMasterKey { #[serde(with = "crate::time_serde::option_zoned")] rotated_at: Option<Zoned>, created_by: Option<String>, + /// Scheduled deletion deadline; absent on records written before deadline + /// persistence landed, so it must stay optional for backward compatibility. + #[serde(default, with = "crate::time_serde::option_zoned")] + deletion_date: Option<Zoned>, /// Encrypted key material (32 bytes encoded in base64 for AES-256) encrypted_key_material: String, /// Nonce used for encryption @@ -770,6 +776,7 @@ impl LocalKmsClient { created_at: stored_key.created_at, rotated_at: stored_key.rotated_at, created_by: stored_key.created_by, + deletion_date: stored_key.deletion_date, }) } @@ -843,6 +850,7 @@ impl LocalKmsClient { created_at: master_key.created_at.clone(), rotated_at: master_key.rotated_at.clone(), created_by: master_key.created_by.clone(), + deletion_date: master_key.deletion_date.clone(), encrypted_key_material, nonce, at_rest_protection, @@ -931,9 +939,12 @@ impl LocalKmsClient { #[async_trait] impl KmsClient for LocalKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> { + async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKeyInfo> { debug!("Generating data key for master key: {}", request.master_key_id); + let key_info = self.describe_key(&request.master_key_id, context).await?; + ensure_key_status_permits(&request.master_key_id, &key_info.status, StateGatedOperation::GenerateDataKey)?; + // Generate random data key material let key_length = match request.key_spec.as_str() { "AES_256" => 32, @@ -972,14 +983,9 @@ impl KmsClient for LocalKmsClient { async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result<EncryptResponse> { debug!("Encrypting data with key: {}", request.key_id); - // Verify key exists and is active + // Verify key exists and its state allows encryption let key_info = self.describe_key(&request.key_id, context).await?; - if key_info.status != KeyStatus::Active { - return Err(KmsError::invalid_operation(format!( - "Key {} is not active (status: {:?})", - request.key_id, key_info.status - ))); - } + ensure_key_status_permits(&request.key_id, &key_info.status, StateGatedOperation::Encrypt)?; let (ciphertext, _nonce) = self.encrypt_with_master_key(&request.key_id, &request.plaintext).await?; @@ -1110,6 +1116,7 @@ impl KmsClient for LocalKmsClient { let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; + ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::Enable)?; master_key.status = KeyStatus::Active; // Preserve the existing key material. Regenerating it on a pure status change would @@ -1127,6 +1134,7 @@ impl KmsClient for LocalKmsClient { let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; + ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::Disable)?; master_key.status = KeyStatus::Disabled; // Preserve the existing key material (see enable_key): a status change must never @@ -1141,14 +1149,16 @@ impl KmsClient for LocalKmsClient { async fn schedule_key_deletion( &self, key_id: &str, - _pending_window_days: u32, + pending_window_days: u32, _context: Option<&OperationContext>, ) -> Result<()> { debug!("Scheduling deletion for key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; + ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?; master_key.status = KeyStatus::PendingDeletion; + master_key.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); // Preserve the existing key material (see enable_key): scheduling deletion must not // regenerate the master key, or cancelling the deletion later would recover a key that @@ -1165,7 +1175,11 @@ impl KmsClient for LocalKmsClient { let _write_guard = self.lock_key_for_write(key_id).await; let mut master_key = self.load_master_key(key_id).await?; + if master_key.status != KeyStatus::PendingDeletion { + return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); + } master_key.status = KeyStatus::Active; + master_key.deletion_date = None; // Preserve the existing key material (see enable_key): cancelling deletion must recover // the ORIGINAL key, not mint a new one that cannot decrypt existing data. @@ -1215,6 +1229,12 @@ pub struct LocalKmsBackend { } impl LocalKmsBackend { + /// Lifecycle driver for the shared state-machine contract tests. + #[cfg(test)] + pub(crate) fn lifecycle_client(&self) -> &LocalKmsClient { + &self.client + } + /// Create a new LocalKmsBackend pub async fn new(config: KmsConfig) -> Result<Self> { config.validate()?; @@ -1328,6 +1348,11 @@ impl KmsBackend for LocalKmsBackend { async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> { let key_info = self.client.describe_key(&request.key_id, None).await?; + let deletion_date = if key_info.status == KeyStatus::PendingDeletion { + self.client.load_master_key(&request.key_id).await?.deletion_date + } else { + None + }; let metadata = KeyMetadata { key_id: key_info.key_id, @@ -1340,7 +1365,7 @@ impl KmsBackend for LocalKmsBackend { key_usage: key_info.usage, description: key_info.description, creation_date: key_info.created_at, - deletion_date: None, + deletion_date, origin: "KMS".to_string(), key_manager: "CUSTOMER".to_string(), tags: key_info.tags, @@ -1371,7 +1396,22 @@ impl KmsBackend for LocalKmsBackend { .map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?; let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) { - // For immediate deletion, actually delete the key from filesystem + // Tombstone first: mark the record Deleted before removing the + // file, so a crash between the two steps leaves a key that is + // already unusable and whose removal can simply be re-run. + match self.client.decode_stored_key(key_id).await { + Ok((_stored, key_material)) => { + let mut tombstone = master_key.clone(); + tombstone.status = KeyStatus::Deleted; + tombstone.deletion_date = Some(Zoned::now()); + self.client.save_master_key(&tombstone, &key_material).await?; + } + Err(error) => { + // A record whose material can no longer be decoded cannot be + // re-encrypted into a tombstone; proceed with the removal. + warn!(key_id, %error, "skipping tombstone for undecodable key record"); + } + } let key_path = self.client.master_key_path(key_id)?; durable_file::remove_durably(key_path) .await @@ -1399,6 +1439,8 @@ impl KmsBackend for LocalKmsBackend { }); } else { // Schedule for deletion (default 30 days) + ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?; + let days = request.pending_window_in_days.unwrap_or(30); if !(7..=30).contains(&days) { return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string())); @@ -1406,6 +1448,7 @@ impl KmsBackend for LocalKmsBackend { let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400); master_key.status = KeyStatus::PendingDeletion; + master_key.deletion_date = Some(deletion_date.clone()); (Some(deletion_date.to_string()), Some(deletion_date)) }; @@ -1459,6 +1502,7 @@ impl KmsBackend for LocalKmsBackend { // Cancel the deletion by resetting the state master_key.status = KeyStatus::Active; + master_key.deletion_date = None; // Save the updated key to disk - this is the missing critical step! // Preserve existing key material instead of generating new one @@ -1496,13 +1540,55 @@ impl KmsBackend for LocalKmsBackend { fn capabilities(&self) -> BackendCapabilities { // Rotation stays unadvertised until historical key versions can be // retained (see LocalKmsClient::rotate_key); without version history - // there is also no versioning capability. Deletion deadlines are not - // yet persisted across restarts, but scheduling itself is supported. + // there is also no versioning capability. BackendCapabilities::minimal() .with_enable_disable(true) .with_schedule_deletion(true) .with_physical_delete(true) } + + async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> { + // The per-key write lock serializes this against a concurrent + // cancellation, closing the check-then-remove race. + let _write_guard = self.client.lock_key_for_write(key_id).await; + + if !fs::try_exists(self.client.master_key_path(key_id)?).await? { + return Ok(ExpiredKeyRemoval::Removed); + } + let master_key = self.client.load_master_key(key_id).await?; + match master_key.status { + // Tombstone left by a crashed removal: complete it. + KeyStatus::Deleted => {} + KeyStatus::PendingDeletion => { + match &master_key.deletion_date { + Some(deadline) if deadline <= now => {} + // Not yet due, or a legacy record without a persisted + // deadline — never auto-remove those. + _ => return Ok(ExpiredKeyRemoval::NotExpired), + } + // Tombstone first (see delete_key): a crash between the state + // write and the file removal must leave an unusable record. + match self.client.decode_stored_key(key_id).await { + Ok((_stored, key_material)) => { + let mut tombstone = master_key.clone(); + tombstone.status = KeyStatus::Deleted; + tombstone.deletion_date = Some(now.clone()); + self.client.save_master_key(&tombstone, &key_material).await?; + } + Err(error) => { + warn!(key_id, %error, "skipping tombstone for undecodable key record"); + } + } + } + KeyStatus::Active | KeyStatus::Disabled => return Ok(ExpiredKeyRemoval::StateChanged), + } + + durable_file::remove_durably(self.client.master_key_path(key_id)?) + .await + .map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?; + debug!(key_id, "Local KMS expired key removed"); + Ok(ExpiredKeyRemoval::Removed) + } } #[cfg(test)] @@ -2617,9 +2703,16 @@ mod tests { client.schedule_key_deletion(key_id, 7, None), client.enable_key(key_id, None), ); - disable.expect("disable"); - schedule.expect("schedule deletion"); - enable.expect("enable"); + // The per-key lock serializes the three transitions in an arbitrary + // order, and the state gate may legitimately reject a transition that + // lost the race (e.g. enable after deletion was scheduled). Any other + // error kind would still mean corrupted storage. + for result in [disable, schedule, enable] { + match result { + Ok(()) | Err(KmsError::InvalidOperation { .. }) => {} + Err(other) => panic!("concurrent transition must only fail with a state rejection, got {other:?}"), + } + } // Whatever the serialization order, the file must be one writer's // complete output with the original material intact. @@ -2634,4 +2727,67 @@ mod tests { "concurrent status updates must never lose or regenerate key material" ); } + + /// Records written before deadline persistence landed have no + /// deletion_date field and must keep deserializing (as None). + #[tokio::test] + async fn stored_master_key_without_deletion_date_still_deserializes() { + let (client, _temp_dir) = create_test_client().await; + client.create_key("legacy-key", "AES_256", None).await.expect("create key"); + + let path = client.master_key_path("legacy-key").expect("key path"); + let bytes = fs::read(&path).await.expect("read stored key"); + let mut value: serde_json::Value = serde_json::from_slice(&bytes).expect("stored key must be JSON"); + value + .as_object_mut() + .expect("stored key must be a JSON object") + .remove("deletion_date") + .expect("current records must carry the field"); + + let stored: StoredMasterKey = serde_json::from_value(value).expect("legacy record must deserialize"); + assert!(stored.deletion_date.is_none()); + } + + #[tokio::test] + async fn remove_expired_key_completes_a_tombstone_and_stays_idempotent() { + let temp_dir = TempDir::new().expect("temp dir"); + let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + let backend = LocalKmsBackend::new(config).await.expect("backend"); + let created = backend + .create_key(CreateKeyRequest { + key_name: Some("tombstoned-key".to_string()), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + }) + .await + .expect("create key"); + let key_id = created.key_id; + + // Craft the state a removal crashed in: tombstone written, file not + // yet removed. + let client = backend.lifecycle_client(); + let (_stored, key_material) = client.decode_stored_key(&key_id).await.expect("decode stored key"); + let mut tombstone = client.load_master_key(&key_id).await.expect("load key"); + tombstone.status = KeyStatus::Deleted; + tombstone.deletion_date = Some(Zoned::now()); + client + .save_master_key(&tombstone, &key_material) + .await + .expect("write tombstone"); + + // The sweep primitive completes the crashed removal... + let outcome = backend + .remove_expired_key(&key_id, &Zoned::now()) + .await + .expect("tombstone completion"); + assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed); + assert!(!client.master_key_path(&key_id).expect("key path").exists()); + + // ...and stays idempotent once the key is gone. + let outcome = backend + .remove_expired_key(&key_id, &Zoned::now()) + .await + .expect("repeat removal"); + assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed); + } } diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index 72d4df7c5..ae2d531b9 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -14,18 +14,89 @@ //! KMS backend implementations -use crate::error::Result; +use crate::error::{KmsError, Result}; use crate::types::*; use async_trait::async_trait; +use jiff::Zoned; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +#[cfg(test)] +mod contract_tests; pub mod local; pub mod static_kms; pub mod vault; pub(crate) mod vault_credentials; pub mod vault_transit; +/// Operations whose availability depends on the key's lifecycle state. +/// +/// Decryption is deliberately absent: RustFS allows decryption with +/// `Disabled` and `PendingDeletion` keys — an explicit deviation from AWS +/// KMS — because rejecting it would break reads of every object encrypted +/// under a key the moment it is disabled. Deletion cancellation is also +/// absent: it is valid exactly when the key is `PendingDeletion`, which call +/// sites enforce directly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StateGatedOperation { + Encrypt, + GenerateDataKey, + Rotate, + Enable, + Disable, + ScheduleDeletion, +} + +impl StateGatedOperation { + fn describe(self) -> &'static str { + match self { + Self::Encrypt => "encryption", + Self::GenerateDataKey => "data key generation", + Self::Rotate => "rotation", + Self::Enable => "enabling", + Self::Disable => "disabling", + Self::ScheduleDeletion => "deletion scheduling", + } + } +} + +/// Enforce the shared key state × operation matrix. +/// +/// - `Enabled`: every operation is allowed. +/// - `Disabled`: enabling, disabling (idempotent) and deletion scheduling are +/// allowed; encryption, data key generation and rotation are rejected. +/// - `PendingDeletion`: every state-gated operation is rejected, including a +/// repeated deletion schedule; only cancellation and decryption proceed. +/// - `PendingImport`/`Unavailable`: the key is not usable and is reported as +/// not found. +pub(crate) fn ensure_key_state_permits(key_id: &str, state: &KeyState, operation: StateGatedOperation) -> Result<()> { + match state { + KeyState::Enabled => Ok(()), + KeyState::Disabled => match operation { + StateGatedOperation::Enable | StateGatedOperation::Disable | StateGatedOperation::ScheduleDeletion => Ok(()), + StateGatedOperation::Encrypt | StateGatedOperation::GenerateDataKey | StateGatedOperation::Rotate => Err( + KmsError::invalid_key_state(format!("Key {key_id} is disabled: {} is not allowed", operation.describe())), + ), + }, + KeyState::PendingDeletion => Err(KmsError::invalid_key_state(format!( + "Key {key_id} is pending deletion: {} is not allowed", + operation.describe() + ))), + KeyState::PendingImport | KeyState::Unavailable => Err(KmsError::key_not_found(key_id)), + } +} + +/// [`ensure_key_state_permits`] for backends that persist [`KeyStatus`]. +pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operation: StateGatedOperation) -> Result<()> { + let state = match status { + KeyStatus::Active => KeyState::Enabled, + KeyStatus::Disabled => KeyState::Disabled, + KeyStatus::PendingDeletion => KeyState::PendingDeletion, + KeyStatus::Deleted => KeyState::Unavailable, + }; + ensure_key_state_permits(key_id, &state, operation) +} + /// Abstract KMS client interface that all backends must implement #[async_trait] pub trait KmsClient: Send + Sync { @@ -195,6 +266,35 @@ pub trait KmsBackend: Send + Sync { fn capabilities(&self) -> BackendCapabilities { BackendCapabilities::minimal() } + + /// Remove a key whose scheduled deletion deadline has passed. + /// + /// Used by the background deletion worker. Implementations must re-check + /// state and deadline under their own write synchronization so that a + /// concurrent cancellation observed after the caller's inspection wins + /// ([`ExpiredKeyRemoval::StateChanged`]), must write a tombstone (a + /// `Deleted`/`Unavailable` record) before destroying material so a crashed + /// removal can simply be re-run, and must treat an already-removed key as + /// success so the operation stays idempotent across restarts and nodes. + /// + /// The default rejects the operation for backends without deletion + /// support. + async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> { + Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key")) + } +} + +/// Outcome of [`KmsBackend::remove_expired_key`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExpiredKeyRemoval { + /// The key's record and material were removed, or were already gone. + Removed, + /// The key is no longer pending deletion (for example the deletion was + /// cancelled after the caller inspected it); nothing was removed. + StateChanged, + /// The key is pending deletion but its deadline has not passed, or it has + /// no persisted deadline (legacy record) and is never auto-removed. + NotExpired, } /// Information about a KMS backend @@ -421,6 +521,18 @@ mod tests { assert!(!capabilities.physical_delete); } + #[tokio::test] + async fn default_remove_expired_key_is_unsupported() { + let error = MinimalBackend + .remove_expired_key("any-key", &jiff::Zoned::now()) + .await + .expect_err("backends without deletion support must reject expired-key removal"); + assert!( + matches!(error, KmsError::UnsupportedCapability { .. }), + "expected UnsupportedCapability, got {error:?}" + ); + } + #[tokio::test] async fn local_backend_capabilities_golden() { let temp_dir = tempfile::tempdir().expect("temp dir should be created"); diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 0b77d6e39..610c70348 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -18,7 +18,10 @@ use crate::backends::vault_credentials::{ CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider, token_source_for, }; -use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{ + BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits, + ensure_key_status_permits, +}; use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; use crate::error::{KmsError, Result}; @@ -64,6 +67,10 @@ struct VaultKeyData { metadata: HashMap<String, String>, /// Key tags tags: HashMap<String, String>, + /// Scheduled deletion deadline; absent on records written before deadline + /// persistence landed, so it must stay optional for backward compatibility. + #[serde(default)] + deletion_date: Option<Zoned>, /// Encrypted key material (base64 encoded) encrypted_key_material: String, /// Version that pre-versioning envelopes (no `master_key_version`) resolve to. @@ -381,6 +388,7 @@ impl VaultKmsClient { description: request.description.clone(), metadata: existing_key_data.metadata.clone(), tags: request.tags.clone(), + deletion_date: existing_key_data.deletion_date.clone(), encrypted_key_material: existing_key_data.encrypted_key_material.clone(), // Preserve the key material baseline_version: existing_key_data.baseline_version, }; @@ -472,6 +480,9 @@ impl KmsClient for VaultKmsClient { async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> { debug!("Generating data key for master key: {}", request.master_key_id); + let key_data = self.get_key_data(&request.master_key_id).await?; + ensure_key_status_permits(&request.master_key_id, &key_data.status, StateGatedOperation::GenerateDataKey)?; + // Generate random data key material using the existing method let plaintext_key = generate_key_material(&request.key_spec)?; @@ -510,8 +521,9 @@ impl KmsClient for VaultKmsClient { async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { debug!("Encrypting data with key: {}", request.key_id); - // Get the master key + // Get the master key and verify its state allows encryption let key_data = self.get_key_data(&request.key_id).await?; + ensure_key_status_permits(&request.key_id, &key_data.status, StateGatedOperation::Encrypt)?; let key_material = self.decrypt_key_material(&key_data.encrypted_key_material).await?; // For simplicity, we'll use a basic encryption approach @@ -593,6 +605,7 @@ impl KmsClient for VaultKmsClient { description: None, metadata: HashMap::new(), tags: HashMap::new(), + deletion_date: None, encrypted_key_material: encrypted_material, baseline_version: None, }; @@ -611,6 +624,7 @@ impl KmsClient for VaultKmsClient { created_at: key_data.created_at, rotated_at: None, created_by: None, + deletion_date: None, }; debug!(key_id, "Vault KMS master key created"); @@ -678,6 +692,7 @@ impl KmsClient for VaultKmsClient { debug!("Enabling key: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Enable)?; key_data.status = KeyStatus::Active; self.store_key_data(key_id, &key_data).await?; @@ -689,6 +704,7 @@ impl KmsClient for VaultKmsClient { debug!("Disabling key: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Disable)?; key_data.status = KeyStatus::Disabled; self.store_key_data(key_id, &key_data).await?; @@ -699,13 +715,15 @@ impl KmsClient for VaultKmsClient { async fn schedule_key_deletion( &self, key_id: &str, - _pending_window_days: u32, + pending_window_days: u32, _context: Option<&OperationContext>, ) -> Result<()> { debug!("Scheduling key deletion: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?; key_data.status = KeyStatus::PendingDeletion; + key_data.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); self.store_key_data(key_id, &key_data).await?; debug!(key_id, "Vault KMS key deletion scheduled"); @@ -716,7 +734,11 @@ impl KmsClient for VaultKmsClient { debug!("Canceling key deletion: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; + if key_data.status != KeyStatus::PendingDeletion { + return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); + } key_data.status = KeyStatus::Active; + key_data.deletion_date = None; self.store_key_data(key_id, &key_data).await?; debug!(key_id, "Vault KMS key deletion canceled"); @@ -818,6 +840,7 @@ impl KmsClient for VaultKmsClient { created_at: key_data.created_at.clone(), rotated_at: Some(Zoned::now()), created_by: None, + deletion_date: key_data.deletion_date.clone(), }) } @@ -860,6 +883,12 @@ pub struct VaultKmsBackend { } impl VaultKmsBackend { + /// Lifecycle driver for the shared state-machine contract tests. + #[cfg(test)] + pub(crate) fn lifecycle_client(&self) -> &VaultKmsClient { + &self.client + } + /// Create a new VaultKmsBackend pub async fn new(config: KmsConfig) -> Result<Self> { config.validate()?; @@ -905,6 +934,7 @@ impl VaultKmsBackend { KeyState::Unavailable => KeyStatus::Deleted, KeyState::PendingImport => KeyStatus::Disabled, // Treat as disabled until import completes }; + key_data.deletion_date = metadata.deletion_date.clone(); // Update the key data in Vault storage self.client.store_key_data(key_id, &key_data).await?; @@ -1004,7 +1034,7 @@ impl KmsBackend for VaultKmsBackend { key_usage: key_info.usage, description: key_info.description, creation_date: key_info.created_at, - deletion_date: None, + deletion_date: key_data.deletion_date.clone(), origin: "VAULT".to_string(), key_manager: "VAULT".to_string(), tags: key_data.tags, @@ -1033,8 +1063,17 @@ impl KmsBackend for VaultKmsBackend { }; let deletion_date = if request.force_immediate.unwrap_or(false) { - // Check if key is already in PendingDeletion state - if key_metadata.key_state == KeyState::PendingDeletion { + // Check if key is already in PendingDeletion state (or a tombstone + // left by a crashed removal, which may simply be completed) + if key_metadata.key_state == KeyState::PendingDeletion || key_metadata.key_state == KeyState::Unavailable { + // Tombstone first: mark the record Deleted before removing it, + // so a crash between the two steps leaves a key that is already + // unusable and whose removal can simply be re-run. + if key_metadata.key_state == KeyState::PendingDeletion { + let mut key_data = self.client.get_key_data(key_id).await?; + key_data.status = KeyStatus::Deleted; + self.client.store_key_data(key_id, &key_data).await?; + } // Force immediate deletion: physically delete the key from Vault storage self.client.delete_key(key_id).await?; @@ -1052,6 +1091,8 @@ impl KmsBackend for VaultKmsBackend { } } else { // Schedule for deletion (default 30 days) + ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?; + let days = request.pending_window_in_days.unwrap_or(30); if !(7..=30).contains(&days) { return Err(crate::error::KmsError::invalid_parameter( @@ -1120,6 +1161,43 @@ impl KmsBackend for VaultKmsBackend { .with_schedule_deletion(true) .with_physical_delete(true) } + + async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> { + // Vault KV2 offers no compare-and-swap here, so a cancellation racing + // the read below can still lose; the window is a single read-write + // gap and the sweep re-reads on every pass. + let mut key_data = match self.client.get_key_data(key_id).await { + Ok(key_data) => key_data, + Err(KmsError::KeyNotFound { .. }) => return Ok(ExpiredKeyRemoval::Removed), + Err(error) => return Err(error), + }; + match key_data.status { + // Tombstone left by a crashed removal: complete it. + KeyStatus::Deleted => {} + KeyStatus::PendingDeletion => { + match &key_data.deletion_date { + Some(deadline) if deadline <= now => {} + // Not yet due, or a legacy record without a persisted + // deadline — never auto-remove those. + _ => return Ok(ExpiredKeyRemoval::NotExpired), + } + // Tombstone first: mark the record Deleted before removing it, + // so a crash between the two steps leaves a key that is + // already unusable and whose removal can simply be re-run. + key_data.status = KeyStatus::Deleted; + self.client.store_key_data(key_id, &key_data).await?; + } + KeyStatus::Active | KeyStatus::Disabled => return Ok(ExpiredKeyRemoval::StateChanged), + } + + match self.client.delete_key(key_id).await { + Ok(()) | Err(KmsError::KeyNotFound { .. }) => { + debug!(key_id, "Vault KV2 expired key removed"); + Ok(ExpiredKeyRemoval::Removed) + } + Err(error) => Err(error), + } + } } #[cfg(test)] @@ -1284,6 +1362,7 @@ mod tests { tags: HashMap::new(), encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]), baseline_version: Some(1), + deletion_date: None, }; let mut value = serde_json::to_value(&key_data).expect("serialize key data"); @@ -1642,4 +1721,41 @@ mod tests { "cancel_key_deletion must persist Active status to Vault, not only mutate the response" ); } + + /// The persisted KV2 record round-trips its deletion deadline, and records + /// written before the field existed keep deserializing (as None). A revert + /// of deadline persistence turns this test red. + #[test] + fn vault_key_data_deletion_date_round_trips_and_stays_backward_compatible() { + let deadline = Zoned::now() + Duration::from_secs(7 * 86400); + let key_data = VaultKeyData { + algorithm: "AES_256".to_string(), + usage: KeyUsage::EncryptDecrypt, + created_at: Zoned::now(), + status: KeyStatus::PendingDeletion, + version: 1, + description: None, + metadata: HashMap::new(), + tags: HashMap::new(), + deletion_date: Some(deadline.clone()), + encrypted_key_material: "material".to_string(), + baseline_version: None, + }; + + let mut value = serde_json::to_value(&key_data).expect("serialize"); + let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip"); + assert_eq!( + restored.deletion_date.as_ref().map(Zoned::timestamp), + Some(deadline.timestamp()), + "deletion deadline must survive the KV2 round trip" + ); + + value + .as_object_mut() + .expect("record must be a JSON object") + .remove("deletion_date") + .expect("current records must carry the field"); + let legacy: VaultKeyData = serde_json::from_value(value).expect("legacy record must deserialize"); + assert!(legacy.deletion_date.is_none()); + } } diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index a1355af75..b6d16b521 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -18,7 +18,9 @@ use crate::backends::vault_credentials::{ CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider, token_source_for, }; -use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{ + BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits, +}; use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; use crate::error::{KmsError, Result}; @@ -84,6 +86,12 @@ impl TransitKeyMetadata { } } + // KNOWN RISK (rustfs/backlog#1571, residual of rustfs/backlog#808): this + // fallback defaults to Enabled, so a key whose KV metadata read fails is + // treated as usable — a disabled or pending-deletion key can transiently + // "revive" on that path. State gates therefore only hold as strongly as + // metadata reads do. Changing the fallback is out of scope here; the + // synthesized_metadata_defaults_to_enabled test pins the current behavior. fn synthesized() -> Self { Self { key_usage: KeyUsage::EncryptDecrypt, @@ -370,14 +378,9 @@ impl VaultTransitKmsClient { }) } - async fn ensure_key_active(&self, key_id: &str) -> Result<TransitKeyMetadata> { + async fn ensure_key_state_allows(&self, key_id: &str, operation: StateGatedOperation) -> Result<TransitKeyMetadata> { let metadata = self.get_key_metadata(key_id).await?; - if metadata.key_state != KeyState::Enabled { - return Err(KmsError::invalid_operation(format!( - "Key {key_id} is not active (state: {:?})", - metadata.key_state - ))); - } + ensure_key_state_permits(key_id, &metadata.key_state, operation)?; Ok(metadata) } } @@ -385,7 +388,8 @@ impl VaultTransitKmsClient { #[async_trait] impl KmsClient for VaultTransitKmsClient { async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> { - self.ensure_key_active(&request.master_key_id).await?; + self.ensure_key_state_allows(&request.master_key_id, StateGatedOperation::GenerateDataKey) + .await?; let plaintext_key = generate_key_material(&request.key_spec)?; let encrypted_key = self @@ -416,7 +420,9 @@ impl KmsClient for VaultTransitKmsClient { } async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { - let metadata = self.ensure_key_active(&request.key_id).await?; + let metadata = self + .ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt) + .await?; let ciphertext = self .transit_encrypt(&request.key_id, &request.plaintext, &request.encryption_context) .await?; @@ -482,6 +488,7 @@ impl KmsClient for VaultTransitKmsClient { created_at: metadata.created_at, rotated_at: None, created_by: metadata.created_by, + deletion_date: None, }) } @@ -528,14 +535,16 @@ impl KmsClient for VaultTransitKmsClient { } async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - let mut metadata = self.get_key_metadata(key_id).await?; + // A pending deletion must be reverted through cancel_key_deletion, not + // silently by enabling, so the gate rejects PendingDeletion here. + let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Enable).await?; metadata.key_state = KeyState::Enabled; metadata.deletion_date = None; self.store_key_metadata(key_id, &metadata).await } async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - let mut metadata = self.get_key_metadata(key_id).await?; + let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Disable).await?; metadata.key_state = KeyState::Disabled; self.store_key_metadata(key_id, &metadata).await } @@ -546,7 +555,9 @@ impl KmsClient for VaultTransitKmsClient { pending_window_days: u32, _context: Option<&OperationContext>, ) -> Result<()> { - let mut metadata = self.get_key_metadata(key_id).await?; + let mut metadata = self + .ensure_key_state_allows(key_id, StateGatedOperation::ScheduleDeletion) + .await?; metadata.key_state = KeyState::PendingDeletion; metadata.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); self.store_key_metadata(key_id, &metadata).await @@ -554,12 +565,17 @@ impl KmsClient for VaultTransitKmsClient { async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { let mut metadata = self.get_key_metadata(key_id).await?; + if metadata.key_state != KeyState::PendingDeletion { + return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); + } metadata.key_state = KeyState::Enabled; metadata.deletion_date = None; self.store_key_metadata(key_id, &metadata).await } async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + self.ensure_key_state_allows(key_id, StateGatedOperation::Rotate).await?; + key::rotate(&self.vault()?.client, &self.config.mount_path, key_id) .await .map_err(|e| KmsError::backend_error(format!("Failed to rotate Vault Transit key {key_id}: {e}")))?; @@ -579,6 +595,7 @@ impl KmsClient for VaultTransitKmsClient { created_at: metadata.created_at, rotated_at: Some(Zoned::now()), created_by: metadata.created_by, + deletion_date: None, }) } @@ -600,6 +617,14 @@ pub struct VaultTransitKmsBackend { } impl VaultTransitKmsBackend { + /// Lifecycle driver for the shared state-machine contract tests. Using the + /// backend's own client keeps its in-process metadata cache coherent with + /// the transitions the tests perform. + #[cfg(test)] + pub(crate) fn lifecycle_client(&self) -> &VaultTransitKmsClient { + &self.client + } + pub async fn new(config: KmsConfig) -> Result<Self> { config.validate()?; @@ -736,6 +761,8 @@ impl KmsBackend for VaultTransitKmsBackend { None } } else { + ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?; + let days = request.pending_window_in_days.unwrap_or(30); if !(7..=30).contains(&days) { return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30")); @@ -788,6 +815,60 @@ impl KmsBackend for VaultTransitKmsBackend { .with_versioning(true) .with_physical_delete(true) } + + async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> { + // The transit key's existence anchors "already removed": once it is + // gone only stale scheduling metadata can remain, so clean that up. + match self.client.read_transit_key(key_id).await { + Ok(_) => {} + Err(KmsError::KeyNotFound { .. }) => { + self.client.delete_key_metadata(key_id).await?; + return Ok(ExpiredKeyRemoval::Removed); + } + Err(error) => return Err(error), + } + + // A metadata read failure synthesizes an Enabled record (see + // TransitKeyMetadata::synthesized), which lands in StateChanged below: + // the worker never destroys material based on synthesized state. + let mut metadata = self.client.get_key_metadata(key_id).await?; + match metadata.key_state { + // Tombstone left by a crashed removal: complete it. + KeyState::Unavailable => {} + KeyState::PendingDeletion => { + match &metadata.deletion_date { + Some(deadline) if deadline <= now => {} + // Not yet due, or no persisted deadline — never auto-remove. + _ => return Ok(ExpiredKeyRemoval::NotExpired), + } + // Tombstone first: an Unavailable record is rejected by every + // state gate, and a crashed removal can simply be re-run. + metadata.key_state = KeyState::Unavailable; + self.client.store_key_metadata(key_id, &metadata).await?; + } + KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => { + return Ok(ExpiredKeyRemoval::StateChanged); + } + } + + if !self.client.read_transit_key(key_id).await?.deletion_allowed { + let mut update_builder = UpdateKeyConfigurationRequestBuilder::default(); + update_builder.deletion_allowed(true); + key::update( + &self.client.vault()?.client, + &self.client.config.mount_path, + key_id, + Some(&mut update_builder), + ) + .await + .map_err(|e| KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")))?; + } + key::delete(&self.client.vault()?.client, &self.client.config.mount_path, key_id) + .await + .map_err(|e| KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")))?; + self.client.delete_key_metadata(key_id).await?; + Ok(ExpiredKeyRemoval::Removed) + } } #[cfg(test)] @@ -999,4 +1080,15 @@ mod tests { // Cleanup so repeated runs against the same Vault do not accumulate keys. let _ = client.schedule_key_deletion(&key_id, 7, None).await; } + + /// Pins the known-risk fallback documented on `TransitKeyMetadata::synthesized`: + /// when KV metadata cannot be read, the synthesized record defaults to Enabled, + /// which weakens every state gate on that path. If this test turns red the + /// fallback semantics changed on purpose — update the comment there as well. + #[test] + fn synthesized_metadata_defaults_to_enabled() { + let metadata = TransitKeyMetadata::synthesized(); + assert_eq!(metadata.key_state, KeyState::Enabled); + assert!(metadata.deletion_date.is_none()); + } } diff --git a/crates/kms/src/deletion_worker.rs b/crates/kms/src/deletion_worker.rs new file mode 100644 index 000000000..fba80b121 --- /dev/null +++ b/crates/kms/src/deletion_worker.rs @@ -0,0 +1,399 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Background worker that completes scheduled key deletions. +//! +//! Every sweep lists keys, picks the ones whose persisted deletion deadline +//! has passed (plus tombstones left by a crashed removal) and hands each to +//! [`KmsBackend::remove_expired_key`], which re-checks state under the +//! backend's own synchronization. The sweep is idempotent and keeps no state +//! of its own, so it is safe to re-run after a restart and safe to run on +//! every node of a deployment concurrently — a key is only ever removed while +//! its (re-read) record is an expired pending deletion or a tombstone. + +use crate::backends::{ExpiredKeyRemoval, KmsBackend}; +use crate::types::{KeyStatus, ListKeysRequest}; +use async_trait::async_trait; +use jiff::Zoned; +use std::sync::Arc; +use std::time::Duration; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; + +/// How often the worker looks for expired pending deletions. +pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60); + +/// Reports configuration that still references a KMS key. +/// +/// Consulted before any material is destroyed; a non-empty result blocks the +/// removal until the references disappear. Implementations live where the +/// referencing configuration lives (for example bucket encryption settings in +/// the server) and are injected via +/// [`crate::service_manager::KmsServiceManager::set_deletion_reference_checker`]. +#[async_trait] +pub trait DeletionReferenceChecker: Send + Sync { + /// Identifiers of configuration still referencing `key_id` (bucket names, + /// settings paths, ...). Errors must be reported as a reference so that + /// an unavailable checker never unblocks a deletion. + async fn references(&self, key_id: &str) -> Vec<String>; +} + +/// Outcome of one sweep, for logging and tests. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SweepReport { + /// Keys whose record and material were removed this sweep. + pub removed: Vec<String>, + /// Keys left in place because configuration still references them. + pub blocked: Vec<String>, + /// Keys that were pending but not yet due, without a persisted deadline, + /// or whose state changed between inspection and removal. + pub skipped: usize, + /// Keys whose removal attempt failed; retried on the next sweep. + pub failed: usize, +} + +pub(crate) struct DeletionWorker { + backend: Arc<dyn KmsBackend>, + default_key_id: Option<String>, + reference_checker: Option<Arc<dyn DeletionReferenceChecker>>, + interval: Duration, +} + +impl DeletionWorker { + pub(crate) fn new( + backend: Arc<dyn KmsBackend>, + default_key_id: Option<String>, + reference_checker: Option<Arc<dyn DeletionReferenceChecker>>, + ) -> Self { + Self { + backend, + default_key_id, + reference_checker, + interval: DEFAULT_SWEEP_INTERVAL, + } + } + + pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { self.run(cancel).await }) + } + + async fn run(self, cancel: CancellationToken) { + let mut ticker = tokio::time::interval(self.interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = cancel.cancelled() => { + debug!("KMS deletion worker stopped"); + return; + } + _ = ticker.tick() => {} + } + let report = self.sweep(&Zoned::now()).await; + if !report.removed.is_empty() || !report.blocked.is_empty() || report.failed > 0 { + info!( + removed = ?report.removed, + blocked = ?report.blocked, + skipped = report.skipped, + failed = report.failed, + "KMS deletion sweep completed" + ); + } + } + } + + /// Run one sweep at the given time. Exposed separately so tests can drive + /// the expiry logic deterministically. + pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport { + let mut report = SweepReport::default(); + let mut marker: Option<String> = None; + loop { + let request = ListKeysRequest { + limit: Some(100), + marker: marker.clone(), + usage_filter: None, + status_filter: None, + }; + let response = match self.backend.list_keys(request).await { + Ok(response) => response, + Err(error) => { + warn!(%error, "KMS deletion sweep could not list keys"); + report.failed += 1; + return report; + } + }; + for key in &response.keys { + if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) { + self.process_key(&key.key_id, now, &mut report).await; + } + } + if !response.truncated { + break; + } + match response.next_marker { + Some(next_marker) => marker = Some(next_marker), + None => break, + } + } + report + } + + async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) { + // Never remove a key that live configuration still points at. The + // default key check is built in; broader references (bucket + // encryption settings, ...) come from the injected checker. + if self.default_key_id.as_deref() == Some(key_id) { + warn!(key_id, "expired KMS key is still the default key; refusing removal"); + report.blocked.push(key_id.to_string()); + return; + } + if let Some(checker) = &self.reference_checker { + let references = checker.references(key_id).await; + if !references.is_empty() { + warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal"); + report.blocked.push(key_id.to_string()); + return; + } + } + + // The backend re-checks state and deadline under its own write + // synchronization, so a cancellation racing this sweep wins there. + match self.backend.remove_expired_key(key_id, now).await { + Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()), + Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1, + Err(error) => { + warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep"); + report.failed += 1; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backends::KmsClient as _; + use crate::backends::local::LocalKmsBackend; + use crate::config::KmsConfig; + use crate::error::KmsError; + use crate::types::{CreateKeyRequest, DeleteKeyRequest, DescribeKeyRequest, KeyState, KeyUsage}; + + async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<LocalKmsBackend> { + let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build")) + } + + async fn create_key(backend: &LocalKmsBackend, key_name: &str) -> String { + backend + .create_key(CreateKeyRequest { + key_name: Some(key_name.to_string()), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + }) + .await + .expect("key should be created") + .key_id + } + + async fn schedule(backend: &LocalKmsBackend, key_id: &str) { + backend + .delete_key(DeleteKeyRequest { + key_id: key_id.to_string(), + pending_window_in_days: Some(7), + force_immediate: None, + }) + .await + .expect("deletion should be scheduled"); + } + + fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker { + DeletionWorker::new(backend, None, None) + } + + fn after_window() -> Zoned { + Zoned::now() + Duration::from_secs(8 * 86400) + } + + async fn assert_key_gone(backend: &LocalKmsBackend, key_id: &str) { + let error = backend + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .expect_err("removed key must not be describable"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}"); + } + + #[tokio::test] + async fn sweep_removes_expired_pending_key_and_is_idempotent() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let backend = local_backend(&temp_dir).await; + let key_id = create_key(&backend, "expired-key").await; + schedule(&backend, &key_id).await; + + let worker = worker(backend.clone()); + + // Not yet due: nothing happens. + let report = worker.sweep(&Zoned::now()).await; + assert!(report.removed.is_empty()); + assert_eq!(report.skipped, 1); + assert_eq!(report.failed, 0); + + // Past the deadline: the key is removed. + let report = worker.sweep(&after_window()).await; + assert_eq!(report.removed, vec![key_id.clone()]); + assert_eq!(report.failed, 0); + assert_key_gone(&backend, &key_id).await; + + // Re-running the sweep after the removal is a no-op. + let report = worker.sweep(&after_window()).await; + assert_eq!(report, SweepReport::default()); + } + + #[tokio::test] + async fn cancelled_deletion_always_beats_the_sweep() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let backend = local_backend(&temp_dir).await; + let cancelled = create_key(&backend, "cancelled-key").await; + let doomed = create_key(&backend, "doomed-key").await; + schedule(&backend, &cancelled).await; + schedule(&backend, &doomed).await; + + backend + .cancel_key_deletion(crate::types::CancelKeyDeletionRequest { + key_id: cancelled.clone(), + }) + .await + .expect("cancel should succeed"); + + let report = worker(backend.clone()).sweep(&after_window()).await; + assert_eq!(report.removed, vec![doomed.clone()]); + assert_eq!(report.failed, 0); + + // The cancelled key survives, enabled and usable. + let described = backend + .describe_key(DescribeKeyRequest { + key_id: cancelled.clone(), + }) + .await + .expect("cancelled key must still exist"); + assert_eq!(described.key_metadata.key_state, KeyState::Enabled); + assert_key_gone(&backend, &doomed).await; + } + + #[tokio::test] + async fn default_key_and_external_references_block_removal() { + struct StaticReferences(Vec<String>); + + #[async_trait] + impl DeletionReferenceChecker for StaticReferences { + async fn references(&self, _key_id: &str) -> Vec<String> { + self.0.clone() + } + } + + let temp_dir = tempfile::tempdir().expect("temp dir"); + let backend = local_backend(&temp_dir).await; + let key_id = create_key(&backend, "referenced-key").await; + schedule(&backend, &key_id).await; + + // Blocked while it is the configured default key. + let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None); + let report = as_default.sweep(&after_window()).await; + assert_eq!(report.blocked, vec![key_id.clone()]); + assert!(report.removed.is_empty()); + + // Blocked while external configuration still references it. + let with_references = DeletionWorker::new( + backend.clone(), + None, + Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))), + ); + let report = with_references.sweep(&after_window()).await; + assert_eq!(report.blocked, vec![key_id.clone()]); + assert!(report.removed.is_empty()); + backend + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .expect("blocked key must still exist"); + + // Removed once nothing references it anymore. + let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new())))); + let report = unreferenced.sweep(&after_window()).await; + assert_eq!(report.removed, vec![key_id.clone()]); + } + + #[tokio::test] + async fn deadline_survives_backend_restart_and_sweep_completes_it() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let key_id; + { + let backend = local_backend(&temp_dir).await; + key_id = create_key(&backend, "restart-key").await; + schedule(&backend, &key_id).await; + } + + // "Restart": a fresh backend over the same directory must still see + // the persisted deadline... + let backend = local_backend(&temp_dir).await; + let described = backend + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .expect("key must survive the restart"); + assert_eq!(described.key_metadata.key_state, KeyState::PendingDeletion); + assert!( + described.key_metadata.deletion_date.is_some(), + "deletion deadline must survive a backend restart" + ); + + // ...and the worker completes the deletion without any new schedule call. + let report = worker(backend.clone()).sweep(&after_window()).await; + assert_eq!(report.removed, vec![key_id.clone()]); + assert_key_gone(&backend, &key_id).await; + } + + #[tokio::test(start_paused = true)] + async fn worker_loop_removes_due_keys_and_stops_on_cancel() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let backend = local_backend(&temp_dir).await; + let key_id = create_key(&backend, "loop-key").await; + // A zero-day window through the lifecycle client produces a deadline + // that is already due for the worker's wall-clock sweep. + backend + .lifecycle_client() + .schedule_key_deletion(&key_id, 0, None) + .await + .expect("schedule with zero window"); + + let cancel = CancellationToken::new(); + let task = worker(backend.clone()).spawn(cancel.clone()); + + // The paused clock auto-advances through the worker's interval ticks. + let mut removed = false; + for _ in 0..100 { + tokio::time::sleep(Duration::from_secs(1)).await; + if backend + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .is_err() + { + removed = true; + break; + } + } + assert!(removed, "worker loop must remove the due key"); + + cancel.cancel(); + task.await.expect("worker task must stop after cancellation"); + } +} diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index f05d45e16..e88369669 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -69,6 +69,7 @@ pub mod backends; pub mod backup; mod cache; pub mod config; +pub mod deletion_worker; mod encryption; mod error; pub mod manager; @@ -89,6 +90,7 @@ pub use api_types::{ UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse, }; pub use config::*; +pub use deletion_worker::DeletionReferenceChecker; pub use encryption::is_data_key_envelope; pub use error::{KmsError, KmsUnavailableError, Result}; pub use manager::KmsManager; diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index c8bf9e115..8f670e1c7 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -164,6 +164,12 @@ impl KmsManager { pub fn backend_capabilities(&self) -> crate::backends::BackendCapabilities { self.backend.capabilities() } + + /// Direct handle to the configured backend, bypassing the metadata cache. + /// Used by background maintenance that must observe fresh state. + pub(crate) fn backend(&self) -> Arc<dyn KmsBackend> { + self.backend.clone() + } } #[cfg(test)] diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index 7ec4453e8..e80d7bc77 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -17,6 +17,7 @@ use crate::backends::vault_credentials::CredentialTaskHandle; use crate::backends::{KmsBackend, local::LocalKmsBackend}; use crate::config::{BackendConfig, KmsConfig}; +use crate::deletion_worker::{DeletionReferenceChecker, DeletionWorker}; use crate::error::{KmsError, Result}; use crate::manager::KmsManager; use crate::service::ObjectEncryptionService; @@ -29,6 +30,7 @@ use std::sync::{ }; use subtle::ConstantTimeEq; use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; const LOG_COMPONENT_KMS: &str = "kms"; @@ -115,6 +117,40 @@ struct ServiceVersion { /// one. Stop shuts it down explicitly; reconfigure recycles it through /// the handle's cancel-on-drop behavior when the old version is discarded. credential_task: Option<Arc<CredentialTaskHandle>>, + /// Background deletion worker owned by this service version, if the + /// backend supports deletion scheduling + deletion_worker: Option<Arc<DeletionWorkerHandle>>, +} + +impl ServiceVersion { + fn shutdown_deletion_worker(&self) { + if let Some(worker) = &self.deletion_worker { + worker.shutdown(); + } + } +} + +/// Cancellation handle for one service version's deletion worker. +struct DeletionWorkerHandle { + cancel: CancellationToken, + task: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>, +} + +impl DeletionWorkerHandle { + fn shutdown(&self) { + self.cancel.cancel(); + if let Ok(mut task) = self.task.lock() { + // Detach: the task observes the cancelled token on its next poll. + drop(task.take()); + } + } +} + +impl Drop for DeletionWorkerHandle { + fn drop(&mut self) { + // Safety net for versions that are replaced without an explicit stop. + self.cancel.cancel(); + } } #[derive(Clone)] @@ -133,6 +169,8 @@ pub struct KmsServiceManager { /// Mutex to protect lifecycle operations (start, stop, reconfigure) /// This ensures only one lifecycle operation happens at a time lifecycle_mutex: Arc<Mutex<()>>, + /// External reference checker consulted before expired keys are removed + deletion_reference_checker: std::sync::RwLock<Option<Arc<dyn DeletionReferenceChecker>>>, } impl KmsServiceManager { @@ -146,9 +184,23 @@ impl KmsServiceManager { }), version_counter: Arc::new(AtomicU64::new(0)), lifecycle_mutex: Arc::new(Mutex::new(())), + deletion_reference_checker: std::sync::RwLock::new(None), } } + /// Install the reference checker consulted before the deletion worker + /// removes an expired key. Takes effect for workers spawned by the next + /// start or reconfigure. + pub fn set_deletion_reference_checker(&self, checker: Arc<dyn DeletionReferenceChecker>) { + if let Ok(mut slot) = self.deletion_reference_checker.write() { + *slot = Some(checker); + } + } + + fn deletion_reference_checker(&self) -> Option<Arc<dyn DeletionReferenceChecker>> { + self.deletion_reference_checker.read().ok().and_then(|slot| slot.clone()) + } + /// Get current service status pub async fn get_status(&self) -> KmsServiceStatus { self.state.load().status.clone() @@ -326,6 +378,9 @@ impl KmsServiceManager { // Atomically clear current service version (lock-free, instant) // Note: Existing Arc references will keep the service alive until operations complete let state = self.state.load_full(); + if let Some(current) = state.current_service.as_ref() { + current.shutdown_deletion_worker(); + } self.state.store(Arc::new(RuntimeState { config: state.config.clone(), status: if state.config.is_some() { @@ -540,6 +595,7 @@ impl KmsServiceManager { service: encryption_service, manager: kms_manager, credential_task, + deletion_worker: None, }) } @@ -551,7 +607,11 @@ impl KmsServiceManager { Ok(service_version) } - fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) { + fn publish_running(&self, config: KmsConfig, mut service_version: ServiceVersion) { + if let Some(previous) = self.state.load().current_service.as_ref() { + previous.shutdown_deletion_worker(); + } + service_version.deletion_worker = self.spawn_deletion_worker(&config, &service_version); self.state.store(Arc::new(RuntimeState { config: Some(config), status: KmsServiceStatus::Running, @@ -559,6 +619,24 @@ impl KmsServiceManager { })); } + /// Spawn the background deletion worker for a service version about to be + /// published, if its backend supports deletion scheduling. The worker is + /// only started at publish time so failed start/reconfigure candidates + /// never leak a running task. + fn spawn_deletion_worker(&self, config: &KmsConfig, service_version: &ServiceVersion) -> Option<Arc<DeletionWorkerHandle>> { + let backend = service_version.manager.backend(); + if !backend.capabilities().schedule_deletion { + return None; + } + let cancel = CancellationToken::new(); + let worker = DeletionWorker::new(backend, config.default_key_id.clone(), self.deletion_reference_checker()); + let task = worker.spawn(cancel.clone()); + Some(Arc::new(DeletionWorkerHandle { + cancel, + task: std::sync::Mutex::new(Some(task)), + })) + } + fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) { let current = self.state.load_full(); if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) { @@ -842,6 +920,66 @@ mod tests { assert!(matches!(current.backend_config, BackendConfig::Local(_))); } + #[tokio::test] + async fn deletion_worker_follows_the_service_lifecycle() { + use tempfile::TempDir; + + let key_dir = TempDir::new().expect("create local KMS directory"); + let mut config = KmsConfig::local(key_dir.path().to_path_buf()); + config.allow_insecure_dev_defaults = true; + let manager = KmsServiceManager::new(); + manager.configure(config).await.expect("configure local KMS"); + manager.start().await.expect("start local KMS"); + + let first_worker = manager + .state + .load() + .current_service + .as_ref() + .expect("running service") + .deletion_worker + .clone() + .expect("local backend must run a deletion worker"); + assert!(!first_worker.cancel.is_cancelled()); + + // Replacing the service version replaces (and cancels) its worker. + manager.restart().await.expect("restart"); + assert!(first_worker.cancel.is_cancelled(), "replaced version's worker must be cancelled"); + let second_worker = manager + .state + .load() + .current_service + .as_ref() + .expect("running service") + .deletion_worker + .clone() + .expect("restarted service must run a fresh worker"); + assert!(!second_worker.cancel.is_cancelled()); + + // Stopping the service stops its worker. + manager.stop().await.expect("stop"); + assert!(second_worker.cancel.is_cancelled(), "stop must cancel the deletion worker"); + } + + #[tokio::test] + async fn static_backend_runs_no_deletion_worker() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + + assert!( + manager + .state + .load() + .current_service + .as_ref() + .expect("running service") + .deletion_worker + .is_none(), + "a backend without deletion scheduling must not run a worker" + ); + } + #[tokio::test] async fn reconfigure_allows_safe_local_runtime_settings_only() { use tempfile::TempDir; diff --git a/crates/kms/src/types.rs b/crates/kms/src/types.rs index 39fda0ff4..f469b5936 100644 --- a/crates/kms/src/types.rs +++ b/crates/kms/src/types.rs @@ -117,6 +117,9 @@ pub struct MasterKeyInfo { pub rotated_at: Option<Zoned>, /// Key creator/owner pub created_by: Option<String>, + /// Scheduled deletion deadline while the key is pending deletion + #[serde(default)] + pub deletion_date: Option<Zoned>, } impl MasterKeyInfo { @@ -142,6 +145,7 @@ impl MasterKeyInfo { created_at: Zoned::now(), rotated_at: None, created_by, + deletion_date: None, } } @@ -173,6 +177,7 @@ impl MasterKeyInfo { created_at: Zoned::now(), rotated_at: None, created_by, + deletion_date: None, } } } From 1d383e239da163fe32e3eefda51540e90d20bc25 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 07:53:06 +0800 Subject: [PATCH 30/52] fix(iam): separate OIDC role and claim policies (#5493) --- crates/iam/src/oidc.rs | 85 ++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/crates/iam/src/oidc.rs b/crates/iam/src/oidc.rs index f0f7dcb2a..838e3bb98 100644 --- a/crates/iam/src/oidc.rs +++ b/crates/iam/src/oidc.rs @@ -1139,8 +1139,11 @@ impl OidcSys { let mut policies = Vec::new(); let mut groups = Vec::new(); - // Add default role policy if configured - if !config.role_policy.is_empty() { + // Role-policy and claim-based authorization are separate OIDC modes. When a + // role policy is configured, group claims still provide group context but + // must not also become policy names. + let has_role_policy = !config.role_policy.trim().is_empty(); + if has_role_policy { for policy in config.role_policy.split(',') { let policy = policy.trim(); if !policy.is_empty() { @@ -1149,21 +1152,20 @@ impl OidcSys { } } - // Map groups claim to policies for group in &claims.groups { groups.push(group.clone()); - let policy_name = if config.claim_prefix.is_empty() { - group.clone() - } else { - format!("{}{}", config.claim_prefix, group) - }; - policies.push(policy_name); + if !has_role_policy { + let policy_name = if config.claim_prefix.is_empty() { + group.clone() + } else { + format!("{}{}", config.claim_prefix, group) + }; + policies.push(policy_name); + } } - // Map primary claim (if different from groups) - if config.claim_name != config.groups_claim { - let claim_values = extract_groups_claim(&claims.raw, &config.claim_name); - for val in claim_values { + if !has_role_policy && config.claim_name != config.groups_claim { + for val in extract_groups_claim(&claims.raw, &config.claim_name) { let policy_name = if config.claim_prefix.is_empty() { val } else { @@ -3201,26 +3203,22 @@ mod tests { } #[test] - fn test_map_claims_to_policies_with_provider() { - let mut config = test_config("okta"); - config.role_policy = "readwrite".to_string(); - config.display_name = "Okta".to_string(); + fn role_policy_does_not_map_groups_as_policies() { + let mut config = test_config("authentik"); + config.role_policy = "consoleAdmin".to_string(); + config.claim_name = "policy".to_string(); let sys = make_test_sys(vec![config]); let claims = OidcClaims { - sub: "user123".to_string(), - email: "user@example.com".to_string(), - username: "user".to_string(), - groups: vec!["admin".to_string(), "devs".to_string()], - raw: HashMap::new(), + groups: vec!["authentik Admins".to_string(), "users".to_string()], + raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]), + ..Default::default() }; - let (policies, groups) = sys.map_claims_to_policies("okta", &claims); - assert_eq!(groups, vec!["admin", "devs"]); - assert!(policies.contains(&"readwrite".to_string())); - assert!(policies.contains(&"admin".to_string())); - assert!(policies.contains(&"devs".to_string())); + let (policies, groups) = sys.map_claims_to_policies("authentik", &claims); + assert_eq!(groups, vec!["authentik Admins", "users"]); + assert_eq!(policies, vec!["consoleAdmin"]); } #[test] @@ -3245,6 +3243,39 @@ mod tests { assert_eq!(policies.len(), 1); } + #[test] + fn blank_role_policy_uses_claim_mapping() { + let mut config = test_config("keycloak"); + config.role_policy = " ".to_string(); + + let sys = make_test_sys(vec![config]); + let claims = OidcClaims { + groups: vec!["readonly".to_string()], + ..Default::default() + }; + + let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims); + assert_eq!(groups, vec!["readonly"]); + assert_eq!(policies, vec!["readonly"]); + } + + #[test] + fn claim_mapping_keeps_groups_with_distinct_primary_claim() { + let mut config = test_config("keycloak"); + config.claim_name = "policy".to_string(); + + let sys = make_test_sys(vec![config]); + let claims = OidcClaims { + groups: vec!["developers".to_string()], + raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]), + ..Default::default() + }; + + let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims); + assert_eq!(groups, vec!["developers"]); + assert_eq!(policies, vec!["developers", "readonly"]); + } + #[test] fn test_list_providers() { let mut config = test_config("keycloak"); From b4901abd174c0a6358f85589dcc225cd2f3ab389 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 08:11:14 +0800 Subject: [PATCH 31/52] fix(admin): keep data usage endpoint available (#5494) --- rustfs/src/app/admin_usecase.rs | 6 ++---- rustfs/src/app/data_usage_snapshot_gating_test.rs | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index 977daf901..c3fe65d60 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -276,10 +276,8 @@ impl DefaultAdminUsecase { .await .map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?; if !Self::data_usage_snapshot_covers_namespace(&info, buckets.into_iter().map(|bucket| bucket.name)) { - return Err(Self::app_error( - S3ErrorCode::ServiceUnavailable, - "authoritative data usage snapshot is not available yet", - )); + // The default wire shape distinguishes unknown usage from a confirmed zero snapshot. + info = DataUsageInfo::default(); } let storage_info = StorageAdminApi::storage_info(store.as_ref()).await; diff --git a/rustfs/src/app/data_usage_snapshot_gating_test.rs b/rustfs/src/app/data_usage_snapshot_gating_test.rs index 8b55894f8..047806f9f 100644 --- a/rustfs/src/app/data_usage_snapshot_gating_test.rs +++ b/rustfs/src/app/data_usage_snapshot_gating_test.rs @@ -274,9 +274,20 @@ async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covere let before_endpoint = live_bucket_usage_computations(); let unknown = DefaultAdminUsecase::query_data_usage_info_with_store(ecstore.clone()) .await - .expect_err("an uncovered snapshot must be reported as temporarily unavailable"); + .expect("an uncovered snapshot must be reported as unknown without failing the endpoint"); - assert_eq!(unknown.code, s3s::S3ErrorCode::ServiceUnavailable); + assert!(unknown.last_update.is_none()); + assert!(!unknown.usage_snapshot_complete); + assert_eq!(unknown.buckets_count, 0); + assert_eq!(unknown.objects_total_count, 0); + assert_eq!(unknown.objects_total_size, 0); + assert!(unknown.buckets_usage.is_empty()); + assert!(unknown.bucket_sizes.is_empty()); + assert!(unknown.total_capacity > 0); + assert_eq!( + unknown.total_used_capacity, + unknown.total_capacity.saturating_sub(unknown.total_free_capacity) + ); assert_eq!( live_bucket_usage_computations(), before_endpoint, From cad8246ffb9f05ca515cbc7be94b76ce3d899ba7 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 08:51:02 +0800 Subject: [PATCH 32/52] feat(kms): route Vault operations through the retry policy engine (#5495) * test(kms): add a scripted loopback Vault for policy wiring tests A minimal HTTP/1.1 responder that serves canned Vault responses in order and records the method/path sequence, so wiring tests can assert exactly how many requests a code path performed (retries, read-confirm) without a live Vault server. * feat(kms): route Vault operations through the retry policy engine Wire every outbound vaultrs call in the KV2 and Transit backends through policy::execute, completing the wiring half of the operation policy work (the engine landed separately): - Reads (KV2 read/read_metadata/read_version/list, transit read/list/ encrypt/decrypt, health checks) run as ReadIdempotent: bounded retries with exponential backoff and jitter on 429, recoverable 5xx, and connection-level failures; 400/401/403/404 stay fatal. - Writes (KV2 set/CAS set/delete_metadata, transit create/update/rotate/ delete, metadata writes) run as MutatingNonIdempotent: exactly one attempt under the per-attempt timeout, never replayed. CAS conflicts in the rotation protocol pass through unchanged as the concurrency signal they are. - Each attempt takes a fresh credential snapshot, so a retry after a credential rotation uses the new token. - Read-confirm recovery for lost create responses: when a create finds an existing key that is exactly what it would have produced (same algorithm, enabled, usable material, and for request-level creates the same usage/description/tags), it reports the stored key as the create result instead of KeyAlreadyExists. Any divergence keeps failing. - Deletes treat already-deleted records as completed deletes (KV2 version records; transit metadata already did), so re-running an interrupted deletion converges. - A failed existence pre-check inside create now fails the create instead of falling through to a blind overwrite (fail closed). - The policy module sheds its allow(dead_code) now that it is wired. Wiring tests run against a scripted loopback Vault and assert request counts and endpoints for the retry, single-attempt, CAS-conflict, and read-confirm paths. Refs rustfs/backlog#1569 (part of rustfs/backlog#1562) --- crates/kms/Cargo.toml | 3 +- crates/kms/src/backends/mod.rs | 2 + crates/kms/src/backends/scripted_vault.rs | 159 +++++++ crates/kms/src/backends/vault.rs | 485 ++++++++++++++++---- crates/kms/src/backends/vault_transit.rs | 522 ++++++++++++++++++---- crates/kms/src/lib.rs | 3 - crates/kms/src/policy.rs | 30 +- 7 files changed, 1029 insertions(+), 175 deletions(-) create mode 100644 crates/kms/src/backends/scripted_vault.rs diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index e492f0e83..3c46a63a3 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -75,7 +75,8 @@ anyhow = { workspace = true } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } -tokio = { workspace = true, features = ["test-util"] } +# "net" backs the scripted loopback Vault used by the policy wiring tests. +tokio = { workspace = true, features = ["net", "test-util"] } [features] default = [] diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index ae2d531b9..eecc236e8 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -24,6 +24,8 @@ use std::collections::HashMap; #[cfg(test)] mod contract_tests; pub mod local; +#[cfg(test)] +pub(crate) mod scripted_vault; pub mod static_kms; pub mod vault; pub(crate) mod vault_credentials; diff --git a/crates/kms/src/backends/scripted_vault.rs b/crates/kms/src/backends/scripted_vault.rs new file mode 100644 index 000000000..19453ea46 --- /dev/null +++ b/crates/kms/src/backends/scripted_vault.rs @@ -0,0 +1,159 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Minimal scripted HTTP responder standing in for a Vault server. +//! +//! Wiring tests need to observe how many Vault requests a code path performs +//! (retries, read-confirm recovery) without a live Vault. The responder serves +//! one canned response per incoming request in order, closes the connection +//! after each response, and records the `METHOD /path` sequence for +//! assertions. It intentionally implements just enough HTTP/1.1 for the +//! `vaultrs` reqwest client: no keep-alive, no chunked bodies. + +use std::sync::{Arc, Mutex}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +/// One canned HTTP response. +pub(crate) struct ScriptedResponse { + status: u16, + body: String, +} + +impl ScriptedResponse { + /// A 200 response carrying `data` inside the standard Vault envelope. + pub(crate) fn ok(data: serde_json::Value) -> Self { + Self { + status: 200, + body: serde_json::json!({ + "request_id": "scripted", + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "data": data, + }) + .to_string(), + } + } + + /// An error response in Vault's `{"errors": [...]}` format. + pub(crate) fn error(status: u16, message: &str) -> Self { + Self { + status, + body: serde_json::json!({ "errors": [message] }).to_string(), + } + } +} + +/// A scripted stand-in Vault listening on a loopback port. +pub(crate) struct ScriptedVault { + /// Base address (`http://127.0.0.1:port`) to point a Vault client at. + pub(crate) address: String, + requests: Arc<Mutex<Vec<String>>>, +} + +impl ScriptedVault { + /// Bind a loopback listener and serve `responses` one per request. + /// + /// Requests beyond the script get a 599 error so a test that under-scripts + /// fails loudly instead of hanging. + pub(crate) async fn serve(responses: Vec<ScriptedResponse>) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scripted vault listener"); + let address = format!("http://{}", listener.local_addr().expect("scripted vault local addr")); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + + tokio::spawn(async move { + let mut responses = responses.into_iter(); + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let Some(request_line) = read_request(&mut stream).await else { + continue; + }; + recorded + .lock() + .expect("scripted vault request log poisoned") + .push(request_line); + let response = responses + .next() + .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted")); + let payload = format!( + "HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + response.status, + response.body.len(), + response.body + ); + let _ = stream.write_all(payload.as_bytes()).await; + let _ = stream.shutdown().await; + } + }); + + Self { address, requests } + } + + /// The `METHOD /path` lines of every request served so far, in order. + pub(crate) fn requests(&self) -> Vec<String> { + self.requests.lock().expect("scripted vault request log poisoned").clone() + } +} + +/// Read one HTTP/1.1 request (head plus content-length body) and return its +/// `METHOD /path` line. Draining the body before responding keeps the client +/// from seeing a connection reset while it is still writing. +async fn read_request(stream: &mut TcpStream) -> Option<String> { + let mut buffer = Vec::new(); + let mut chunk = [0u8; 4096]; + let head_end = loop { + if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + break position + 4; + } + let read = stream.read(&mut chunk).await.ok()?; + if read == 0 { + return None; + } + buffer.extend_from_slice(&chunk[..read]); + }; + + let head = String::from_utf8_lossy(&buffer[..head_end]).into_owned(); + let mut lines = head.lines(); + let request_line = lines.next()?; + let mut parts = request_line.split_whitespace(); + let method = parts.next()?; + let path = parts.next()?; + // rustify appends a lone "?" when an endpoint has no query parameters; + // strip it so assertions can use the plain path. + let path = path.strip_suffix('?').unwrap_or(path); + + let content_length: usize = lines + .filter_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse().ok())? + }) + .next() + .unwrap_or(0); + let mut remaining = content_length.saturating_sub(buffer.len() - head_end); + while remaining > 0 { + let read = stream.read(&mut chunk).await.ok()?; + if read == 0 { + break; + } + remaining = remaining.saturating_sub(read); + } + + Some(format!("{method} {path}")) +} diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 610c70348..ebc417833 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -25,14 +25,17 @@ use crate::backends::{ use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; use crate::error::{KmsError, Result}; +use crate::policy::{self, AttemptError, OpClass, RetryPolicy}; use crate::types::*; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose}; use jiff::Zoned; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::future::Future; use std::sync::Arc; use std::time::Duration; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; use vaultrs::{api::kv2::requests::SetSecretRequestOptions, error::ClientError, kv2}; @@ -46,6 +49,13 @@ pub struct VaultKmsClient { key_path_prefix: String, /// DEK encryption implementation dek_crypto: AesDekCrypto, + /// Budgets wrapping every outbound Vault call (see `crate::policy`). + retry: RetryPolicy, + /// Cancellation point for the operation executor: aborts in-flight + /// attempts and backoff sleeps. Owned by the client and currently never + /// triggered — shutdown drops the whole client — but kept as the single + /// hook a future lifecycle owner can cancel through. + cancel: CancellationToken, } /// Key data stored in Vault @@ -192,6 +202,8 @@ impl VaultKmsClient { key_path_prefix: config.key_path_prefix.clone(), config, dek_crypto: AesDekCrypto::new(), + retry: RetryPolicy::from_config(kms_config), + cancel: CancellationToken::new(), }) } @@ -204,6 +216,19 @@ impl VaultKmsClient { self.credentials.current() } + /// Run one Vault call under the operation policy. + /// + /// The closure performs a single classified attempt and takes a fresh + /// credential snapshot per attempt, so a retry after a credential rotation + /// uses the new token. + async fn run<T, F, Fut>(&self, operation: &'static str, class: OpClass, attempt: F) -> Result<T> + where + F: FnMut() -> Fut, + Fut: Future<Output = std::result::Result<T, AttemptError>>, + { + policy::execute(operation, class, &self.retry, &self.cancel, attempt).await + } + /// Get the full path for a key in Vault fn key_path(&self, key_id: &str) -> String { format!("{}/{}", self.key_path_prefix, key_id) @@ -244,14 +269,20 @@ impl VaultKmsClient { async fn get_key_version_record(&self, key_id: &str, version: u32) -> Result<VaultKeyVersionRecord> { let path = self.key_version_path(key_id, version); - let record: VaultKeyVersionRecord = - kv2::read(&self.vault()?.client, &self.kv_mount, &path) - .await - .map_err(|e| match e { - ClientError::ResponseWrapError => KmsError::key_version_not_found(key_id, version), - ClientError::APIError { code: 404, .. } => KmsError::key_version_not_found(key_id, version), - _ => KmsError::backend_error(format!("Failed to read key version record from Vault: {e}")), - })?; + let path = path.as_str(); + let record: VaultKeyVersionRecord = self + .run("vault_kv2_read_key_version", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::read(&vault.client, &self.kv_mount, path).await.map_err(|e| { + AttemptError::from_vaultrs(e, |e| match e { + ClientError::ResponseWrapError | ClientError::APIError { code: 404, .. } => { + KmsError::key_version_not_found(key_id, version) + } + e => KmsError::backend_error(format!("Failed to read key version record from Vault: {e}")), + }) + }) + }) + .await?; if record.version != version { return Err(KmsError::material_corrupt( @@ -284,26 +315,42 @@ impl VaultKmsClient { /// later write can be check-and-set against exactly this snapshot. async fn get_key_data_versioned(&self, key_id: &str) -> Result<(u32, VaultKeyData)> { let path = self.key_path(key_id); + let path = path.as_str(); - let metadata = kv2::read_metadata(&self.vault()?.client, &self.kv_mount, &path) - .await - .map_err(|e| match e { - ClientError::ResponseWrapError => KmsError::key_not_found(key_id), - ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), - _ => KmsError::backend_error(format!("Failed to read key metadata from Vault: {e}")), - })?; + let metadata = self + .run("vault_kv2_read_key_metadata", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::read_metadata(&vault.client, &self.kv_mount, path).await.map_err(|e| { + AttemptError::from_vaultrs(e, |e| match e { + ClientError::ResponseWrapError | ClientError::APIError { code: 404, .. } => { + KmsError::key_not_found(key_id) + } + e => KmsError::backend_error(format!("Failed to read key metadata from Vault: {e}")), + }) + }) + }) + .await?; let cas = u32::try_from(metadata.current_version) .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32")))?; // Read the exact secret version from the metadata to keep the (cas, data) // pair consistent even if another writer lands in between. - let key_data: VaultKeyData = kv2::read_version(&self.vault()?.client, &self.kv_mount, &path, metadata.current_version) - .await - .map_err(|e| match e { - ClientError::ResponseWrapError => KmsError::key_not_found(key_id), - ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), - _ => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), - })?; + let secret_version = metadata.current_version; + let key_data: VaultKeyData = self + .run("vault_kv2_read_key_at_version", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::read_version(&vault.client, &self.kv_mount, path, secret_version) + .await + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| match e { + ClientError::ResponseWrapError | ClientError::APIError { code: 404, .. } => { + KmsError::key_not_found(key_id) + } + e => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), + }) + }) + }) + .await?; Ok((cas, key_data)) } @@ -315,19 +362,29 @@ impl VaultKmsClient { /// further check-and-set writes. async fn cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result<u32> { let path = self.key_path(key_id); + let path = path.as_str(); - let written = - kv2::set_with_options(&self.vault()?.client, &self.kv_mount, &path, key_data, SetSecretRequestOptions { cas }) - .await - .map_err(|e| { - if is_cas_conflict(&e) { - KmsError::invalid_operation(format!( - "Concurrent modification of key {key_id} detected, retry the rotation" - )) - } else { - KmsError::backend_error(format!("Failed to store key in Vault: {e}")) - } - })?; + // Single attempt: replaying a lost-response write would double-apply + // the mutation, and a CAS conflict is a normal concurrency signal that + // must reach the caller untouched. + let written = self + .run("vault_kv2_cas_write_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas }) + .await + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + if is_cas_conflict(&e) { + KmsError::invalid_operation(format!( + "Concurrent modification of key {key_id} detected, retry the rotation" + )) + } else { + KmsError::backend_error(format!("Failed to store key in Vault: {e}")) + } + }) + }) + }) + .await?; u32::try_from(written.version) .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32"))) @@ -340,23 +397,42 @@ impl VaultKmsClient { /// existing record is acceptable. The record is never overwritten. async fn try_create_key_version_record(&self, key_id: &str, record: &VaultKeyVersionRecord) -> Result<bool> { let path = self.key_version_path(key_id, record.version); + let path = path.as_str(); - match kv2::set_with_options(&self.vault()?.client, &self.kv_mount, &path, record, SetSecretRequestOptions { cas: 0 }) - .await - { - Ok(_) => Ok(true), - Err(e) if is_cas_conflict(&e) => Ok(false), - Err(e) => Err(KmsError::backend_error(format!("Failed to store key version record in Vault: {e}"))), - } + // Single attempt: the create-only CAS makes a duplicate replay fail + // with a conflict, which the caller resolves by reading the record + // back, so retrying here would only mask that recovery path. + self.run("vault_kv2_create_key_version", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::set_with_options(&vault.client, &self.kv_mount, path, record, SetSecretRequestOptions { cas: 0 }).await { + Ok(_) => Ok(true), + Err(e) if is_cas_conflict(&e) => Ok(false), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to store key version record in Vault: {e}")) + })), + } + }) + .await } /// Store key data in Vault async fn store_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> { let path = self.key_path(key_id); + let path = path.as_str(); - kv2::set(&self.vault()?.client, &self.kv_mount, &path, key_data) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to store key in Vault: {e}")))?; + // Single attempt: this is a whole-record overwrite without a CAS + // precondition, so a replay after a lost response could clobber a + // concurrent writer. + self.run("vault_kv2_write_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::set(&vault.client, &self.kv_mount, path, key_data) + .await + .map(|_| ()) + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| KmsError::backend_error(format!("Failed to store key in Vault: {e}"))) + }) + }) + .await?; debug!("Stored key {} in Vault at path {}", key_id, path); Ok(()) @@ -404,14 +480,21 @@ impl VaultKmsClient { /// Retrieve key data from Vault async fn get_key_data(&self, key_id: &str) -> Result<VaultKeyData> { let path = self.key_path(key_id); + let path = path.as_str(); - let secret: VaultKeyData = kv2::read(&self.vault()?.client, &self.kv_mount, &path) - .await - .map_err(|e| match e { - vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id), - vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), - _ => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), - })?; + let secret: VaultKeyData = self + .run("vault_kv2_read_key", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::read(&vault.client, &self.kv_mount, path).await.map_err(|e| { + AttemptError::from_vaultrs(e, |e| match e { + ClientError::ResponseWrapError | ClientError::APIError { code: 404, .. } => { + KmsError::key_not_found(key_id) + } + e => KmsError::backend_error(format!("Failed to read key from Vault: {e}")), + }) + }) + }) + .await?; debug!("Retrieved key {} from Vault, tags: {:?}", key_id, secret.tags); Ok(secret) @@ -419,56 +502,87 @@ impl VaultKmsClient { /// List all keys stored in Vault async fn list_vault_keys(&self) -> Result<Vec<String>> { - // List keys under the prefix - match kv2::list(&self.vault()?.client, &self.kv_mount, &self.key_path_prefix).await { - Ok(keys) => { + // List keys under the prefix; `None` means the prefix does not exist + // yet (no keys were ever created). + let keys = self + .run("vault_kv2_list_keys", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::list(&vault.client, &self.kv_mount, &self.key_path_prefix).await { + Ok(keys) => Ok(Some(keys)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to list keys in Vault: {e}")) + })), + } + }) + .await?; + + match keys { + Some(keys) => { let keys = filter_key_directory_entries(keys); debug!("Found {} keys in Vault", keys.len()); Ok(keys) } - Err(vaultrs::error::ClientError::ResponseWrapError) => { - // No keys exist yet - Ok(Vec::new()) - } - Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => { - // Path doesn't exist - no keys exist yet + None => { debug!("Key path doesn't exist in Vault (404), returning empty list"); Ok(Vec::new()) } - Err(e) => Err(KmsError::backend_error(format!("Failed to list keys in Vault: {e}"))), } } /// Physically delete a key from Vault storage async fn delete_key(&self, key_id: &str) -> Result<()> { let path = self.key_path(key_id); + let path = path.as_str(); // Purge immutable version records first: if any purge fails, the top-level // record still exists and the deletion can be retried. The reverse order // would leave orphaned master key material in Vault after the key vanished. let versions_dir = self.key_versions_dir(key_id); - match kv2::list(&self.vault()?.client, &self.kv_mount, &versions_dir).await { - Ok(versions) => { - for version in versions { - let version_path = format!("{versions_dir}/{version}"); - kv2::delete_metadata(&self.vault()?.client, &self.kv_mount, &version_path) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to delete key version record from Vault: {e}")))?; + let versions_dir = versions_dir.as_str(); + // `None` means no version records exist (the key was never rotated). + let versions = self + .run("vault_kv2_list_key_versions", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::list(&vault.client, &self.kv_mount, versions_dir).await { + Ok(versions) => Ok(Some(versions)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to list key version records in Vault: {e}")) + })), } - } - // No version records exist (the key was never rotated). - Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => {} - Err(e) => return Err(KmsError::backend_error(format!("Failed to list key version records in Vault: {e}"))), + }) + .await?; + for version in versions.unwrap_or_default() { + let version_path = format!("{versions_dir}/{version}"); + let version_path = version_path.as_str(); + self.run("vault_kv2_delete_key_version", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::delete_metadata(&vault.client, &self.kv_mount, version_path).await { + // A version record that is already gone is a completed + // delete (e.g. this deletion is being re-run after a lost + // response), not a failure. + Ok(_) | Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(()), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to delete key version record from Vault: {e}")) + })), + } + }) + .await?; } // For this specific key path, we can safely delete the metadata // since each key has its own unique path under the prefix - kv2::delete_metadata(&self.vault()?.client, &self.kv_mount, &path) - .await - .map_err(|e| match e { - vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), - _ => KmsError::backend_error(format!("Failed to delete key metadata from Vault: {e}")), - })?; + self.run("vault_kv2_delete_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::delete_metadata(&vault.client, &self.kv_mount, path).await.map_err(|e| { + AttemptError::from_vaultrs(e, |e| match e { + ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), + e => KmsError::backend_error(format!("Failed to delete key metadata from Vault: {e}")), + }) + }) + }) + .await?; debug!("Permanently deleted key {} metadata from Vault at path {}", key_id, path); Ok(()) @@ -586,9 +700,43 @@ impl KmsClient for VaultKmsClient { async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { debug!("Creating master key: {} with algorithm: {}", key_id, algorithm); - // Check if key already exists - if self.get_key_data(key_id).await.is_ok() { - return Err(KmsError::key_already_exists(key_id)); + // Existence pre-check with read-confirm recovery: a create whose + // response was lost gets retried by callers, and used to be + // misreported as KeyAlreadyExists. If the stored key is exactly what + // this create would have produced (same algorithm, active, usable + // material), report the stored key as the create result. Anything + // else keeps failing: create never adopts a key it would not have + // produced. A failed pre-check read must fail the create rather than + // fall through to a blind overwrite of a possibly existing key. + match self.get_key_data(key_id).await { + Ok(existing) => { + return if existing.algorithm == algorithm + && existing.status == KeyStatus::Active + && decode_stored_key_material(key_id, &existing.encrypted_key_material).is_ok() + { + info!( + key_id, + "Vault KMS create found an identical active key; treating it as a recovered create" + ); + Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: existing.version, + algorithm: existing.algorithm, + usage: existing.usage, + status: existing.status, + description: existing.description, + metadata: existing.metadata, + created_at: existing.created_at, + rotated_at: None, + created_by: None, + deletion_date: existing.deletion_date, + }) + } else { + Err(KmsError::key_already_exists(key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } // Generate key material @@ -1203,8 +1351,183 @@ impl KmsBackend for VaultKmsBackend { #[cfg(test)] mod tests { use super::*; + use crate::backends::scripted_vault::{ScriptedResponse, ScriptedVault}; use crate::config::{VaultAuthMethod, VaultConfig}; + /// Vault + KMS config pair pointing at a scripted loopback Vault. + fn scripted_configs(address: &str) -> (VaultConfig, KmsConfig) { + let vault_config = VaultConfig { + address: address.to_string(), + auth_method: VaultAuthMethod::Token { + token: "scripted-token".to_string(), + }, + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/keys".to_string(), + mount_path: "transit".to_string(), + namespace: None, + tls: None, + }; + let kms_config = KmsConfig { + timeout: Duration::from_secs(5), + retry_attempts: 3, + ..KmsConfig::default() + }; + (vault_config, kms_config) + } + + async fn scripted_client(responses: Vec<ScriptedResponse>) -> (ScriptedVault, VaultKmsClient) { + let vault = ScriptedVault::serve(responses).await; + let (vault_config, kms_config) = scripted_configs(&vault.address); + let client = VaultKmsClient::new(vault_config, &kms_config) + .await + .expect("scripted Vault client"); + (vault, client) + } + + fn healthy_key_data() -> VaultKeyData { + VaultKeyData { + algorithm: "AES_256".to_string(), + usage: KeyUsage::EncryptDecrypt, + created_at: Zoned::now(), + status: KeyStatus::Active, + version: 1, + description: None, + metadata: HashMap::new(), + tags: HashMap::new(), + deletion_date: None, + encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]), + baseline_version: None, + } + } + + /// KV2 read payload (the `data` field of the Vault envelope) for a key record. + fn kv2_read_data(key_data: &VaultKeyData) -> serde_json::Value { + serde_json::json!({ + "data": serde_json::to_value(key_data).expect("serialize key data"), + "metadata": { + "created_time": "2026-01-01T00:00:00Z", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1, + }, + }) + } + + #[tokio::test] + async fn wired_read_retries_transient_status_then_succeeds() { + let (vault, client) = scripted_client(vec![ + ScriptedResponse::error(503, "temporarily unavailable"), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ]) + .await; + + let key_data = client + .get_key_data("wired-key") + .await + .expect("read must retry past a transient 503"); + assert_eq!(key_data.algorithm, "AES_256"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "one failed attempt plus one retry: {requests:?}"); + assert!( + requests + .iter() + .all(|line| line == "GET /v1/secret/data/rustfs/kms/keys/wired-key"), + "both attempts must hit the same read endpoint: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_read_does_not_retry_permission_errors() { + let (vault, client) = scripted_client(vec![ScriptedResponse::error(403, "permission denied")]).await; + + client + .get_key_data("wired-key") + .await + .expect_err("a 403 must fail the read outright"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 1, "fatal statuses must not be retried: {requests:?}"); + } + + #[tokio::test] + async fn wired_write_is_never_retried_on_transient_status() { + let (vault, client) = scripted_client(vec![ScriptedResponse::error(503, "sealed")]).await; + + let error = client + .store_key_data("wired-key", &healthy_key_data()) + .await + .expect_err("the scripted 503 must fail the write"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!( + requests, + vec!["POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string()], + "a non-idempotent write must run exactly once even on a retryable status" + ); + } + + #[tokio::test] + async fn wired_cas_conflict_is_surfaced_without_retry() { + let (vault, client) = scripted_client(vec![ScriptedResponse::error( + 400, + "check-and-set parameter did not match the current version", + )]) + .await; + + let error = client + .cas_store_key_data("wired-key", &healthy_key_data(), 7) + .await + .expect_err("the scripted CAS conflict must fail the write"); + assert!( + matches!(error, KmsError::InvalidOperation { .. }), + "a CAS conflict is a concurrency signal, not a backend failure: {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 1, "a CAS conflict must never be retried: {requests:?}"); + } + + #[tokio::test] + async fn wired_create_key_read_confirms_identical_existing_key() { + // The stored key is exactly what create_key("wired-key", "AES_256") + // would have produced, so a retried create whose first response was + // lost recovers by reading it back instead of failing. + let (vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&healthy_key_data()))]).await; + + let recovered = client + .create_key("wired-key", "AES_256", None) + .await + .expect("an identical active key must read-confirm as a recovered create"); + assert_eq!(recovered.version, 1); + assert_eq!(recovered.algorithm, "AES_256"); + + let requests = vault.requests(); + assert_eq!( + requests, + vec!["GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string()], + "a recovered create must not write anything" + ); + } + + #[tokio::test] + async fn wired_create_key_still_fails_on_mismatched_existing_key() { + let mut disabled = healthy_key_data(); + disabled.status = KeyStatus::Disabled; + let (vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&disabled))]).await; + + let error = client + .create_key("wired-key", "AES_256", None) + .await + .expect_err("a non-active existing key must keep failing the create"); + assert!(matches!(error, KmsError::KeyAlreadyExists { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 1, "the mismatch must be decided from the single read: {requests:?}"); + } + /// Poison matrix for the read-side material gate. Every corruption class must fail /// closed with its typed error; reintroducing any "self-heal" (regenerate on empty or /// undecodable material) turns one of these expected errors into an Ok and fails the diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index b6d16b521..79d5219b7 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -24,15 +24,19 @@ use crate::backends::{ use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; use crate::error::{KmsError, Result}; +use crate::policy::{self, AttemptError, OpClass, RetryPolicy}; use crate::types::*; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use jiff::Zoned; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; +use std::future::Future; use std::sync::Arc; use std::time::Duration; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; +use tracing::info; use vaultrs::{ api::transit::{ KeyType, @@ -147,6 +151,13 @@ pub struct VaultTransitKmsClient { /// Path prefix under metadata_kv_mount for storing transit key metadata records metadata_key_prefix: String, metadata_cache: RwLock<HashMap<String, TransitKeyMetadata>>, + /// Budgets wrapping every outbound Vault call (see `crate::policy`). + retry: RetryPolicy, + /// Cancellation point for the operation executor: aborts in-flight + /// attempts and backoff sleeps. Owned by the client and currently never + /// triggered — shutdown drops the whole client — but kept as the single + /// hook a future lifecycle owner can cancel through. + cancel: CancellationToken, } impl VaultTransitKmsClient { @@ -171,6 +182,8 @@ impl VaultTransitKmsClient { metadata_key_prefix: config.metadata_key_prefix.clone(), config, metadata_cache: RwLock::new(HashMap::new()), + retry: RetryPolicy::from_config(kms_config), + cancel: CancellationToken::new(), }) } @@ -183,6 +196,19 @@ impl VaultTransitKmsClient { self.credentials.current() } + /// Run one Vault call under the operation policy. + /// + /// The closure performs a single classified attempt and takes a fresh + /// credential snapshot per attempt, so a retry after a credential rotation + /// uses the new token. + async fn run<T, F, Fut>(&self, operation: &'static str, class: OpClass, attempt: F) -> Result<T> + where + F: FnMut() -> Fut, + Fut: Future<Output = std::result::Result<T, AttemptError>>, + { + policy::execute(operation, class, &self.retry, &self.cancel, attempt).await + } + fn canonicalize_context(encryption_context: &HashMap<String, String>) -> Result<Option<String>> { if encryption_context.is_empty() { return Ok(None); @@ -196,28 +222,40 @@ impl VaultTransitKmsClient { Ok(Some(BASE64.encode(serialized))) } - fn map_vault_error<T>(key_id: &str, error: vaultrs::error::ClientError, operation: &str) -> Result<T> { + fn map_vault_error(key_id: &str, error: vaultrs::error::ClientError, operation: &str) -> KmsError { match error { - vaultrs::error::ClientError::ResponseWrapError => Err(KmsError::key_not_found(key_id)), - vaultrs::error::ClientError::APIError { code: 404, .. } => Err(KmsError::key_not_found(key_id)), - other => Err(KmsError::backend_error(format!( - "Vault Transit {operation} failed for key {key_id}: {other}" - ))), + vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id), + vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id), + other => KmsError::backend_error(format!("Vault Transit {operation} failed for key {key_id}: {other}")), } } async fn read_transit_key(&self, key_id: &str) -> Result<vaultrs::api::transit::responses::ReadKeyResponse> { - key::read(&self.vault()?.client, &self.config.mount_path, key_id) - .await - .or_else(|e| Self::map_vault_error(key_id, e, "read")) + self.run("vault_transit_read_key", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + key::read(&vault.client, &self.config.mount_path, key_id) + .await + .map_err(|e| AttemptError::from_vaultrs(e, |e| Self::map_vault_error(key_id, e, "read"))) + }) + .await } async fn create_transit_key(&self, key_id: &str) -> Result<()> { - let mut builder = CreateKeyRequestBuilder::default(); - builder.key_type(KeyType::Aes256Gcm96); - key::create(&self.vault()?.client, &self.config.mount_path, key_id, Some(&mut builder)) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to create Vault Transit key {key_id}: {e}"))) + // Single attempt: create carries external side effects and the caller + // owns the read-confirm recovery for lost responses. + self.run("vault_transit_create_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + let mut builder = CreateKeyRequestBuilder::default(); + builder.key_type(KeyType::Aes256Gcm96); + key::create(&vault.client, &self.config.mount_path, key_id, Some(&mut builder)) + .await + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to create Vault Transit key {key_id}: {e}")) + }) + }) + }) + .await } async fn transit_encrypt( @@ -227,14 +265,26 @@ impl VaultTransitKmsClient { encryption_context: &HashMap<String, String>, ) -> Result<String> { let plaintext_b64 = BASE64.encode(plaintext); - let mut builder = EncryptDataRequestBuilder::default(); - if let Some(aad) = Self::canonicalize_context(encryption_context)? { - builder.associated_data(aad); - } + let plaintext_b64 = plaintext_b64.as_str(); + let aad = Self::canonicalize_context(encryption_context)?; + let aad = aad.as_deref(); - let response = data::encrypt(&self.vault()?.client, &self.config.mount_path, key_id, &plaintext_b64, Some(&mut builder)) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to encrypt data with Vault Transit key {key_id}: {e}")))?; + let response = self + .run("vault_transit_encrypt", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + let mut builder = EncryptDataRequestBuilder::default(); + if let Some(aad) = aad { + builder.associated_data(aad); + } + data::encrypt(&vault.client, &self.config.mount_path, key_id, plaintext_b64, Some(&mut builder)) + .await + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to encrypt data with Vault Transit key {key_id}: {e}")) + }) + }) + }) + .await?; Ok(response.ciphertext) } @@ -245,14 +295,25 @@ impl VaultTransitKmsClient { ciphertext: &str, encryption_context: &HashMap<String, String>, ) -> Result<Vec<u8>> { - let mut builder = DecryptDataRequestBuilder::default(); - if let Some(aad) = Self::canonicalize_context(encryption_context)? { - builder.associated_data(aad); - } + let aad = Self::canonicalize_context(encryption_context)?; + let aad = aad.as_deref(); - let response = data::decrypt(&self.vault()?.client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to decrypt data with Vault Transit key {key_id}: {e}")))?; + let response = self + .run("vault_transit_decrypt", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + let mut builder = DecryptDataRequestBuilder::default(); + if let Some(aad) = aad { + builder.associated_data(aad); + } + data::decrypt(&vault.client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) + .await + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to decrypt data with Vault Transit key {key_id}: {e}")) + }) + }) + }) + .await?; BASE64 .decode(response.plaintext) @@ -265,33 +326,93 @@ impl VaultTransitKmsClient { async fn read_metadata_from_kv(&self, key_id: &str) -> Result<Option<TransitKeyMetadata>> { let path = self.metadata_key_path(key_id); - match kv2::read::<TransitKeyMetadataPersisted>(&self.vault()?.client, &self.metadata_kv_mount, &path).await { - Ok(persisted) => Ok(Some(persisted.into())), - Err(vaultrs::error::ClientError::ResponseWrapError) - | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(None), - Err(e) => Err(KmsError::backend_error(format!("Failed to read transit key metadata from Vault KV: {e}"))), - } + let path = path.as_str(); + self.run("vault_transit_read_metadata", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::read::<TransitKeyMetadataPersisted>(&vault.client, &self.metadata_kv_mount, path).await { + Ok(persisted) => Ok(Some(persisted.into())), + Err(vaultrs::error::ClientError::ResponseWrapError) + | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to read transit key metadata from Vault KV: {e}")) + })), + } + }) + .await } async fn write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { let path = self.metadata_key_path(key_id); + let path = path.as_str(); let persisted: TransitKeyMetadataPersisted = metadata.clone().into(); - kv2::set(&self.vault()?.client, &self.metadata_kv_mount, &path, &persisted) - .await - .map(|_| ()) - .map_err(|e| KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}"))) + let persisted = &persisted; + // Single attempt: this is a whole-record overwrite without a CAS + // precondition, so a replay after a lost response could clobber a + // concurrent writer. + self.run("vault_transit_write_metadata", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + kv2::set(&vault.client, &self.metadata_kv_mount, path, persisted) + .await + .map(|_| ()) + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}")) + }) + }) + }) + .await } async fn delete_metadata_from_kv(&self, key_id: &str) -> Result<()> { let path = self.metadata_key_path(key_id); - match kv2::delete_metadata(&self.vault()?.client, &self.metadata_kv_mount, &path).await { - Ok(_) => Ok(()), - Err(vaultrs::error::ClientError::ResponseWrapError) - | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(()), - Err(e) => Err(KmsError::backend_error(format!( - "Failed to delete transit key metadata from Vault KV: {e}" - ))), - } + let path = path.as_str(); + self.run("vault_transit_delete_metadata", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::delete_metadata(&vault.client, &self.metadata_kv_mount, path).await { + // Metadata that is already gone is a completed delete. + Ok(_) + | Err(vaultrs::error::ClientError::ResponseWrapError) + | Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(()), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to delete transit key metadata from Vault KV: {e}")) + })), + } + }) + .await + } + + /// Flip `deletion_allowed` on the transit key so it can be deleted. + async fn allow_transit_key_deletion(&self, key_id: &str) -> Result<()> { + self.run("vault_transit_allow_deletion", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + let mut builder = UpdateKeyConfigurationRequestBuilder::default(); + builder.deletion_allowed(true); + key::update(&vault.client, &self.config.mount_path, key_id, Some(&mut builder)) + .await + .map(|_| ()) + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")) + }) + }) + }) + .await + } + + /// Physically delete the transit key material in Vault. + async fn delete_transit_key(&self, key_id: &str) -> Result<()> { + self.run("vault_transit_delete_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + key::delete(&vault.client, &self.config.mount_path, key_id) + .await + .map(|_| ()) + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")) + }) + }) + }) + .await } async fn get_key_metadata(&self, key_id: &str) -> Result<TransitKeyMetadata> { @@ -462,8 +583,40 @@ impl KmsClient for VaultTransitKmsClient { return Err(KmsError::unsupported_algorithm(algorithm)); } - if self.read_transit_key(key_id).await.is_ok() { - return Err(KmsError::key_already_exists(key_id)); + // Existence pre-check with read-confirm recovery: a create whose + // response was lost gets retried by callers, and used to be + // misreported as KeyAlreadyExists. Transit keys are always AES-256, + // so an existing enabled key of the default usage is exactly what + // this create would have produced; report it as the create result. + // Anything else keeps failing. A failed pre-check read must fail the + // create rather than fall through to re-creating over an unknown key. + match self.read_transit_key(key_id).await { + Ok(_) => { + let existing = self.get_key_metadata(key_id).await?; + return if existing.key_state == KeyState::Enabled && existing.key_usage == KeyUsage::EncryptDecrypt { + info!( + key_id, + "Vault Transit create found an identical enabled key; treating it as a recovered create" + ); + Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: existing.current_version, + algorithm: algorithm.to_string(), + usage: existing.key_usage, + status: KeyStatus::Active, + description: existing.description, + metadata: existing.tags.clone(), + created_at: existing.created_at, + rotated_at: None, + created_by: existing.created_by, + deletion_date: None, + }) + } else { + Err(KmsError::key_already_exists(key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } self.create_transit_key(key_id).await?; @@ -497,9 +650,14 @@ impl KmsClient for VaultTransitKmsClient { } async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { - let all_keys = key::list(&self.vault()?.client, &self.config.mount_path) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to list Vault Transit keys: {e}")))? + let all_keys = self + .run("vault_transit_list_keys", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + key::list(&vault.client, &self.config.mount_path).await.map_err(|e| { + AttemptError::from_vaultrs(e, |e| KmsError::backend_error(format!("Failed to list Vault Transit keys: {e}"))) + }) + }) + .await? .keys; let mut filtered = Vec::new(); @@ -576,9 +734,20 @@ impl KmsClient for VaultTransitKmsClient { async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { self.ensure_key_state_allows(key_id, StateGatedOperation::Rotate).await?; - key::rotate(&self.vault()?.client, &self.config.mount_path, key_id) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to rotate Vault Transit key {key_id}: {e}")))?; + // Single attempt, never retried: replaying a rotate whose response was + // lost would advance the key version once more per replay. + self.run("vault_transit_rotate_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + key::rotate(&vault.client, &self.config.mount_path, key_id) + .await + .map(|_| ()) + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to rotate Vault Transit key {key_id}: {e}")) + }) + }) + }) + .await?; let mut metadata = self.get_key_metadata(key_id).await?; metadata.current_version += 1; @@ -600,10 +769,16 @@ impl KmsClient for VaultTransitKmsClient { } async fn health_check(&self) -> Result<()> { - key::list(&self.vault()?.client, &self.config.mount_path) - .await - .map(|_| ()) - .map_err(|e| KmsError::backend_error(format!("Vault Transit health check failed: {e}"))) + self.run("vault_transit_health_check", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + key::list(&vault.client, &self.config.mount_path) + .await + .map(|_| ()) + .map_err(|e| { + AttemptError::from_vaultrs(e, |e| KmsError::backend_error(format!("Vault Transit health check failed: {e}"))) + }) + }) + .await } fn backend_info(&self) -> BackendInfo { @@ -660,8 +835,46 @@ impl VaultTransitKmsBackend { impl KmsBackend for VaultTransitKmsBackend { async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> { let key_id = request.key_name.clone().unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); - if self.client.read_transit_key(&key_id).await.is_ok() { - return Err(KmsError::key_already_exists(&key_id)); + + // Existence pre-check with read-confirm recovery: a create whose + // response was lost gets retried by callers, and used to be + // misreported as KeyAlreadyExists. If the stored record is exactly + // what this request would have written, report it as the create + // result; any divergence keeps failing so a create can never adopt or + // reshape a key it would not have produced. + match self.client.read_transit_key(&key_id).await { + Ok(_) => { + let existing = self.client.get_key_metadata(&key_id).await?; + let requested = TransitKeyMetadata::from_create_request(&request); + return if existing.key_state == KeyState::Enabled + && existing.key_usage == requested.key_usage + && existing.description == requested.description + && existing.tags == requested.tags + { + info!( + key_id, + "Vault Transit create found an identical enabled key; treating it as a recovered create" + ); + Ok(CreateKeyResponse { + key_id: key_id.clone(), + key_metadata: KeyMetadata { + key_id, + key_state: existing.key_state, + key_usage: existing.key_usage, + description: existing.description, + creation_date: existing.created_at, + deletion_date: existing.deletion_date, + origin: existing.origin, + key_manager: "VAULT_TRANSIT".to_string(), + tags: existing.tags, + }, + }) + } else { + Err(KmsError::key_already_exists(&key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } self.client.create_transit_key(&key_id).await?; @@ -734,22 +947,9 @@ impl KmsBackend for VaultTransitKmsBackend { let deletion_date = if request.force_immediate.unwrap_or(false) { if key_metadata.key_state == KeyState::PendingDeletion { if !self.client.read_transit_key(&key_id).await?.deletion_allowed { - let mut update_builder = UpdateKeyConfigurationRequestBuilder::default(); - update_builder.deletion_allowed(true); - key::update( - &self.client.vault()?.client, - &self.client.config.mount_path, - &key_id, - Some(&mut update_builder), - ) - .await - .map_err(|e| { - KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")) - })?; + self.client.allow_transit_key_deletion(&key_id).await?; } - key::delete(&self.client.vault()?.client, &self.client.config.mount_path, &key_id) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")))?; + self.client.delete_transit_key(&key_id).await?; self.client.delete_key_metadata(&key_id).await?; None } else { @@ -852,20 +1052,9 @@ impl KmsBackend for VaultTransitKmsBackend { } if !self.client.read_transit_key(key_id).await?.deletion_allowed { - let mut update_builder = UpdateKeyConfigurationRequestBuilder::default(); - update_builder.deletion_allowed(true); - key::update( - &self.client.vault()?.client, - &self.client.config.mount_path, - key_id, - Some(&mut update_builder), - ) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")))?; + self.client.allow_transit_key_deletion(key_id).await?; } - key::delete(&self.client.vault()?.client, &self.client.config.mount_path, key_id) - .await - .map_err(|e| KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")))?; + self.client.delete_transit_key(key_id).await?; self.client.delete_key_metadata(key_id).await?; Ok(ExpiredKeyRemoval::Removed) } @@ -874,10 +1063,167 @@ impl KmsBackend for VaultTransitKmsBackend { #[cfg(test)] mod tests { use super::*; + use crate::backends::scripted_vault::{ScriptedResponse, ScriptedVault}; use crate::config::{ DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, VaultAuthMethod, VaultTransitConfig, }; use crate::types::KeyStatus; + use vaultrs::api::transit::responses::{ReadKeyData, ReadKeyResponse}; + + async fn scripted_client(responses: Vec<ScriptedResponse>) -> (ScriptedVault, VaultTransitKmsClient) { + let vault = ScriptedVault::serve(responses).await; + let config = VaultTransitConfig { + address: vault.address.clone(), + ..test_vault_transit_config() + }; + let kms_config = KmsConfig { + timeout: Duration::from_secs(5), + retry_attempts: 3, + ..KmsConfig::default() + }; + let client = VaultTransitKmsClient::new(config, &kms_config) + .await + .expect("scripted Vault Transit client"); + (vault, client) + } + + /// KV2 read payload for a persisted transit metadata record. + fn metadata_read_data(metadata: &TransitKeyMetadata) -> serde_json::Value { + let persisted: TransitKeyMetadataPersisted = metadata.clone().into(); + serde_json::json!({ + "data": serde_json::to_value(&persisted).expect("serialize transit metadata"), + "metadata": { + "created_time": "2026-01-01T00:00:00Z", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1, + }, + }) + } + + /// Transit read-key payload for an existing symmetric key. + fn transit_key_read_data(key_id: &str) -> serde_json::Value { + let response = ReadKeyResponse { + key_type: KeyType::Aes256Gcm96, + deletion_allowed: false, + derived: false, + exportable: false, + allow_plaintext_backup: false, + keys: ReadKeyData::Symmetric(HashMap::from([("1".to_string(), 1_700_000_000_u64)])), + min_decryption_version: 1, + min_encryption_version: 0, + name: key_id.to_string(), + supports_encryption: true, + supports_decryption: true, + supports_derivation: false, + supports_signing: false, + imported: Some(false), + }; + serde_json::to_value(&response).expect("serialize transit key read response") + } + + #[tokio::test] + async fn wired_transit_encrypt_retries_transient_status() { + let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(metadata_read_data(&metadata)), + ScriptedResponse::error(429, "throttled"), + ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })), + ]) + .await; + + let response = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect("encrypt must retry past a transient 429"); + assert_eq!(response.ciphertext, b"vault:v1:scripted".to_vec()); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "metadata read plus two encrypt attempts: {requests:?}"); + assert_eq!(requests[1], "POST /v1/transit/encrypt/wired-key"); + assert_eq!(requests[2], "POST /v1/transit/encrypt/wired-key"); + } + + #[tokio::test] + async fn wired_transit_rotate_is_never_retried() { + let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(metadata_read_data(&metadata)), + ScriptedResponse::error(503, "standby"), + ]) + .await; + + let error = client + .rotate_key("wired-key", None) + .await + .expect_err("the scripted 503 must fail the rotation"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "metadata read plus exactly one rotate attempt: {requests:?}"); + assert_eq!( + requests[1], "POST /v1/transit/keys/wired-key/rotate", + "a rotation must never be replayed: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_transit_create_read_confirms_identical_existing_key() { + // The stored key and metadata are exactly what this create would have + // produced, so a retried create whose first response was lost recovers + // by reading them back instead of failing. + let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::ok(metadata_read_data(&metadata)), + ]) + .await; + + let recovered = client + .create_key("wired-key", "AES_256", None) + .await + .expect("an identical enabled key must read-confirm as a recovered create"); + assert_eq!(recovered.status, KeyStatus::Active); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "read-confirm must be decided from reads alone: {requests:?}"); + assert!( + requests.iter().all(|line| line.starts_with("GET ")), + "a recovered create must not write anything: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_transit_create_still_fails_on_mismatched_existing_key() { + let mut metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + metadata.key_state = KeyState::Disabled; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::ok(metadata_read_data(&metadata)), + ]) + .await; + + let error = client + .create_key("wired-key", "AES_256", None) + .await + .expect_err("a non-enabled existing key must keep failing the create"); + assert!(matches!(error, KmsError::KeyAlreadyExists { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert!( + requests.iter().all(|line| line.starts_with("GET ")), + "the rejected create must not write anything: {requests:?}" + ); + } fn test_vault_transit_config() -> VaultTransitConfig { VaultTransitConfig { diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index e88369669..892ed5a35 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -73,9 +73,6 @@ pub mod deletion_worker; mod encryption; mod error; pub mod manager; -// The executor is wired into the Vault backends in a follow-up change; until -// then the module is only exercised by its own tests. -#[allow(dead_code)] mod policy; pub mod service; pub mod service_manager; diff --git a/crates/kms/src/policy.rs b/crates/kms/src/policy.rs index c2014b0e0..58411a51b 100644 --- a/crates/kms/src/policy.rs +++ b/crates/kms/src/policy.rs @@ -16,8 +16,8 @@ //! //! Vault-backed operations leave the process boundary, so every call needs a //! per-attempt timeout, a total operation deadline, and classification-driven -//! bounded retries. This module provides the engine only; the Vault backends -//! wire their call sites through [`execute`] in a follow-up change. +//! bounded retries. The Vault backends and the credential provider wire every +//! outbound `vaultrs` call through [`execute`]. //! //! Retry safety is driven by two orthogonal classifications: //! - [`OpClass`] states whether replaying the operation is safe at all. @@ -112,6 +112,32 @@ pub(crate) struct AttemptError { pub(crate) error: KmsError, } +impl AttemptError { + /// A failure that must never be retried, regardless of operation class. + pub(crate) fn fatal(error: KmsError) -> Self { + Self { + class: ErrorClass::Fatal, + error, + } + } + + /// Classify a `vaultrs` failure and map it onto a domain error. + /// + /// Classification reads the raw error before `map` consumes it, so call + /// sites keep their site-specific error mapping (404 to key-not-found and + /// so on) without losing the status code the retry decision needs. + pub(crate) fn from_vaultrs( + error: vaultrs::error::ClientError, + map: impl FnOnce(vaultrs::error::ClientError) -> KmsError, + ) -> Self { + let class = classify_vaultrs(&error); + Self { + class, + error: map(error), + } + } +} + /// Budgets applied by [`execute`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct RetryPolicy { From a2fe5d7d8829bf7592059c798399d2a2156007ef Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 09:37:07 +0800 Subject: [PATCH 33/52] feat(admin): add KMS key lifecycle endpoints and deletion reference gate (#5496) * feat(kms): add key lifecycle operations to the backend contract Add enable_key/disable_key/rotate_key to KmsBackend with conservative defaults returning the typed UnsupportedCapability error, mirroring remove_expired_key. KmsManager gains matching pass-through methods and drops cached key metadata after every successful state mutation so the next describe observes backend truth. The local backend overrides enable/disable, delegating to its state-machine-gated client methods; rotation stays rejected, matching its advertised capabilities. New dedicated policy actions kms:EnableKey and kms:DisableKey complete the KMS action taxonomy alongside the existing kms:RotateKey. * feat(admin): add KMS key enable/disable/rotate endpoints POST /v3/kms/keys/enable, /v3/kms/keys/disable and /v3/kms/keys/rotate, following the existing /v3/kms/keys handler conventions: key_id body with keyId query fallback, {success, message, key_id, key_metadata} responses, and 503 JSON while the KMS service is absent. Error mapping keeps InvalidOperation/ValidationError at 400 like the sibling handlers and surfaces UnsupportedCapability as 501 so a backend capability gap is never mistaken for a missing key. Existing /v3/kms/keys handlers are untouched apart from a visibility change on a private query helper. * feat(kms): gate scheduled key deletion on bucket encryption references Implement the DeletionReferenceChecker seam left by the deletion worker: before any material is destroyed, every bucket's SSE configuration is checked for a default KMS key reference and a hit blocks the removal. The gate fails closed - an unpublished object store, a failed bucket listing or an unreadable per-bucket encryption config all report a blocking reference - because destroying key material is irreversible while a blocked removal is simply retried on the next sweep. Registered during init_kms_system before the service can start, so every worker spawn observes it. Storage access goes through a new kms section of the root storage facade. --- crates/kms/src/backends/local.rs | 8 + crates/kms/src/backends/mod.rs | 43 ++ crates/kms/src/manager.rs | 76 +++ crates/policy/src/policy/action.rs | 6 + rustfs/src/admin/handlers/kms.rs | 3 +- .../src/admin/handlers/kms_key_lifecycle.rs | 456 ++++++++++++++++++ rustfs/src/admin/handlers/kms_keys.rs | 2 +- rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/route_policy.rs | 11 + rustfs/src/admin/route_registration_test.rs | 3 + rustfs/src/init.rs | 7 + rustfs/src/kms_deletion_gate.rs | 147 ++++++ rustfs/src/lib.rs | 1 + rustfs/src/storage_api.rs | 18 + 14 files changed, 780 insertions(+), 2 deletions(-) create mode 100644 rustfs/src/admin/handlers/kms_key_lifecycle.rs create mode 100644 rustfs/src/kms_deletion_gate.rs diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 3c48ff2c7..2d410f6d7 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -1533,6 +1533,14 @@ impl KmsBackend for LocalKmsBackend { }) } + async fn enable_key(&self, key_id: &str) -> Result<()> { + self.client.enable_key(key_id, None).await + } + + async fn disable_key(&self, key_id: &str) -> Result<()> { + self.client.disable_key(key_id, None).await + } + async fn health_check(&self) -> Result<bool> { self.client.health_check().await.map(|_| true) } diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index eecc236e8..a74ed9a38 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -256,6 +256,34 @@ pub trait KmsBackend: Send + Sync { /// Cancel key deletion async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse>; + /// Enable a disabled key so it can be used for cryptographic operations + /// again. + /// + /// Backends that advertise [`BackendCapabilities::enable_disable`] must + /// override this method; the default rejects the operation. + async fn enable_key(&self, _key_id: &str) -> Result<()> { + Err(KmsError::unsupported_capability("backend without enable/disable support", "enable_key")) + } + + /// Disable a key, rejecting new cryptographic use while existing data + /// remains decryptable. + /// + /// Backends that advertise [`BackendCapabilities::enable_disable`] must + /// override this method; the default rejects the operation. + async fn disable_key(&self, _key_id: &str) -> Result<()> { + Err(KmsError::unsupported_capability("backend without enable/disable support", "disable_key")) + } + + /// Rotate a key to a new version while prior versions remain available + /// for decryption. + /// + /// Only backends that advertise [`BackendCapabilities::rotate`] (that is, + /// backends with retained version history) may override this method; the + /// default rejects the operation. + async fn rotate_key(&self, _key_id: &str) -> Result<()> { + Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key")) + } + /// Health check async fn health_check(&self) -> Result<bool>; @@ -523,6 +551,21 @@ mod tests { assert!(!capabilities.physical_delete); } + #[tokio::test] + async fn default_lifecycle_operations_are_unsupported() { + for (operation, result) in [ + ("enable_key", MinimalBackend.enable_key("any-key").await), + ("disable_key", MinimalBackend.disable_key("any-key").await), + ("rotate_key", MinimalBackend.rotate_key("any-key").await), + ] { + let error = result.expect_err("backends must opt in to lifecycle operations by overriding them"); + assert!( + matches!(error, KmsError::UnsupportedCapability { .. }), + "expected UnsupportedCapability for {operation}, got {error:?}" + ); + } + } + #[tokio::test] async fn default_remove_expired_key_is_unsupported() { let error = MinimalBackend diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 8f670e1c7..7ae6bc055 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -155,6 +155,36 @@ impl KmsManager { Ok(response) } + /// Enable a disabled key + pub async fn enable_key(&self, key_id: &str) -> Result<()> { + self.backend.enable_key(key_id).await?; + self.invalidate_cached_metadata(key_id).await; + Ok(()) + } + + /// Disable a key; existing data remains decryptable + pub async fn disable_key(&self, key_id: &str) -> Result<()> { + self.backend.disable_key(key_id).await?; + self.invalidate_cached_metadata(key_id).await; + Ok(()) + } + + /// Rotate a key to a new version + pub async fn rotate_key(&self, key_id: &str) -> Result<()> { + self.backend.rotate_key(key_id).await?; + self.invalidate_cached_metadata(key_id).await; + Ok(()) + } + + /// Drop cached metadata after a state mutation so the next describe + /// observes backend truth instead of the pre-mutation snapshot. + async fn invalidate_cached_metadata(&self, key_id: &str) { + if self.enable_cache { + let mut cache = self.cache.write().await; + cache.remove_key_metadata(key_id).await; + } + } + /// Perform health check on the KMS backend pub async fn health_check(&self) -> Result<bool> { self.backend.health_check().await @@ -230,6 +260,52 @@ mod tests { assert!(health); } + #[tokio::test] + async fn lifecycle_round_trip_invalidates_cached_metadata() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + + let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend")); + let manager = KmsManager::new(backend, config); + + let key_id = manager + .create_key(CreateKeyRequest { + key_name: Some("lifecycle-round-trip".to_string()), + ..Default::default() + }) + .await + .expect("Failed to create key") + .key_id; + + let describe = |key_id: String| { + let manager = manager.clone(); + async move { + manager + .describe_key(DescribeKeyRequest { key_id }) + .await + .expect("describe should succeed") + .key_metadata + .key_state + } + }; + + // Warm the metadata cache, then flip states; each describe must see + // the post-mutation state, proving the cache entry was dropped. + assert_eq!(describe(key_id.clone()).await, KeyState::Enabled); + manager.disable_key(&key_id).await.expect("disable should succeed"); + assert_eq!(describe(key_id.clone()).await, KeyState::Disabled); + manager.enable_key(&key_id).await.expect("enable should succeed"); + assert_eq!(describe(key_id.clone()).await, KeyState::Enabled); + + // The local backend does not retain version history, so rotation is + // reported as a capability gap rather than a missing key. + let error = manager.rotate_key(&key_id).await.expect_err("local rotate must be rejected"); + assert!( + matches!(error, crate::error::KmsError::UnsupportedCapability { .. }), + "expected UnsupportedCapability, got {error:?}" + ); + } + #[tokio::test] async fn generate_data_key_does_not_reuse_context_bound_ciphertext() { let temp_dir = tempdir().expect("Failed to create temp dir"); diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index 45db55f75..9d0790dae 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -718,6 +718,10 @@ pub enum KmsAction { GenerateDataKeyAction, #[strum(serialize = "kms:DeleteKey")] DeleteKeyAction, + #[strum(serialize = "kms:EnableKey")] + EnableKeyAction, + #[strum(serialize = "kms:DisableKey")] + DisableKeyAction, #[strum(serialize = "kms:RotateKey")] RotateKeyAction, #[strum(serialize = "kms:ListKeys")] @@ -755,6 +759,8 @@ mod tests { ("kms:ClearCache", KmsAction::ClearCacheAction), ("kms:GenerateDataKey", KmsAction::GenerateDataKeyAction), ("kms:DeleteKey", KmsAction::DeleteKeyAction), + ("kms:EnableKey", KmsAction::EnableKeyAction), + ("kms:DisableKey", KmsAction::DisableKeyAction), ("kms:RotateKey", KmsAction::RotateKeyAction), ("kms:ListKeys", KmsAction::ListKeysAction), ("kms:DescribeKey", KmsAction::DescribeKeyAction), diff --git a/rustfs/src/admin/handlers/kms.rs b/rustfs/src/admin/handlers/kms.rs index fe0fde543..f89565bb3 100644 --- a/rustfs/src/admin/handlers/kms.rs +++ b/rustfs/src/admin/handlers/kms.rs @@ -14,12 +14,13 @@ //! KMS admin handlers for HTTP API -use super::{kms_dynamic, kms_keys, kms_management}; +use super::{kms_dynamic, kms_key_lifecycle, kms_keys, kms_management}; use crate::admin::router::{AdminOperation, S3Router}; pub fn register_kms_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> { kms_management::register_kms_management_route(r)?; kms_dynamic::register_kms_dynamic_route(r)?; kms_keys::register_kms_key_route(r)?; + kms_key_lifecycle::register_kms_key_lifecycle_route(r)?; Ok(()) } diff --git a/rustfs/src/admin/handlers/kms_key_lifecycle.rs b/rustfs/src/admin/handlers/kms_key_lifecycle.rs new file mode 100644 index 000000000..a7875b56b --- /dev/null +++ b/rustfs/src/admin/handlers/kms_key_lifecycle.rs @@ -0,0 +1,456 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! KMS key lifecycle admin API handlers: enable, disable and rotate. + +use super::kms_keys::extract_query_params; +use crate::admin::auth::validate_admin_request; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::admin::runtime_sources::current_kms_runtime_service_manager; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use hyper::{HeaderMap, Method, StatusCode}; +use matchit::Params; +use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; +use rustfs_kms::{ + KmsError, KmsManager, + types::{DescribeKeyRequest, KeyMetadata}, +}; +use rustfs_policy::policy::action::{Action, KmsAction}; +use s3s::header::CONTENT_TYPE; +use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::{error, info, warn}; + +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_KMS_KEYS: &str = "kms_key_lifecycle"; +const EVENT_ADMIN_KMS_KEYS_STATE: &str = "admin_kms_keys_state"; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct KmsKeyLifecycleRequest { + pub key_id: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct KmsKeyLifecycleResponse { + pub success: bool, + pub message: String, + pub key_id: String, + pub key_metadata: Option<KeyMetadata>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LifecycleOperation { + Enable, + Disable, + Rotate, +} + +impl LifecycleOperation { + fn action(self) -> &'static str { + match self { + Self::Enable => "enable_key", + Self::Disable => "disable_key", + Self::Rotate => "rotate_key", + } + } + + fn verb(self) -> &'static str { + match self { + Self::Enable => "enable", + Self::Disable => "disable", + Self::Rotate => "rotate", + } + } + + fn past_tense(self) -> &'static str { + match self { + Self::Enable => "enabled", + Self::Disable => "disabled", + Self::Rotate => "rotated", + } + } +} + +fn kms_enable_key_actions() -> Vec<Action> { + vec![Action::KmsAction(KmsAction::EnableKeyAction)] +} + +fn kms_disable_key_actions() -> Vec<Action> { + vec![Action::KmsAction(KmsAction::DisableKeyAction)] +} + +fn kms_rotate_key_actions() -> Vec<Action> { + vec![Action::KmsAction(KmsAction::RotateKeyAction)] +} + +pub fn register_kms_key_lifecycle_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> { + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/kms/keys/enable").as_str(), + AdminOperation(&EnableKmsKeyHandler {}), + )?; + + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/kms/keys/disable").as_str(), + AdminOperation(&DisableKmsKeyHandler {}), + )?; + + r.insert( + Method::POST, + format!("{}{}", ADMIN_PREFIX, "/v3/kms/keys/rotate").as_str(), + AdminOperation(&RotateKmsKeyHandler {}), + )?; + + Ok(()) +} + +/// Map a lifecycle failure to an HTTP status. +/// +/// `UnsupportedCapability` is a permanent gap of the configured backend, not a +/// missing resource: it must surface as 501, never 404. State-machine +/// rejections (`InvalidOperation`) keep the 400 mapping used by the other +/// `/v3/kms/keys` handlers. +fn lifecycle_error_status(error: &KmsError) -> StatusCode { + match error { + KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND, + KmsError::UnsupportedCapability { .. } => StatusCode::NOT_IMPLEMENTED, + KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST, + // Damaged or missing key material is an integrity fault of an existing + // key: it must surface as a server error, never as NOT_FOUND (the key + // exists) and never as a retryable backend outage. + KmsError::MaterialMissing { .. } + | KmsError::MaterialCorrupt { .. } + | KmsError::MaterialAuthenticationFailed { .. } + | KmsError::UnsupportedFormatVersion { .. } => StatusCode::INTERNAL_SERVER_ERROR, + _ => StatusCode::INTERNAL_SERVER_ERROR, + } +} + +/// Run one lifecycle operation against the manager and build the HTTP reply. +/// Split from the handlers so the status/response contract is unit-testable +/// without the admin auth stack. +async fn execute_lifecycle( + manager: &KmsManager, + key_id: &str, + operation: LifecycleOperation, +) -> (StatusCode, KmsKeyLifecycleResponse) { + let result = match operation { + LifecycleOperation::Enable => manager.enable_key(key_id).await, + LifecycleOperation::Disable => manager.disable_key(key_id).await, + LifecycleOperation::Rotate => manager.rotate_key(key_id).await, + }; + + match result { + Ok(()) => { + info!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = operation.action(), + key_id = %key_id, + state = "completed", + "admin kms keys state" + ); + // Best effort: the state change already succeeded, so a failing + // describe only drops the metadata echo from the response. + let key_metadata = match manager + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + { + Ok(response) => Some(response.key_metadata), + Err(error) => { + warn!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = operation.action(), + key_id = %key_id, + error = %error, + "key lifecycle change succeeded but describe failed" + ); + None + } + }; + ( + StatusCode::OK, + KmsKeyLifecycleResponse { + success: true, + message: format!("key {} successfully", operation.past_tense()), + key_id: key_id.to_string(), + key_metadata, + }, + ) + } + Err(error) => { + error!( + event = EVENT_ADMIN_KMS_KEYS_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS_KEYS, + action = operation.action(), + key_id = %key_id, + result = "failed", + error = %error, + "admin kms keys state" + ); + ( + lifecycle_error_status(&error), + KmsKeyLifecycleResponse { + success: false, + message: format!("Failed to {} key: {error}", operation.verb()), + key_id: key_id.to_string(), + key_metadata: None, + }, + ) + } + } +} + +fn json_response(status: StatusCode, response: &KmsKeyLifecycleResponse) -> S3Result<S3Response<(StatusCode, Body)>> { + let data = serde_json::to_vec(response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, "application/json".parse().expect("static content type should parse")); + Ok(S3Response::with_headers((status, Body::from(data)), headers)) +} + +fn unavailable_response(message: &str, key_id: String) -> S3Result<S3Response<(StatusCode, Body)>> { + json_response( + StatusCode::SERVICE_UNAVAILABLE, + &KmsKeyLifecycleResponse { + success: false, + message: message.to_string(), + key_id, + key_metadata: None, + }, + ) +} + +async fn handle_lifecycle_request( + mut req: S3Request<Body>, + actions: Vec<Action>, + operation: LifecycleOperation, +) -> S3Result<S3Response<(StatusCode, Body)>> { + let Some(cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + actions, + req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)), + ) + .await?; + + let body = req + .input + .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?; + + let request: KmsKeyLifecycleRequest = if body.is_empty() { + let query_params = extract_query_params(&req.uri); + let Some(key_id) = query_params.get("keyId") else { + return json_response( + StatusCode::BAD_REQUEST, + &KmsKeyLifecycleResponse { + success: false, + message: "missing required parameter: 'keyId'".to_string(), + key_id: "".to_string(), + key_metadata: None, + }, + ); + }; + KmsKeyLifecycleRequest { key_id: key_id.clone() } + } else { + serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))? + }; + + let Some(service_manager) = kms_service_manager_from_context() else { + return unavailable_response("kms service manager is not initialized", request.key_id); + }; + + let Some(manager) = service_manager.get_manager().await else { + return unavailable_response("kms service is not running", request.key_id); + }; + + let (status, response) = execute_lifecycle(&manager, &request.key_id, operation).await; + json_response(status, &response) +} + +fn kms_service_manager_from_context() -> Option<Arc<rustfs_kms::KmsServiceManager>> { + current_kms_runtime_service_manager() +} + +/// Enable a disabled KMS key +pub struct EnableKmsKeyHandler; + +#[async_trait::async_trait] +impl Operation for EnableKmsKeyHandler { + async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { + handle_lifecycle_request(req, kms_enable_key_actions(), LifecycleOperation::Enable).await + } +} + +/// Disable a KMS key; existing data remains decryptable +pub struct DisableKmsKeyHandler; + +#[async_trait::async_trait] +impl Operation for DisableKmsKeyHandler { + async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { + handle_lifecycle_request(req, kms_disable_key_actions(), LifecycleOperation::Disable).await + } +} + +/// Rotate a KMS key to a new version +pub struct RotateKmsKeyHandler; + +#[async_trait::async_trait] +impl Operation for RotateKmsKeyHandler { + async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { + handle_lifecycle_request(req, kms_rotate_key_actions(), LifecycleOperation::Rotate).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_kms::backends::local::LocalKmsBackend; + use rustfs_kms::config::KmsConfig; + use rustfs_kms::types::{CreateKeyRequest, KeyState}; + use rustfs_policy::policy::action::AdminAction; + + async fn local_manager(temp_dir: &tempfile::TempDir) -> KmsManager { + let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + let backend = Arc::new( + LocalKmsBackend::new(config.clone()) + .await + .expect("local backend should build"), + ); + KmsManager::new(backend, config) + } + + async fn create_key(manager: &KmsManager, key_name: &str) -> String { + manager + .create_key(CreateKeyRequest { + key_name: Some(key_name.to_string()), + ..Default::default() + }) + .await + .expect("key should be created") + .key_id + } + + fn key_state(response: &KmsKeyLifecycleResponse) -> KeyState { + response + .key_metadata + .as_ref() + .expect("lifecycle response should echo key metadata") + .key_state + .clone() + } + + #[tokio::test] + async fn enable_disable_enable_round_trip() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let manager = local_manager(&temp_dir).await; + let key_id = create_key(&manager, "lifecycle-round-trip").await; + + let (status, response) = execute_lifecycle(&manager, &key_id, LifecycleOperation::Disable).await; + assert_eq!(status, StatusCode::OK); + assert!(response.success); + assert_eq!(key_state(&response), KeyState::Disabled); + + let (status, response) = execute_lifecycle(&manager, &key_id, LifecycleOperation::Enable).await; + assert_eq!(status, StatusCode::OK); + assert!(response.success); + assert_eq!(key_state(&response), KeyState::Enabled); + + // Enabling an already enabled key stays idempotent. + let (status, response) = execute_lifecycle(&manager, &key_id, LifecycleOperation::Enable).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(key_state(&response), KeyState::Enabled); + } + + #[tokio::test] + async fn local_rotate_is_unsupported_capability_not_missing_key() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let manager = local_manager(&temp_dir).await; + let key_id = create_key(&manager, "rotate-unsupported").await; + + let (status, response) = execute_lifecycle(&manager, &key_id, LifecycleOperation::Rotate).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + assert_ne!(status, StatusCode::NOT_FOUND); + assert!(!response.success); + assert!( + response.message.contains("not supported"), + "message should name the capability gap: {}", + response.message + ); + + // A disabled key must not change the picture: rotation stays rejected. + let (status, _) = execute_lifecycle(&manager, &key_id, LifecycleOperation::Disable).await; + assert_eq!(status, StatusCode::OK); + let (status, response) = execute_lifecycle(&manager, &key_id, LifecycleOperation::Rotate).await; + assert_ne!(status, StatusCode::OK); + assert!(!response.success); + } + + #[tokio::test] + async fn lifecycle_on_missing_key_maps_to_not_found() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let manager = local_manager(&temp_dir).await; + + let (status, response) = execute_lifecycle(&manager, "no-such-key", LifecycleOperation::Enable).await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert!(!response.success); + } + + #[test] + fn lifecycle_error_status_contract() { + let unsupported = KmsError::unsupported_capability("local", "rotate_key"); + assert_eq!(lifecycle_error_status(&unsupported), StatusCode::NOT_IMPLEMENTED); + assert_ne!(lifecycle_error_status(&unsupported), StatusCode::NOT_FOUND); + + assert_eq!(lifecycle_error_status(&KmsError::invalid_key_state("disabled")), StatusCode::BAD_REQUEST); + assert_eq!(lifecycle_error_status(&KmsError::key_not_found("gone")), StatusCode::NOT_FOUND); + } + + #[test] + fn lifecycle_auth_actions_use_dedicated_kms_actions() { + assert_eq!(kms_enable_key_actions(), vec![Action::KmsAction(KmsAction::EnableKeyAction)]); + assert_eq!(kms_disable_key_actions(), vec![Action::KmsAction(KmsAction::DisableKeyAction)]); + assert_eq!(kms_rotate_key_actions(), vec![Action::KmsAction(KmsAction::RotateKeyAction)]); + + for actions in [kms_enable_key_actions(), kms_disable_key_actions(), kms_rotate_key_actions()] { + assert!(!actions.contains(&Action::AdminAction(AdminAction::ServerInfoAdminAction))); + } + } + + #[test] + fn lifecycle_request_rejects_unknown_fields() { + let error = serde_json::from_str::<KmsKeyLifecycleRequest>(r#"{"key_id":"key","unexpected_field":true}"#) + .expect_err("unknown lifecycle request field should fail"); + assert!(error.to_string().contains("unknown field")); + } +} diff --git a/rustfs/src/admin/handlers/kms_keys.rs b/rustfs/src/admin/handlers/kms_keys.rs index decc97247..7e39531ac 100644 --- a/rustfs/src/admin/handlers/kms_keys.rs +++ b/rustfs/src/admin/handlers/kms_keys.rs @@ -94,7 +94,7 @@ pub struct GenerateDataKeyApiResponse { pub ciphertext_blob: String, // Base64 encoded } -fn extract_query_params(uri: &hyper::Uri) -> HashMap<String, String> { +pub(super) fn extract_query_params(uri: &hyper::Uri) -> HashMap<String, String> { let mut params = HashMap::new(); if let Some(query) = uri.query() { query.split('&').for_each(|pair| { diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index bb4273274..17347d5e3 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -33,6 +33,7 @@ pub mod inspect_archive; pub mod is_admin; pub mod kms; pub mod kms_dynamic; +pub mod kms_key_lifecycle; pub mod kms_keys; pub mod kms_management; pub mod metrics; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 77d9fc855..a42f63185 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -58,8 +58,11 @@ const KMS_CLEAR_CACHE: AdminActionRef = AdminActionRef::new("kms:ClearCache"); const KMS_CONFIGURE: AdminActionRef = AdminActionRef::new("kms:Configure"); const KMS_DELETE_KEY: AdminActionRef = AdminActionRef::new("kms:DeleteKey"); const KMS_DESCRIBE_KEY: AdminActionRef = AdminActionRef::new("kms:DescribeKey"); +const KMS_DISABLE_KEY: AdminActionRef = AdminActionRef::new("kms:DisableKey"); +const KMS_ENABLE_KEY: AdminActionRef = AdminActionRef::new("kms:EnableKey"); const KMS_GENERATE_DATA_KEY: AdminActionRef = AdminActionRef::new("kms:GenerateDataKey"); const KMS_LIST_KEYS: AdminActionRef = AdminActionRef::new("kms:ListKeys"); +const KMS_ROTATE_KEY: AdminActionRef = AdminActionRef::new("kms:RotateKey"); const KMS_SERVICE_CONTROL: AdminActionRef = AdminActionRef::new("kms:ServiceControl"); const LIST_GROUPS: AdminActionRef = AdminActionRef::new("ListGroupsAdminAction"); const LIST_TEMPORARY_ACCOUNTS: AdminActionRef = AdminActionRef::new("ListTemporaryAccountsAdminAction"); @@ -777,6 +780,14 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ KMS_DESCRIBE_KEY, RouteRiskLevel::Sensitive, ), + admin(HttpMethod::Post, "/rustfs/admin/v3/kms/keys/enable", KMS_ENABLE_KEY, RouteRiskLevel::High), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/kms/keys/disable", + KMS_DISABLE_KEY, + RouteRiskLevel::High, + ), + admin(HttpMethod::Post, "/rustfs/admin/v3/kms/keys/rotate", KMS_ROTATE_KEY, RouteRiskLevel::High), public( HttpMethod::Get, "/rustfs/admin/v3/oidc/providers", diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index cd61ba7e8..65a053b28 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -340,6 +340,9 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> { admin_route(Method::POST, "/v3/kms/keys/cancel-deletion"), admin_route(Method::GET, "/v3/kms/keys"), admin_route_sample(Method::GET, "/v3/kms/keys/{key_id}", "/v3/kms/keys/test-key"), + admin_route(Method::POST, "/v3/kms/keys/enable"), + admin_route(Method::POST, "/v3/kms/keys/disable"), + admin_route(Method::POST, "/v3/kms/keys/rotate"), admin_route(Method::GET, "/v3/oidc/providers"), admin_route_sample(Method::GET, "/v3/oidc/authorize/{provider_id}", "/v3/oidc/authorize/default"), admin_route_sample(Method::GET, "/v3/oidc/callback/{provider_id}", "/v3/oidc/callback/default"), diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 8ac27e2e6..23069e953 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -468,6 +468,13 @@ pub async fn init_kms_system(config: &config::Config) -> std::io::Result<()> { // Initialize global KMS service manager (starts in NotConfigured state) let service_manager = startup_runtime_sources::init_kms_service_manager(); + // A key referenced by any bucket's encryption configuration must never be + // deleted. Register the gate before the service can start so every + // deletion-worker spawn observes it; the gate fails closed while the + // object store is not ready. + service_manager + .set_deletion_reference_checker(std::sync::Arc::new(crate::kms_deletion_gate::BucketEncryptionReferenceChecker)); + // If KMS is enabled in configuration, configure and start the service if config.kms_enable { info!( diff --git a/rustfs/src/kms_deletion_gate.rs b/rustfs/src/kms_deletion_gate.rs new file mode 100644 index 000000000..9cff095ad --- /dev/null +++ b/rustfs/src/kms_deletion_gate.rs @@ -0,0 +1,147 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Reference gate consulted before a scheduled KMS key deletion destroys +//! material: a key that any bucket's encryption configuration still points at +//! must not be removed. Object-level references (existing envelopes) are out +//! of scope here; those objects stay decryptable only until the key is gone, +//! which is why the pending-deletion window exists. + +use crate::runtime_sources::current_object_store_handle; +use crate::storage_api::kms::contract::bucket::{BucketOperations, BucketOptions}; +use crate::storage_api::kms::{ECStore, StorageError, get_bucket_sse_config}; +use async_trait::async_trait; +use rustfs_kms::DeletionReferenceChecker; +use s3s::dto::ServerSideEncryptionConfiguration; +use std::sync::Arc; +use tracing::warn; + +/// Reference reported when bucket configuration cannot be inspected at all. +const BUCKET_CONFIG_UNAVAILABLE: &str = "bucket-encryption-config:unavailable"; + +/// Blocks deletion of keys referenced by any bucket's SSE configuration +/// (default bucket KMS key). Registered on the KMS service manager at startup +/// and consulted by the background deletion worker. +pub(crate) struct BucketEncryptionReferenceChecker; + +#[async_trait] +impl DeletionReferenceChecker for BucketEncryptionReferenceChecker { + async fn references(&self, key_id: &str) -> Vec<String> { + collect_references(key_id, current_object_store_handle()).await + } +} + +async fn collect_references(key_id: &str, store: Option<Arc<ECStore>>) -> Vec<String> { + // Fail closed: destroying key material is irreversible while the deletion + // worker retries every sweep, so an uninspectable configuration must block + // the removal rather than wave it through. This also covers the worker + // racing server startup, before the object store is published. + let Some(store) = store else { + warn!(key_id, "KMS deletion reference check: object store not ready; blocking removal"); + return vec![BUCKET_CONFIG_UNAVAILABLE.to_string()]; + }; + let buckets = match store.list_bucket(&BucketOptions::default()).await { + Ok(buckets) => buckets, + Err(error) => { + warn!(key_id, %error, "KMS deletion reference check: listing buckets failed; blocking removal"); + return vec![BUCKET_CONFIG_UNAVAILABLE.to_string()]; + } + }; + + let mut references = Vec::new(); + for bucket in &buckets { + let lookup = get_bucket_sse_config(&bucket.name).await; + references.extend(bucket_reference(&bucket.name, lookup, key_id)); + } + references +} + +/// `Some(reference)` when the bucket's encryption configuration references +/// `key_id` or cannot be read (fail closed); `None` when the bucket provably +/// does not reference the key. +fn bucket_reference( + bucket: &str, + lookup: Result<ServerSideEncryptionConfiguration, StorageError>, + key_id: &str, +) -> Option<String> { + match lookup { + Ok(config) if sse_config_references_key(&config, key_id) => Some(format!("bucket:{bucket}")), + Ok(_) => None, + Err(StorageError::ConfigNotFound) => None, + Err(error) => { + warn!( + bucket, + key_id, + %error, + "KMS deletion reference check: unreadable bucket encryption config; blocking removal" + ); + Some(format!("bucket:{bucket}:encryption-config-unreadable")) + } + } +} + +fn sse_config_references_key(config: &ServerSideEncryptionConfiguration, key_id: &str) -> bool { + config.rules.iter().any(|rule| { + rule.apply_server_side_encryption_by_default + .as_ref() + .and_then(|sse| sse.kms_master_key_id.as_deref()) + .is_some_and(|configured| configured == key_id) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use s3s::dto::{ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionRule}; + + fn sse_kms_config(key_id: Option<&str>) -> ServerSideEncryptionConfiguration { + ServerSideEncryptionConfiguration { + rules: vec![ServerSideEncryptionRule { + apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault { + sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + kms_master_key_id: key_id.map(str::to_string), + }), + bucket_key_enabled: None, + }], + } + } + + #[test] + fn referencing_bucket_blocks_deletion() { + assert_eq!( + bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1"), + Some("bucket:sse-bucket".to_string()) + ); + } + + #[test] + fn non_referencing_buckets_allow_deletion() { + // Different key, no configured key, and no SSE configuration at all. + assert_eq!(bucket_reference("other-key", Ok(sse_kms_config(Some("kms-key-2"))), "kms-key-1"), None); + assert_eq!(bucket_reference("no-key", Ok(sse_kms_config(None)), "kms-key-1"), None); + assert_eq!(bucket_reference("plain", Err(StorageError::ConfigNotFound), "kms-key-1"), None); + } + + #[test] + fn unreadable_config_blocks_deletion() { + let reference = bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1"); + assert_eq!(reference, Some("bucket:broken:encryption-config-unreadable".to_string())); + } + + #[tokio::test] + async fn missing_object_store_blocks_deletion() { + let references = collect_references("kms-key-1", None).await; + assert_eq!(references, vec![BUCKET_CONFIG_UNAVAILABLE.to_string()]); + } +} diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index 3a9034113..ec3880bf6 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -82,6 +82,7 @@ pub mod diagnose; pub mod embedded; pub mod error; pub mod init; +pub(crate) mod kms_deletion_gate; pub mod license; pub mod memory_observability; pub mod profiling; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 3953874c5..7218ce244 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -72,6 +72,24 @@ pub(crate) mod error { pub(crate) use crate::storage::storage_api::{QuotaError, StorageError}; } +pub(crate) mod kms { + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions}; + } + } + + pub(crate) use crate::storage::storage_api::{ECStore, StorageError}; + + /// Bucket SSE configuration for the KMS deletion reference gate; + /// `Err(StorageError::ConfigNotFound)` when the bucket has none. + pub(crate) async fn get_bucket_sse_config(bucket: &str) -> Result<s3s::dto::ServerSideEncryptionConfiguration, StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_sse_config(bucket) + .await + .map(|(config, _)| config) + } +} + pub(crate) mod protocols { pub(crate) mod client { pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo; From e06c9c02c63186acf1a5bb2283eb2ab7bd905169 Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Fri, 31 Jul 2026 09:59:13 +0800 Subject: [PATCH 34/52] feat: add hotpath primitive profiling coverage (#5492) * chore(deps): refresh google cloud dependencies Co-Authored-By: heihutu <heihutu@gmail.com> * feat: add hotpath primitive coverage Co-Authored-By: heihutu <heihutu@gmail.com> * feat: extend hotpath profiling features Co-Authored-By: heihutu <heihutu@gmail.com> * fix: gate OPA hotpath client wrapping Co-Authored-By: heihutu <heihutu@gmail.com> * fix: avoid request-scoped hotpath primitive wrappers Preserve the original bounded channel and stream semantics in EC and RIO request paths while keeping hotpath CPU profiling as a separate opt-in feature that implies base hotpath instrumentation. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> --- Cargo.lock | 77 +++++++++++++++++++-------------- Cargo.toml | 4 +- crates/ecstore/Cargo.toml | 6 +++ crates/filemeta/Cargo.toml | 1 + crates/policy/Cargo.toml | 7 +++ crates/policy/src/policy/opa.rs | 10 ++++- crates/rio/Cargo.toml | 1 + rustfs/Cargo.toml | 10 +++++ 8 files changed, 81 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5bd0f4e1..97584df3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1528,7 +1528,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1547,7 +1547,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1598,7 +1598,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.20.11", "ident_case", "prettyplease", "proc-macro2", @@ -1870,7 +1870,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -2328,7 +2328,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -2353,11 +2353,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "typenum", ] @@ -3554,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3813,7 +3813,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff 0.13.1", - "generic-array 0.14.9", + "generic-array 0.14.7", "group 0.13.0", "hkdf 0.12.4", "pem-rfc7468 0.7.0", @@ -4258,9 +4258,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4273,7 +4273,7 @@ version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rustversion", "typenum", ] @@ -4372,9 +4372,9 @@ dependencies = [ [[package]] name = "google-cloud-auth" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3494870d06f3cbbb3561ada6f234982549e3a2fb31e719ef258e6eadb9ae09a" +checksum = "f54aab44c16b8463ae11b165a87c3d484780231f157bb1ed65843d591beb5abd" dependencies = [ "async-trait", "aws-lc-rs", @@ -4401,9 +4401,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3103a4a9013f1aed573ca56e19a9680b0211643a99ea85caf524b397d6be8be3" +checksum = "b9a46dd0fd026bbc4a5d84e6ab0c941cee6e3b057976a0bb107fdb5238ce598f" dependencies = [ "bytes", "futures", @@ -4420,9 +4420,9 @@ dependencies = [ [[package]] name = "google-cloud-gax-internal" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0df265fba091ed7e00ecd0755009423310163f8b52820f007b6b4d97f4c6617" +checksum = "fb04c54317ace06d489213f761797240b3046142a9b7ce6b9a82a9d134e193d1" dependencies = [ "bytes", "futures", @@ -4524,9 +4524,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b1d78c88db5c2530b12461e373a7d0d3a6caa3f6c1fc14e5d824cf2aeb307" +checksum = "9227f65175fa91a6e41f246797917697efdadfe09dd8ea84ad8b737a71efbd28" dependencies = [ "async-trait", "base64 0.22.1", @@ -4577,9 +4577,9 @@ dependencies = [ [[package]] name = "google-cloud-wkt" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46df1fcc3ab69164af3f4199ed21f45b5dbc56d9f03211eb4fa20116d442364b" +checksum = "7fccf98cfd5481a5f5a285181ab0c62123d7d47cd2bb7299448440649349e4e7" dependencies = [ "base64 0.22.1", "bytes", @@ -4920,6 +4920,7 @@ dependencies = [ "async-trait", "cfg-if", "crossbeam-channel", + "flate2", "futures-channel", "futures-util", "hdrhistogram", @@ -4927,6 +4928,7 @@ dependencies = [ "hotpath-meta", "http 1.5.0", "libc", + "object 0.36.7", "parking_lot", "pin-project-lite", "prettytable-rs", @@ -4934,6 +4936,7 @@ dependencies = [ "regex", "reqwest", "reqwest-middleware", + "rustc-demangle", "serde", "serde_json", "tiny_http", @@ -5047,9 +5050,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "ctutils", "subtle", @@ -5297,7 +5300,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding 0.3.3", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -6807,6 +6810,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "object" version = "0.37.3" @@ -7844,7 +7856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -7864,7 +7876,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "petgraph 0.8.3", @@ -7885,7 +7897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", @@ -7898,7 +7910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", @@ -9716,6 +9728,7 @@ dependencies = [ "base64-simd", "chrono", "futures", + "hotpath", "ipnetwork", "jsonwebtoken 11.0.0", "moka", @@ -10550,7 +10563,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.10", - "generic-array 0.14.9", + "generic-array 0.14.7", "pkcs8 0.10.2", "subtle", "zeroize", @@ -11540,7 +11553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 1b5bbe8b9..112df1c40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -250,8 +250,8 @@ enumset = "1.1.14" faster-hex = "0.10.0" flate2 = "1.1.9" glob = "0.3.4" -google-cloud-storage = "1.16.0" -google-cloud-auth = "1.14.0" +google-cloud-storage = "1.17.0" +google-cloud-auth = "1.15.0" hashbrown = { version = "0.17.1" } hex = "0.4.3" hex-simd = "0.8.0" diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 3b6ce2ce8..726c5f2a2 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -48,6 +48,12 @@ hotpath-alloc = [ "rustfs-filemeta/hotpath-alloc", "rustfs-rio/hotpath-alloc", ] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", + "rustfs-rio/hotpath-cpu", +] # Exposes shared lifecycle/tier test utilities (MockWarmBackend, fault # injection, xl.meta transition assertions) via `api::tier::test_util`. # Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6). diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index 3fdea23e0..f1d2bbad2 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -29,6 +29,7 @@ documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/" default = [] hotpath = ["hotpath/hotpath", "hotpath/tokio"] hotpath-alloc = ["hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] hotpath.workspace = true diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml index 6d10ac82d..b85ded8b5 100644 --- a/crates/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -28,6 +28,12 @@ documentation = "https://docs.rs/rustfs-policy/latest/rustfs_policy/" [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/reqwest-0-13"] +hotpath-alloc = ["hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] rustfs-credentials = { workspace = true } rustfs-config = { workspace = true, features = ["opa"] } @@ -49,6 +55,7 @@ moka = { workspace = true, features = ["future"] } async-trait.workspace = true futures.workspace = true pollster.workspace = true +hotpath.workspace = true [dev-dependencies] pollster.workspace = true diff --git a/crates/policy/src/policy/opa.rs b/crates/policy/src/policy/opa.rs index be2e033a8..5ff6e3ea0 100644 --- a/crates/policy/src/policy/opa.rs +++ b/crates/policy/src/policy/opa.rs @@ -32,10 +32,15 @@ impl Args { #[derive(Debug, Clone)] pub struct AuthZPlugin { - client: reqwest::Client, + client: OpaHttpClient, args: Args, } +#[cfg(feature = "hotpath")] +type OpaHttpClient = hotpath::wrap::reqwest::Client; +#[cfg(not(feature = "hotpath"))] +type OpaHttpClient = reqwest::Client; + #[derive(Debug, thiserror::Error)] pub enum OpaConfigError { #[error("Missing required env var: {0}")] @@ -141,6 +146,9 @@ impl AuthZPlugin { reqwest::Client::new() }); + #[cfg(feature = "hotpath")] + let client = hotpath::http!(client, label = "Policy::OPA"); + Self { client, args: config } } diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index c0b7b52d6..4d0f3916e 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -32,6 +32,7 @@ workspace = true default = [] hotpath = ["hotpath/hotpath", "hotpath/tokio", "hotpath/futures", "hotpath/reqwest-0-13"] hotpath-alloc = ["hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] hotpath.workspace = true diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index cbef7e4f0..53db5ca19 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -62,14 +62,24 @@ hotpath = [ "hotpath/reqwest-0-13", "rustfs-ecstore/hotpath", "rustfs-filemeta/hotpath", + "rustfs-policy/hotpath", "rustfs-rio/hotpath", ] hotpath-alloc = [ "hotpath/hotpath-alloc", "rustfs-ecstore/hotpath-alloc", "rustfs-filemeta/hotpath-alloc", + "rustfs-policy/hotpath-alloc", "rustfs-rio/hotpath-alloc", ] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-rio/hotpath-cpu", +] [lints] workspace = true From f718e72e247a3867b0b4ab55199872b828305d00 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 10:27:03 +0800 Subject: [PATCH 35/52] perf(ecstore): deduplicate concurrent lazy bucket-metadata loads (#5498) --- crates/ecstore/src/bucket/metadata_sys.rs | 99 +++++++++++++++++++++-- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 6dfce351e..41243137a 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -579,8 +579,26 @@ pub struct BucketMetadataSys { /// Serializes metadata-map commits and their derived cache updates for one /// bucket. Namespace locks, when present, are acquired before this lock. metadata_publish_locks: Arc<MetadataPublishLockRegistry>, + /// Deduplicates concurrent lazy loads of one bucket's metadata, so N + /// simultaneous cache misses issue a single disk read instead of N. + /// + /// This is the `singleflight` that upstream applies to its own lazy + /// `GetConfig`. Without it the namespace *read* lock the load holds is no + /// help: read locks are shared, so it excludes concurrent config writers + /// but not concurrent readers, and every caller still pays a full + /// erasure-set metadata fanout. A separate registry from + /// `metadata_publish_locks`, reusing the same per-bucket lock machinery. + /// + /// Lock order: this lock, then the namespace lock, then the publish lock, + /// then the metadata map. It is only ever taken as the first of those, so + /// it cannot invert against a path that already holds one of the others. + lazy_load_locks: Arc<MetadataPublishLockRegistry>, #[cfg(test)] lazy_load_lock_probe: std::sync::atomic::AtomicBool, + /// Counts disk loads taken by the lazy `get_config` path, so a test can + /// prove concurrent misses collapse into one. + #[cfg(test)] + lazy_disk_loads: std::sync::atomic::AtomicUsize, /// Buckets recently observed to have no persisted metadata. Serving the /// fabricated default from here (instead of re-reading disk) keeps the /// per-request cost of repeated lookups for such names bounded — without @@ -599,8 +617,13 @@ impl BucketMetadataSys { metadata_publish_locks: Arc::new(MetadataPublishLockRegistry { locks: StdMutex::new(HashMap::new()), }), + lazy_load_locks: Arc::new(MetadataPublishLockRegistry { + locks: StdMutex::new(HashMap::new()), + }), #[cfg(test)] lazy_load_lock_probe: std::sync::atomic::AtomicBool::new(false), + #[cfg(test)] + lazy_disk_loads: std::sync::atomic::AtomicUsize::new(0), absent_metadata: moka::future::Cache::builder() .max_capacity(ABSENT_BUCKET_METADATA_MAX_ENTRIES) .time_to_live(ABSENT_BUCKET_METADATA_TTL) @@ -615,16 +638,22 @@ impl BucketMetadataSys { } fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> { - let mut locks = self - .metadata_publish_locks - .locks - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + Self::bucket_lock_in(&self.metadata_publish_locks, bucket) + } + + /// Per-bucket gate for the lazy `get_config` disk load. See + /// [`Self::lazy_load_locks`]. + fn lazy_load_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> { + Self::bucket_lock_in(&self.lazy_load_locks, bucket) + } + + fn bucket_lock_in(registry: &Arc<MetadataPublishLockRegistry>, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> { + let mut locks = registry.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); locks.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| { let lock = Arc::new_cyclic(|lock| { Mutex::new(MetadataPublishLockState { bucket: bucket.to_string(), - registry: Arc::downgrade(&self.metadata_publish_locks), + registry: Arc::downgrade(registry), lock: lock.clone(), }) }); @@ -1079,6 +1108,27 @@ impl BucketMetadataSys { return Ok((Arc::new(bm), true)); } + // Collapse concurrent misses for this bucket into one disk load. + // Taken before the namespace lock — see `lazy_load_locks` for the + // ordering rule. + let load_lock = self.lazy_load_lock(bucket); + let _load_guard = load_lock.lock_owned().await; + + // Re-check both caches: whoever held the gate before us may have + // already answered this exact question, and repeating the fanout + // is the whole cost this gate exists to avoid. + if let Some(bm) = self.metadata_map.read().await.get(bucket).cloned() { + return Ok((bm, true)); + } + if self.absent_metadata.get(bucket).await.is_some() { + let mut bm = BucketMetadata::new(bucket); + bm.default_timestamps(); + return Ok((Arc::new(bm), true)); + } + + #[cfg(test)] + self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let lock = self.api.new_ns_lock(bucket, bucket).await?; let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?; #[cfg(test)] @@ -1431,6 +1481,43 @@ mod tests { use serial_test::serial; use tokio::time::timeout; + /// Concurrent cache misses for one bucket must collapse into a single disk + /// load. + /// + /// The namespace read lock the lazy path already holds does not provide + /// this: read locks are shared, so it excludes concurrent config writers + /// but not concurrent readers. Without the dedup gate every caller pays its + /// own namespace-lock acquisition plus a full erasure-set metadata fanout — + /// and the paths that reach `get_config` are per-request, so the multiplier + /// is request concurrency. + #[tokio::test] + async fn concurrent_lazy_loads_of_one_bucket_issue_a_single_disk_read() { + use std::sync::atomic::Ordering; + + let (_dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = Arc::new(BucketMetadataSys::new(ecstore)); + + // A name with no persisted metadata: every caller misses the map, and + // the absent-cache entry does not exist until the first load records it. + let bucket = "singleflight-bucket"; + let waiters = 8; + + let results = futures::future::join_all((0..waiters).map(|_| { + let sys = Arc::clone(&sys); + async move { sys.get_config(bucket).await.map(|(bm, _)| bm.name.clone()) } + })) + .await; + + for result in results { + assert_eq!(result.expect("every caller must get an answer"), bucket); + } + assert_eq!( + sys.lazy_disk_loads.load(Ordering::Relaxed), + 1, + "concurrent misses for one bucket must share a single disk load" + ); + } + /// Pins the fail-closed caching contract of the lazy `get_config` path /// and the refresh no-replace rule: fabricated defaults are returned but /// never served by the map-only `get()`, persisted metadata is cached on From dd11145a260225ba77e949275fabaf61ef0a4592 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 10:56:44 +0800 Subject: [PATCH 36/52] feat(kms): record operation metrics in the retry policy engine (#5500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): record operation metrics in the retry policy engine Instrument policy::execute — the single choke point every outbound Vault call and credential exchange already flows through — so no call site needs its own instrumentation: - rustfs_kms_backend_operations_total (counter): operation, op_class, outcome (success / fatal / budget_exhausted / deadline_exceeded / cancelled) - rustfs_kms_backend_attempt_failures_total (counter): operation, error_class (retryable_conn / retryable_status / fatal / attempt_timeout) - rustfs_kms_backend_operation_duration_seconds (histogram): wall-clock duration including retries and backoff - rustfs_kms_backend_operation_attempts (histogram): attempts used Metric labels carry only static enum values (operation names, classes, outcomes) — never key identifiers, key material, ciphertext, or tokens. Emission goes through the process-global metrics facade recorder, the same pattern the rest of the workspace uses, so no new wiring is needed in rustfs/src. Tests drive a paused-clock runtime under a thread-local debugging recorder, so counts, attempts, and even the recorded (virtual-clock) durations are asserted deterministically with zero real sleeps. Refs rustfs/backlog#1569 (part of rustfs/backlog#1562) * test(kms): add Vault fault-injection matrix Offline cases inject transport faults locally and are fully deterministic: a refused connection is retried up to the configured budget, and a stalled connection is cut off by the per-attempt timeout instead of hanging. Ignored cases run against a real dev Vault (RUSTFS_KMS_VAULT_ADDR) and pin the fail-closed auth behavior: an invalid token and a missing key each resolve in exactly one attempt. Every case asserts through the policy metrics recorded by a thread-local debugging recorder, which doubles as the request-count assertion even against a real server. Throttling and recoverable 5xx responses cannot be forced on a stock dev Vault; those paths stay pinned by the scripted-Vault wiring tests and the engine tests. Refs rustfs/backlog#1569 (part of rustfs/backlog#1562) --- Cargo.lock | 2 + crates/kms/Cargo.toml | 4 + crates/kms/src/policy.rs | 530 +++++++++++++++++++++- crates/kms/tests/vault_fault_injection.rs | 255 +++++++++++ 4 files changed, 769 insertions(+), 22 deletions(-) create mode 100644 crates/kms/tests/vault_fault_injection.rs diff --git a/Cargo.lock b/Cargo.lock index 97584df3a..0d52e411b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9502,6 +9502,8 @@ dependencies = [ "insta", "jiff", "md-5 0.11.0", + "metrics", + "metrics-util", "moka", "rand 0.10.2", "reqwest", diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 3c46a63a3..33fa8e3bd 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -37,6 +37,8 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } tracing = { workspace = true } thiserror = { workspace = true } +# Operation metrics emitted by the retry policy engine (crate::policy). +metrics = { workspace = true } # Cryptography aes-gcm = { workspace = true, features = ["rand_core"] } @@ -72,6 +74,8 @@ tokio-util = { workspace = true } [dev-dependencies] anyhow = { workspace = true } +# Debugging recorder for asserting emitted metrics in tests. +metrics-util = { version = "0.20", features = ["debugging"] } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } diff --git a/crates/kms/src/policy.rs b/crates/kms/src/policy.rs index 58411a51b..bc7fcc664 100644 --- a/crates/kms/src/policy.rs +++ b/crates/kms/src/policy.rs @@ -27,6 +27,12 @@ //! retried automatically: a response lost after the server applied the write //! would otherwise be replayed into duplicate side effects (extra key versions, //! repeated deletes). +//! +//! Every execution also records operation metrics (attempt failures by retry +//! class, terminal outcome, attempts used, wall-clock duration) through the +//! process-global `metrics` recorder. Metric labels carry only static enum +//! values — operation names, classes, outcomes — never key identifiers, key +//! material, ciphertext, or tokens. use std::future::Future; use std::time::Duration; @@ -200,6 +206,130 @@ fn equal_jitter(rng: &mut impl RngExt, cap: Duration) -> Duration { half + Duration::from_nanos(rng.random_range(0..=spread)) } +// --------------------------------------------------------------------------- +// Metrics +// +// Every execution is recorded here, at the single choke point all backend +// calls flow through, so instrumenting a new call site costs nothing beyond +// naming its operation. Label values are exclusively static enum strings +// (operation names, classes, outcomes) — key identifiers, key material, +// ciphertext, and tokens must never reach a metric label. +// --------------------------------------------------------------------------- + +/// Counter: operations executed, by `operation`, `op_class`, and `outcome`. +const METRIC_OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; +/// Counter: failed attempts, by `operation` and `error_class` (including +/// `attempt_timeout` for attempts cut off by the per-attempt timeout). +const METRIC_ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; +/// Histogram: wall-clock duration of a whole operation (attempts plus +/// backoff), in seconds, by `operation` and `outcome`. +const METRIC_OPERATION_DURATION_SECONDS: &str = "rustfs_kms_backend_operation_duration_seconds"; +/// Histogram: attempts one operation used before completing, by `operation` +/// and `outcome`. +const METRIC_OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts"; + +impl OpClass { + fn as_label(self) -> &'static str { + match self { + OpClass::ReadIdempotent => "read_idempotent", + OpClass::MutatingNonIdempotent => "mutating_non_idempotent", + OpClass::Auth => "auth", + } + } +} + +impl ErrorClass { + fn as_label(self) -> &'static str { + match self { + ErrorClass::RetryableConn => "retryable_conn", + ErrorClass::RetryableStatus => "retryable_status", + ErrorClass::Fatal => "fatal", + } + } +} + +/// How one policy execution terminated, for the `outcome` metric label. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Outcome { + Success, + /// A fatal-classified failure ended the operation on its first observation. + Fatal, + /// The attempt budget ran out; the last failure was retryable (including a + /// timed-out final attempt). + BudgetExhausted, + /// The operation deadline ran out before another attempt could complete. + DeadlineExceeded, + Cancelled, +} + +impl Outcome { + fn as_label(self) -> &'static str { + match self { + Outcome::Success => "success", + Outcome::Fatal => "fatal", + Outcome::BudgetExhausted => "budget_exhausted", + Outcome::DeadlineExceeded => "deadline_exceeded", + Outcome::Cancelled => "cancelled", + } + } +} + +/// Register metric descriptions once per process. +fn describe_metrics() { + static DESCRIBE: std::sync::Once = std::sync::Once::new(); + DESCRIBE.call_once(|| { + metrics::describe_counter!( + METRIC_OPERATIONS_TOTAL, + "Total KMS backend operations executed under the operation policy, by operation, operation class, and outcome" + ); + metrics::describe_counter!( + METRIC_ATTEMPT_FAILURES_TOTAL, + "Total failed KMS backend attempts, by operation and retry classification" + ); + metrics::describe_histogram!( + METRIC_OPERATION_DURATION_SECONDS, + "Wall-clock duration of KMS backend operations including retries and backoff, in seconds" + ); + metrics::describe_histogram!( + METRIC_OPERATION_ATTEMPTS, + "Number of attempts a KMS backend operation used before completing" + ); + }); +} + +/// Record one failed attempt with its retry classification. +fn record_attempt_failure(operation: &'static str, error_class: &'static str) { + metrics::counter!( + METRIC_ATTEMPT_FAILURES_TOTAL, + "operation" => operation, + "error_class" => error_class + ) + .increment(1); +} + +/// Record the terminal outcome of one policy execution. +fn record_operation(operation: &'static str, class: OpClass, outcome: Outcome, attempts: u32, elapsed: Duration) { + metrics::counter!( + METRIC_OPERATIONS_TOTAL, + "operation" => operation, + "op_class" => class.as_label(), + "outcome" => outcome.as_label() + ) + .increment(1); + metrics::histogram!( + METRIC_OPERATION_DURATION_SECONDS, + "operation" => operation, + "outcome" => outcome.as_label() + ) + .record(elapsed.as_secs_f64()); + metrics::histogram!( + METRIC_OPERATION_ATTEMPTS, + "operation" => operation, + "outcome" => outcome.as_label() + ) + .record(f64::from(attempts)); +} + /// Run `attempt` under the policy. /// /// Each attempt is bounded by `attempt_timeout` (further capped by whatever is @@ -230,13 +360,37 @@ where /// [`execute`] with an injectable jitter source so tests can pin deterministic /// backoff durations instead of asserting around random sleeps. pub(crate) async fn execute_with_jitter<T, F, Fut, J>( + operation: &'static str, + class: OpClass, + policy: &RetryPolicy, + cancel: &CancellationToken, + jitter: J, + attempt: F, +) -> Result<T> +where + F: FnMut() -> Fut, + Fut: Future<Output = std::result::Result<T, AttemptError>>, + J: FnMut(Duration) -> Duration, +{ + describe_metrics(); + let started = Instant::now(); + let mut attempts_made = 0u32; + let (outcome, result) = drive_attempts(operation, class, policy, cancel, jitter, attempt, &mut attempts_made).await; + record_operation(operation, class, outcome, attempts_made, started.elapsed()); + result +} + +/// The attempt loop behind [`execute_with_jitter`], returning the terminal +/// outcome alongside the result so the caller can record it exactly once. +async fn drive_attempts<T, F, Fut, J>( operation: &'static str, class: OpClass, policy: &RetryPolicy, cancel: &CancellationToken, mut jitter: J, mut attempt: F, -) -> Result<T> + attempts_made: &mut u32, +) -> (Outcome, Result<T>) where F: FnMut() -> Fut, Fut: Future<Output = std::result::Result<T, AttemptError>>, @@ -247,48 +401,68 @@ where let mut attempt_no = 0u32; loop { - attempt_no += 1; if cancel.is_cancelled() { - return Err(KmsError::operation_cancelled(format!( - "{operation} cancelled before attempt {attempt_no}" - ))); + return ( + Outcome::Cancelled, + Err(KmsError::operation_cancelled(format!( + "{operation} cancelled before attempt {}", + attempt_no + 1 + ))), + ); } let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { - return Err(KmsError::operation_timed_out(format!( - "{operation} exceeded operation deadline of {:?}", - policy.op_deadline - ))); + return ( + Outcome::DeadlineExceeded, + Err(KmsError::operation_timed_out(format!( + "{operation} exceeded operation deadline of {:?}", + policy.op_deadline + ))), + ); } + attempt_no += 1; + *attempts_made = attempt_no; let attempt_budget = policy.attempt_timeout.min(remaining); let outcome = tokio::select! { biased; _ = cancel.cancelled() => { - return Err(KmsError::operation_cancelled(format!("{operation} cancelled during attempt {attempt_no}"))); + return ( + Outcome::Cancelled, + Err(KmsError::operation_cancelled(format!("{operation} cancelled during attempt {attempt_no}"))), + ); } outcome = tokio::time::timeout(attempt_budget, attempt()) => outcome, }; let failure = match outcome { - Ok(Ok(value)) => return Ok(value), - Ok(Err(failure)) => failure, - Err(_) => AttemptError { - class: ErrorClass::RetryableConn, - error: KmsError::operation_timed_out(format!( - "{operation} attempt {attempt_no} timed out after {attempt_budget:?}" - )), - }, + Ok(Ok(value)) => return (Outcome::Success, Ok(value)), + Ok(Err(failure)) => { + record_attempt_failure(operation, failure.class.as_label()); + failure + } + Err(_) => { + record_attempt_failure(operation, "attempt_timeout"); + AttemptError { + class: ErrorClass::RetryableConn, + error: KmsError::operation_timed_out(format!( + "{operation} attempt {attempt_no} timed out after {attempt_budget:?}" + )), + } + } }; - if failure.class == ErrorClass::Fatal || attempt_no >= max_attempts { - return Err(failure.error); + if failure.class == ErrorClass::Fatal { + return (Outcome::Fatal, Err(failure.error)); + } + if attempt_no >= max_attempts { + return (Outcome::BudgetExhausted, Err(failure.error)); } let backoff = jitter(backoff_cap(policy, attempt_no)); if backoff >= deadline.saturating_duration_since(Instant::now()) { // Not enough deadline budget left for another attempt. - return Err(failure.error); + return (Outcome::DeadlineExceeded, Err(failure.error)); } tracing::warn!( operation, @@ -300,7 +474,10 @@ where tokio::select! { biased; _ = cancel.cancelled() => { - return Err(KmsError::operation_cancelled(format!("{operation} cancelled during retry backoff"))); + return ( + Outcome::Cancelled, + Err(KmsError::operation_cancelled(format!("{operation} cancelled during retry backoff"))), + ); } _ = tokio::time::sleep(backoff) => {} } @@ -628,4 +805,313 @@ mod tests { assert_eq!(classify_vaultrs(&ClientError::ResponseEmptyError), ErrorClass::Fatal); assert_eq!(classify_vaultrs(&ClientError::InvalidLoginMethodError), ErrorClass::Fatal); } + + // -- Metric emission ---------------------------------------------------- + // + // Each test installs a thread-local debugging recorder and drives a + // paused-clock current-thread runtime inside it, so the emitted metrics + // (including virtual-clock durations) are fully deterministic. + + use metrics_util::MetricKind; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + type MetricEntry = ( + metrics_util::CompositeKey, + Option<metrics::Unit>, + Option<metrics::SharedString>, + DebugValue, + ); + + /// Run `test` on a paused current-thread runtime under a debugging + /// recorder and return one snapshot of everything it emitted. + /// + /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains + /// the recorded state, so taking it per assertion would only show the + /// first assertion any data. + fn record_metrics<Out>(test: impl FnOnce() -> std::pin::Pin<Box<dyn Future<Output = Out>>>) -> (Vec<MetricEntry>, Out) { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let out = metrics::with_local_recorder(&recorder, || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .expect("current-thread runtime must build"); + runtime.block_on(test()) + }); + (snapshotter.snapshot().into_vec(), out) + } + + fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool { + labels + .iter() + .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected)) + } + + fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 { + snapshot + .iter() + .filter_map(|(composite, _unit, _description, value)| { + let matches = composite.kind() == MetricKind::Counter + && composite.key().name() == name + && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Counter(count)) => Some(*count), + _ => None, + } + }) + .sum() + } + + fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> { + snapshot + .iter() + .filter_map(|(composite, _unit, _description, value)| { + let matches = composite.kind() == MetricKind::Histogram + && composite.key().name() == name + && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Histogram(values)) => Some(values), + _ => None, + } + }) + .flatten() + .map(|value| value.into_inner()) + .collect() + } + + #[test] + fn metrics_record_retried_success_with_attempts_and_duration() { + let calls_in_test = Arc::new(AtomicU32::new(0)); + let (snapshot, ()) = record_metrics(move || { + Box::pin(async move { + let policy = policy_of(1_000, 60_000, 3, 100, 2_000); + let cancel = CancellationToken::new(); + let calls_in_attempt = calls_in_test.clone(); + execute_with_jitter("metrics_read", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + if calls.fetch_add(1, Ordering::SeqCst) < 2 { + Err(AttemptError { + class: ErrorClass::RetryableStatus, + error: KmsError::backend_error("throttled (429)"), + }) + } else { + Ok(()) + } + } + }) + .await + .expect("retries within budget must succeed"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[ + ("operation", "metrics_read"), + ("op_class", "read_idempotent"), + ("outcome", "success") + ] + ), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + METRIC_ATTEMPT_FAILURES_TOTAL, + &[("operation", "metrics_read"), ("error_class", "retryable_status")] + ), + 2 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_read"), ("outcome", "success")] + ), + vec![3.0] + ); + // Full-cap backoffs of 100ms and 200ms on the paused clock. + let durations = histogram_values( + &snapshot, + METRIC_OPERATION_DURATION_SECONDS, + &[("operation", "metrics_read"), ("outcome", "success")], + ); + assert_eq!(durations.len(), 1); + assert!((durations[0] - 0.3).abs() < 1e-9, "expected 0.3s of virtual backoff, got {durations:?}"); + } + + #[test] + fn metrics_record_fatal_outcome_with_single_attempt() { + let (snapshot, ()) = record_metrics(|| { + Box::pin(async { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let result: Result<()> = + execute_with_jitter("metrics_fatal", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, || async { + Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::access_denied("permission denied (403)"), + }) + }) + .await; + result.expect_err("a fatal failure must end the operation"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[("operation", "metrics_fatal"), ("outcome", "fatal")] + ), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + METRIC_ATTEMPT_FAILURES_TOTAL, + &[("operation", "metrics_fatal"), ("error_class", "fatal")] + ), + 1 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_fatal"), ("outcome", "fatal")] + ), + vec![1.0] + ); + } + + #[test] + fn metrics_record_mutating_budget_exhausted_after_one_attempt() { + let (snapshot, ()) = record_metrics(|| { + Box::pin(async { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let result: Result<()> = execute_with_jitter( + "metrics_rotate", + OpClass::MutatingNonIdempotent, + &policy, + &cancel, + full_jitter, + || async { Err(retryable_conn_error()) }, + ) + .await; + result.expect_err("a mutating operation must not retry a retryable failure"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[ + ("operation", "metrics_rotate"), + ("op_class", "mutating_non_idempotent"), + ("outcome", "budget_exhausted") + ] + ), + 1 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_rotate"), ("outcome", "budget_exhausted")] + ), + vec![1.0] + ); + } + + #[test] + fn metrics_record_timeouts_and_deadline_outcome() { + let (snapshot, ()) = record_metrics(|| { + Box::pin(async { + // Hung attempts: 10s each against a 25s deadline (see + // total_duration_never_exceeds_deadline for the timeline). + let policy = policy_of(10_000, 25_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let result: Result<()> = + execute_with_jitter("metrics_hung", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, || { + std::future::pending::<AttemptResult<()>>() + }) + .await; + result.expect_err("hung attempts must exhaust the deadline"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[("operation", "metrics_hung"), ("outcome", "deadline_exceeded")] + ), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + METRIC_ATTEMPT_FAILURES_TOTAL, + &[("operation", "metrics_hung"), ("error_class", "attempt_timeout")] + ), + 3 + ); + let durations = histogram_values( + &snapshot, + METRIC_OPERATION_DURATION_SECONDS, + &[("operation", "metrics_hung"), ("outcome", "deadline_exceeded")], + ); + assert_eq!(durations.len(), 1); + assert!((durations[0] - 25.0).abs() < 1e-9, "expected the full 25s deadline, got {durations:?}"); + } + + #[test] + fn metrics_record_cancelled_outcome() { + let calls_in_test = Arc::new(AtomicU32::new(0)); + let (snapshot, ()) = record_metrics(move || { + Box::pin(async move { + let policy = policy_of(1_000, 600_000, 5, 10_000, 10_000); + let cancel = CancellationToken::new(); + let canceller = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + canceller.cancel(); + }); + let calls_in_attempt = calls_in_test.clone(); + let result: Result<()> = + execute_with_jitter("metrics_cancel", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Err(retryable_conn_error()) + } + }) + .await; + result.expect_err("cancellation must abort the backoff"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[("operation", "metrics_cancel"), ("outcome", "cancelled")] + ), + 1 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_cancel"), ("outcome", "cancelled")] + ), + vec![1.0] + ); + } } diff --git a/crates/kms/tests/vault_fault_injection.rs b/crates/kms/tests/vault_fault_injection.rs new file mode 100644 index 000000000..14176fdb9 --- /dev/null +++ b/crates/kms/tests/vault_fault_injection.rs @@ -0,0 +1,255 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Fault-injection matrix for the Vault backend operation policy. +//! +//! Offline cases run against locally injected transport faults (a closed +//! port, a listener that never responds) — deterministic, no external +//! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault +//! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`). +//! +//! Throttling (429) and recoverable 5xx responses cannot be forced on a stock +//! dev Vault, so their retry and metric behavior is pinned deterministically +//! by the scripted-Vault wiring tests in `backends::vault` and the engine +//! tests in `policy.rs`. Pointing `RUSTFS_KMS_VAULT_ADDR` at a +//! fault-injecting proxy reuses the ignored cases here unchanged. +//! +//! Every case installs a thread-local debugging metrics recorder and drives a +//! current-thread runtime inside it, so the policy metrics double as the +//! request-count assertion even against a real server. + +use std::time::Duration; + +use metrics_util::MetricKind; +use metrics_util::debugging::{DebugValue, DebuggingRecorder}; +use rustfs_kms::backends::KmsClient; +use rustfs_kms::backends::vault::VaultKmsClient; +use rustfs_kms::{KmsConfig, KmsError, VaultAuthMethod, VaultConfig}; + +const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; +const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; + +fn vault_config(address: &str, token: &str) -> VaultConfig { + VaultConfig { + address: address.to_string(), + auth_method: VaultAuthMethod::Token { + token: token.to_string(), + }, + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/fault-injection".to_string(), + tls: None, + } +} + +fn kms_config(attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig { + KmsConfig { + timeout: attempt_timeout, + retry_attempts, + ..KmsConfig::default() + } +} + +type MetricEntry = ( + metrics_util::CompositeKey, + Option<metrics::Unit>, + Option<metrics::SharedString>, + DebugValue, +); + +/// Run `test` on a current-thread runtime under a debugging metrics recorder +/// and return one snapshot of everything it emitted. +/// +/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains the +/// recorded state, so taking it per assertion would only show the first +/// assertion any data. +fn record_metrics(test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()>>>) -> Vec<MetricEntry> { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime must build"); + runtime.block_on(test()); + }); + snapshotter.snapshot().into_vec() +} + +/// Sum of counters with `name` whose labels include every `(key, value)` pair. +fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 { + snapshot + .iter() + .filter_map(|(composite, _unit, _description, value)| { + let key = composite.key(); + let matches = composite.kind() == MetricKind::Counter + && key.name() == name + && labels + .iter() + .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected)); + match (matches, value) { + (true, DebugValue::Counter(count)) => Some(*count), + _ => None, + } + }) + .sum() +} + +/// Connection refused: connection-class failures are retried up to the +/// configured budget, then surface as a backend error. +#[test] +fn connection_refused_is_retried_within_budget() { + // Reserve a loopback port and release it so nothing is listening there. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve a loopback port"); + let address = format!("http://{}", listener.local_addr().expect("reserved port addr")); + drop(listener); + + let snapshot = record_metrics(|| { + Box::pin(async move { + let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_secs(2), 2)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-refused", None) + .await + .expect_err("a refused connection must fail the operation"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + }) + }); + + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]), + 2, + "both budgeted attempts must observe the refused connection" + ); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1); + // The static-token login records its own success; the Vault read must not. + assert_eq!( + counter_value( + &snapshot, + OPERATIONS_TOTAL, + &[("operation", "vault_kv2_read_key"), ("outcome", "success")] + ), + 0 + ); +} + +/// Stalled connection: a server that accepts but never responds is cut off by +/// the per-attempt timeout (either the policy timer or the equally sized HTTP +/// client timeout, whichever fires first) instead of hanging forever. +#[test] +fn stalled_connection_is_cut_off_by_the_attempt_timeout() { + let snapshot = record_metrics(|| { + Box::pin(async move { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind stall listener"); + let address = format!("http://{}", listener.local_addr().expect("stall listener addr")); + // Accept and park every connection without ever responding. + tokio::spawn(async move { + let mut parked = Vec::new(); + loop { + let Ok((socket, _)) = listener.accept().await else { return }; + parked.push(socket); + } + }); + + let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_millis(250), 1)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-stalled", None) + .await + .expect_err("a stalled request must be cut off by the attempt timeout"); + assert!( + matches!(error, KmsError::OperationTimedOut { .. } | KmsError::BackendError { .. }), + "got {error:?}" + ); + }) + }); + + // The policy timer reports attempt_timeout; the client-level HTTP timeout + // surfaces as a connection-class failure. Either way it is exactly one + // attempt that was cut off. + let cut_off = counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "attempt_timeout")]) + + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]); + assert_eq!(cut_off, 1, "the single budgeted attempt must be cut off by a timeout"); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1); +} + +fn real_vault_address() -> String { + std::env::var("RUSTFS_KMS_VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()) +} + +/// Invalid token against a real Vault: the 403 is fatal — exactly one +/// attempt, no retry, and the operation fails closed. +#[test] +#[ignore] // Requires a running Vault dev server +fn real_vault_invalid_token_is_fatal_and_never_retried() { + let snapshot = record_metrics(|| { + Box::pin(async { + let config = vault_config(&real_vault_address(), "fault-injection-invalid-token"); + let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-forbidden", None) + .await + .expect_err("an invalid token must be rejected"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + }) + }); + + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]), + 1, + "a 403 must be observed by exactly one attempt" + ); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1); + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_status")]), + 0, + "an auth failure must never be classified as retryable" + ); +} + +/// Healthy read against a real Vault: a missing key resolves in one attempt +/// (404 is fatal for retry purposes) and records a fatal outcome rather than +/// burning the retry budget. +#[test] +#[ignore] // Requires a running Vault dev server +fn real_vault_missing_key_is_resolved_in_one_attempt() { + let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-only-token".to_string()); + let snapshot = record_metrics(|| { + Box::pin(async move { + let config = vault_config(&real_vault_address(), &token); + let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-definitely-missing", None) + .await + .expect_err("a missing key must resolve to key-not-found"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}"); + }) + }); + + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]), + 1, + "a 404 must be observed by exactly one attempt" + ); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1); +} From 78d6918c526c031c85935741c77a360a97e9762f Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Fri, 31 Jul 2026 13:42:19 +0800 Subject: [PATCH 37/52] feat: extend hotpath coverage across crates (#5505) Add opt-in hotpath feature surfaces to every workspace crate and wire the root rustfs feature passthrough for function, allocation, and CPU profiling. Add a focused set of function-level measurements for scanner, heal, lock, target replay, IAM, KMS, Keystone, trusted proxy, and capacity paths without adding request-scoped primitive wrappers. Co-authored-by: heihutu <heihutu@gmail.com> --- Cargo.lock | 42 +++++++ crates/audit/Cargo.toml | 26 ++++ crates/checksums/Cargo.toml | 7 ++ crates/common/Cargo.toml | 7 ++ crates/concurrency/Cargo.toml | 7 ++ crates/config/Cargo.toml | 4 + crates/credentials/Cargo.toml | 7 ++ crates/crypto/Cargo.toml | 4 + crates/data-usage/Cargo.toml | 7 ++ crates/e2e_test/Cargo.toml | 48 ++++++++ crates/ecstore/Cargo.toml | 64 ++++++++++ crates/ecstore/src/lib.rs | 2 + crates/extension-schema/Cargo.toml | 7 ++ crates/filemeta/Cargo.toml | 6 +- crates/heal/Cargo.toml | 41 +++++++ crates/heal/src/heal/erasure_healer.rs | 1 + crates/heal/src/heal/task.rs | 3 + crates/iam/Cargo.toml | 48 ++++++++ crates/iam/src/store/object.rs | 2 + crates/io-core/Cargo.toml | 7 ++ crates/io-metrics/Cargo.toml | 25 ++++ crates/keystone/Cargo.toml | 27 +++++ crates/keystone/src/client.rs | 2 + crates/kms/Cargo.toml | 15 +++ crates/kms/src/manager.rs | 3 + crates/lifecycle/Cargo.toml | 28 +++++ crates/lock/Cargo.toml | 14 +++ crates/lock/src/distributed_lock.rs | 4 + crates/log-analyzer/Cargo.toml | 7 ++ crates/madmin/Cargo.toml | 7 ++ crates/notify/Cargo.toml | 31 +++++ crates/object-capacity/Cargo.toml | 26 ++++ .../object-capacity/src/capacity_manager.rs | 2 + crates/object-capacity/src/scan.rs | 1 + crates/object-data-cache/Cargo.toml | 7 ++ crates/obs/Cargo.toml | 45 +++++++ crates/policy/Cargo.toml | 24 +++- crates/protocols/Cargo.toml | 50 ++++++++ crates/protos/Cargo.toml | 31 +++++ crates/replication/Cargo.toml | 7 ++ crates/rio-v2/Cargo.toml | 25 ++++ crates/rio/Cargo.toml | 29 ++++- crates/s3-ops/Cargo.toml | 7 ++ crates/s3-types/Cargo.toml | 7 ++ crates/s3select-api/Cargo.toml | 30 +++++ crates/s3select-query/Cargo.toml | 13 ++ crates/scanner/Cargo.toml | 41 +++++++ crates/scanner/src/scanner.rs | 1 + crates/security-governance/Cargo.toml | 7 ++ crates/signer/Cargo.toml | 7 ++ crates/storage-api/Cargo.toml | 7 ++ crates/targets/Cargo.toml | 34 ++++++ crates/targets/src/runtime/mod.rs | 5 + crates/targets/src/target/webhook.rs | 2 + crates/test-utils/Cargo.toml | 25 ++++ crates/tls-runtime/Cargo.toml | 7 ++ crates/trusted-proxies/Cargo.toml | 13 ++ crates/trusted-proxies/src/cloud/ranges.rs | 1 + crates/utils/Cargo.toml | 4 + crates/zip/Cargo.toml | 7 ++ rustfs/Cargo.toml | 112 ++++++++++++++++++ 61 files changed, 1071 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d52e411b..416cbd5c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3672,6 +3672,7 @@ dependencies = [ "flate2", "futures", "hex", + "hotpath", "http 1.5.0", "http-body-util", "hyper", @@ -9039,6 +9040,7 @@ dependencies = [ "const-str", "futures", "hashbrown 0.17.1", + "hotpath", "metrics", "rustfs-config", "rustfs-s3-types", @@ -9059,6 +9061,7 @@ dependencies = [ "base64-simd", "bytes", "crc-fast", + "hotpath", "http 1.5.0", "md-5 0.11.0", "pretty_assertions", @@ -9072,6 +9075,7 @@ name = "rustfs-common" version = "1.0.0-beta.12" dependencies = [ "chrono", + "hotpath", "metrics", "rmp-serde", "s3s", @@ -9086,6 +9090,7 @@ dependencies = [ name = "rustfs-concurrency" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "insta", "rustfs-io-core", "serde", @@ -9099,6 +9104,7 @@ name = "rustfs-config" version = "1.0.0-beta.12" dependencies = [ "const-str", + "hotpath", "serde", "serde_json", ] @@ -9109,6 +9115,7 @@ version = "1.0.0-beta.12" dependencies = [ "base64-simd", "hmac 0.13.0", + "hotpath", "rand 0.10.2", "serde", "serde_json", @@ -9124,6 +9131,7 @@ dependencies = [ "argon2", "base64-simd", "chacha20poly1305", + "hotpath", "jsonwebtoken 11.0.0", "pbkdf2 0.13.0", "rand 0.10.2", @@ -9141,6 +9149,7 @@ name = "rustfs-data-usage" version = "1.0.0-beta.12" dependencies = [ "async-trait", + "hotpath", "rmp-serde", "rustfs-filemeta", "serde", @@ -9286,6 +9295,7 @@ dependencies = [ name = "rustfs-extension-schema" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "serde", "serde_json", "thiserror 2.0.19", @@ -9324,6 +9334,7 @@ dependencies = [ "async-trait", "base64 0.23.0", "futures", + "hotpath", "http 1.5.0", "metrics", "rustfs-common", @@ -9355,6 +9366,7 @@ dependencies = [ "async-trait", "base64-simd", "futures", + "hotpath", "http 1.5.0", "jsonwebtoken 11.0.0", "moka", @@ -9389,6 +9401,7 @@ name = "rustfs-io-core" version = "1.0.0-beta.12" dependencies = [ "bytes", + "hotpath", "memmap2", "rustfs-io-metrics", "thiserror 2.0.19", @@ -9401,6 +9414,7 @@ name = "rustfs-io-metrics" version = "1.0.0-beta.12" dependencies = [ "criterion", + "hotpath", "metrics", "metrics-util", "num_cpus", @@ -9467,6 +9481,7 @@ version = "1.0.0-beta.12" dependencies = [ "bytes", "futures", + "hotpath", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -9499,6 +9514,7 @@ dependencies = [ "base64 0.23.0", "chacha20poly1305", "hex", + "hotpath", "insta", "jiff", "md-5 0.11.0", @@ -9531,6 +9547,7 @@ name = "rustfs-lifecycle" version = "1.0.0-beta.12" dependencies = [ "async-trait", + "hotpath", "metrics", "metrics-util", "proptest", @@ -9555,6 +9572,7 @@ dependencies = [ "async-trait", "crossbeam-queue", "futures", + "hotpath", "parking_lot", "rand 0.10.2", "rustfs-io-metrics", @@ -9576,6 +9594,7 @@ version = "1.0.0-beta.12" dependencies = [ "chrono", "flate2", + "hotpath", "regex", "serde", "serde_json", @@ -9593,6 +9612,7 @@ name = "rustfs-madmin" version = "1.0.0-beta.12" dependencies = [ "chrono", + "hotpath", "humantime", "hyper", "rmp-serde", @@ -9613,6 +9633,7 @@ dependencies = [ "criterion", "form_urlencoded", "hashbrown 0.17.1", + "hotpath", "metrics", "percent-encoding", "quick-xml", @@ -9642,6 +9663,7 @@ version = "1.0.0-beta.12" dependencies = [ "criterion", "futures", + "hotpath", "rustfs-config", "rustfs-io-metrics", "rustfs-utils", @@ -9660,6 +9682,7 @@ version = "1.0.0-beta.12" dependencies = [ "bytes", "criterion", + "hotpath", "metrics", "metrics-util", "moka", @@ -9682,6 +9705,7 @@ dependencies = [ "flate2", "futures-util", "glob", + "hotpath", "jiff", "libc", "metrics", @@ -9767,6 +9791,7 @@ dependencies = [ "futures-util", "hex", "hmac 0.13.0", + "hotpath", "http 1.5.0", "http-body-util", "hyper", @@ -9819,6 +9844,7 @@ name = "rustfs-protos" version = "1.0.0-beta.12" dependencies = [ "flatbuffers", + "hotpath", "prost 0.14.4", "rmp-serde", "rustfs-common", @@ -9843,6 +9869,7 @@ version = "1.0.0-beta.12" dependencies = [ "byteorder", "bytes", + "hotpath", "regex", "rmp", "rmp-serde", @@ -9901,6 +9928,7 @@ dependencies = [ "chacha20poly1305", "hex", "hmac 0.13.0", + "hotpath", "minlz", "pin-project-lite", "rand 0.10.2", @@ -9918,6 +9946,7 @@ dependencies = [ name = "rustfs-s3-ops" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "rustfs-s3-types", ] @@ -9925,6 +9954,7 @@ dependencies = [ name = "rustfs-s3-types" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "serde", "serde_json", ] @@ -9939,6 +9969,7 @@ dependencies = [ "datafusion", "futures", "futures-core", + "hotpath", "http 1.5.0", "metrics", "parking_lot", @@ -9967,6 +9998,7 @@ dependencies = [ "datafusion", "derive_builder", "futures", + "hotpath", "parking_lot", "rustfs-s3select-api", "s3s", @@ -9984,6 +10016,7 @@ dependencies = [ "futures", "hex-simd", "hmac 0.13.0", + "hotpath", "http 1.5.0", "metrics", "rand 0.10.2", @@ -10016,6 +10049,7 @@ dependencies = [ name = "rustfs-security-governance" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "thiserror 2.0.19", ] @@ -10025,6 +10059,7 @@ version = "1.0.0-beta.12" dependencies = [ "base64-simd", "bytes", + "hotpath", "http 1.5.0", "hyper", "rustfs-utils", @@ -10041,6 +10076,7 @@ name = "rustfs-storage-api" version = "1.0.0-beta.12" dependencies = [ "async-trait", + "hotpath", "insta", "rustfs-filemeta", "serde", @@ -10062,6 +10098,7 @@ dependencies = [ "deadpool-postgres", "futures-util", "hashbrown 0.17.1", + "hotpath", "hyper", "hyper-rustls", "lapin", @@ -10107,6 +10144,7 @@ dependencies = [ name = "rustfs-test-utils" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "rustfs-data-usage", "rustfs-ecstore", "rustfs-storage-api", @@ -10123,6 +10161,7 @@ name = "rustfs-tls-runtime" version = "1.0.0-beta.12" dependencies = [ "arc-swap", + "hotpath", "metrics", "rcgen", "rustfs-common", @@ -10144,6 +10183,7 @@ version = "1.0.0-beta.12" dependencies = [ "async-trait", "axum", + "hotpath", "http 1.5.0", "ipnetwork", "metrics", @@ -10190,6 +10230,7 @@ dependencies = [ "hex-simd", "highway", "hmac 0.13.0", + "hotpath", "http 1.5.0", "hyper", "local-ip-address", @@ -10222,6 +10263,7 @@ dependencies = [ "astral-tokio-tar", "async-compression", "criterion", + "hotpath", "tempfile", "thiserror 2.0.19", "tokio", diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml index d598bc281..421b2b3a2 100644 --- a/crates/audit/Cargo.toml +++ b/crates/audit/Cargo.toml @@ -25,7 +25,33 @@ documentation = "https://docs.rs/rustfs-audit/latest/rustfs_audit/" keywords = ["audit", "target", "management", "fan-out", "RustFS"] categories = ["web-programming", "development-tools", "asynchronous", "api-bindings"] +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-config/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-targets/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-targets/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-targets/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-targets = { workspace = true } rustfs-config = { workspace = true, features = ["audit", "server-config-model"] } rustfs-s3-types = { workspace = true } diff --git a/crates/checksums/Cargo.toml b/crates/checksums/Cargo.toml index 6af2106b1..0229dd3a8 100644 --- a/crates/checksums/Cargo.toml +++ b/crates/checksums/Cargo.toml @@ -28,7 +28,14 @@ documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/" [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } crc-fast = { workspace = true } http = { workspace = true } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 93fe21df7..df98511d6 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools", "data-structures"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } tonic = { workspace = true, features = ["gzip", "deflate"] } uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] } diff --git a/crates/concurrency/Cargo.toml b/crates/concurrency/Cargo.toml index 55eb20eb9..ebc7ae095 100644 --- a/crates/concurrency/Cargo.toml +++ b/crates/concurrency/Cargo.toml @@ -13,7 +13,14 @@ categories = ["concurrency", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"] + [dependencies] +hotpath.workspace = true # Internal crates rustfs-io-core = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 9dcd0ec2f..7b61b1c5f 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -25,6 +25,7 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "config"] [dependencies] +hotpath.workspace = true const-str = { workspace = true, optional = true, features = ["std", "proc"] } serde = { workspace = true, optional = true, features = ["derive"] } serde_json = { workspace = true, optional = true, features = ["raw_value"] } @@ -34,6 +35,9 @@ workspace = true [features] default = ["constants"] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] audit = ["dep:const-str", "constants"] constants = ["dep:const-str"] notify = ["dep:const-str", "constants"] diff --git a/crates/credentials/Cargo.toml b/crates/credentials/Cargo.toml index bc2eca866..605d257d5 100644 --- a/crates/credentials/Cargo.toml +++ b/crates/credentials/Cargo.toml @@ -24,7 +24,14 @@ description = "Credentials management utilities for RustFS, enabling secure hand keywords = ["rustfs", "Minio", "credentials", "authentication", "authorization"] categories = ["web-programming", "development-tools", "data-structures", "security"] +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true base64-simd = { workspace = true } hmac = { workspace = true } rand = { workspace = true, features = ["serde"] } diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index e7cdf3357..080db7364 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -29,6 +29,7 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/" workspace = true [dependencies] +hotpath.workspace = true aes-gcm = { workspace = true, optional = true, features = ["rand_core"] } argon2 = { workspace = true, optional = true } chacha20poly1305 = { workspace = true, optional = true } @@ -49,6 +50,9 @@ time = { workspace = true, features = ["parsing", "formatting", "macros", "serde [features] default = ["crypto", "fips"] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] fips = [] crypto = [ "dep:aes-gcm", diff --git a/crates/data-usage/Cargo.toml b/crates/data-usage/Cargo.toml index b239e3dd6..3e33b530e 100644 --- a/crates/data-usage/Cargo.toml +++ b/crates/data-usage/Cargo.toml @@ -27,7 +27,14 @@ categories = ["data-structures", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "rustfs-filemeta/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-filemeta/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"] + [dependencies] +hotpath.workspace = true serde = { workspace = true, features = ["derive"] } rmp-serde = { workspace = true } async-trait = { workspace = true } diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index 00ddbc47a..b0bc6ed2a 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -25,10 +25,58 @@ workspace = true [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-data-usage/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-filemeta/hotpath", + "rustfs-lock/hotpath", + "rustfs-madmin/hotpath", + "rustfs-protos/hotpath", + "rustfs-rio/hotpath", + "rustfs-signer/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", + "rustfs-lock/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-protos/hotpath-alloc", + "rustfs-rio/hotpath-alloc", + "rustfs-signer/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", + "rustfs-lock/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-protos/hotpath-cpu", + "rustfs-rio/hotpath-cpu", + "rustfs-signer/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] ftps = [] sftp = [] [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["constants"] } rustfs-credentials.workspace = true rustfs-ecstore.workspace = true diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 726c5f2a2..84ad18647 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -40,19 +40,83 @@ hotpath = [ "hotpath/async-channel", "hotpath/parking_lot", "hotpath/reqwest-0-13", + "rustfs-checksums/hotpath", + "rustfs-common/hotpath", + "rustfs-concurrency/hotpath", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-data-usage/hotpath", "rustfs-filemeta/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-lifecycle/hotpath", + "rustfs-lock/hotpath", + "rustfs-madmin/hotpath", + "rustfs-object-capacity/hotpath", + "rustfs-policy/hotpath", + "rustfs-protos/hotpath", + "rustfs-replication/hotpath", "rustfs-rio/hotpath", + "rustfs-rio-v2?/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-signer/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", + "rustfs-crypto/hotpath", ] hotpath-alloc = [ + "hotpath", "hotpath/hotpath-alloc", + "rustfs-checksums/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-concurrency/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", "rustfs-filemeta/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-lifecycle/hotpath-alloc", + "rustfs-lock/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-object-capacity/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-protos/hotpath-alloc", + "rustfs-replication/hotpath-alloc", "rustfs-rio/hotpath-alloc", + "rustfs-rio-v2?/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-signer/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", ] hotpath-cpu = [ "hotpath", "hotpath/hotpath-cpu", + "rustfs-checksums/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-concurrency/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", "rustfs-filemeta/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-lifecycle/hotpath-cpu", + "rustfs-lock/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-object-capacity/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-protos/hotpath-cpu", + "rustfs-replication/hotpath-cpu", "rustfs-rio/hotpath-cpu", + "rustfs-rio-v2?/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-signer/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", ] # Exposes shared lifecycle/tier test utilities (MockWarmBackend, fault # injection, xl.meta transition assertions) via `api::tier::test_util`. diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index e0c7d7b6d..2ed643ead 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![recursion_limit = "256"] + /// Scope-based hotpath measurement for `#[async_trait]` methods, where /// `#[hotpath::measure]` would only time the boxed-future construction. /// The guard records wall time from this statement until the enclosing diff --git a/crates/extension-schema/Cargo.toml b/crates/extension-schema/Cargo.toml index 84d9449a1..5e6af760d 100644 --- a/crates/extension-schema/Cargo.toml +++ b/crates/extension-schema/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools"] [lib] doctest = false +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true serde = { workspace = true, features = ["derive"] } thiserror.workspace = true diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index f1d2bbad2..78d6e1c1c 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -27,9 +27,9 @@ documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/" [features] default = [] -hotpath = ["hotpath/hotpath", "hotpath/tokio"] -hotpath-alloc = ["hotpath/hotpath-alloc"] -hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-utils/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-utils/hotpath-cpu"] [dependencies] hotpath.workspace = true diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index e213c7da5..668c4af24 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -29,7 +29,48 @@ categories = ["web-programming", "development-tools", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-common/hotpath", + "rustfs-concurrency/hotpath", + "rustfs-config/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-madmin/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-concurrency/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-concurrency/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true } rustfs-concurrency = { workspace = true } rustfs-ecstore = { workspace = true } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 2f4c6f87d..d08367f3f 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -159,6 +159,7 @@ impl ErasureSetHealer { /// execute erasure set heal with resume #[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))] + #[hotpath::measure] pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> { debug!( target: "rustfs::heal::erasure_healer", diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index dbd505bb3..8057f6967 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -584,6 +584,7 @@ impl HealTask { } #[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))] + #[hotpath::measure] pub async fn execute(&self) -> Result<()> { // update status and timestamps atomically to avoid race conditions let now = SystemTime::now(); @@ -759,6 +760,7 @@ impl HealTask { // specific heal implementation method #[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))] + #[hotpath::measure] async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -1404,6 +1406,7 @@ impl HealTask { self.heal_bucket_objects(bucket, prefix).await } + #[hotpath::measure] async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> { let mut continuation_token: Option<String> = None; let mut scanned = 0u64; diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index 8c4a3c2aa..0ef8ef1e9 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -28,7 +28,55 @@ documentation = "https://docs.rs/rustfs-iam/latest/rustfs_iam/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-crypto/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-madmin/hotpath", + "rustfs-policy/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-credentials = { workspace = true } rustfs-config = { workspace = true, features = ["server-config-model"] } tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index f7e652118..03530084e 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -469,6 +469,7 @@ impl ObjectStore { }); } + #[hotpath::measure] async fn list_all_iamconfig_items(&self) -> Result<HashMap<String, Vec<String>>> { let (tx, mut rx) = mpsc::channel::<StringOrErr>(100); @@ -508,6 +509,7 @@ impl ObjectStore { Ok(res) } + #[hotpath::measure] async fn load_policy_doc_concurrent(&self, names: &[String], mode: LoadMode) -> Result<Vec<PolicyDoc>> { let mut futures = Vec::with_capacity(names.len()); diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index 5d53c9288..9c309633b 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -27,7 +27,14 @@ categories = ["development-tools", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] } diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index d77725ae5..64eac650b 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -28,7 +28,32 @@ categories = ["development-tools", "filesystem"] name = "metrics_pipeline" harness = false +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-common/hotpath", + "rustfs-s3-ops/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-s3-ops/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-s3-ops/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true metrics = { workspace = true } rustfs-common = { workspace = true } rustfs-s3-ops = { workspace = true } diff --git a/crates/keystone/Cargo.toml b/crates/keystone/Cargo.toml index 50110221e..e67adf7f5 100644 --- a/crates/keystone/Cargo.toml +++ b/crates/keystone/Cargo.toml @@ -28,7 +28,34 @@ authors.workspace = true [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-credentials/hotpath", + "rustfs-policy/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } reqwest = { workspace = true, features = ["json"] } serde = { workspace = true, features = ["derive"] } diff --git a/crates/keystone/src/client.rs b/crates/keystone/src/client.rs index ffb5fcdd4..d5df169e8 100644 --- a/crates/keystone/src/client.rs +++ b/crates/keystone/src/client.rs @@ -95,6 +95,7 @@ impl KeystoneClient { } /// Validate a Keystone token + #[hotpath::measure] pub async fn validate_token(&self, token: &str) -> Result<KeystoneToken> { match self.version { KeystoneVersion::V3 => self.validate_token_v3(token).await, @@ -238,6 +239,7 @@ impl KeystoneClient { } /// Get EC2 credentials for a user + #[hotpath::measure] pub async fn get_ec2_credentials(&self, user_id: &str, project_id: Option<&str>) -> Result<Vec<EC2Credential>> { let admin_token = self.get_admin_token().await?; diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 33fa8e3bd..1dd5b9385 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -28,6 +28,7 @@ categories = ["cryptography", "web-programming", "authentication"] workspace = true [dependencies] +hotpath.workspace = true # Core dependencies async-trait = { workspace = true } tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread", "sync", "time"] } @@ -84,3 +85,17 @@ tokio = { workspace = true, features = ["net", "test-util"] } [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/reqwest-0-13", + "rustfs-security-governance/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-security-governance/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-security-governance/hotpath-cpu", "rustfs-utils/hotpath-cpu"] diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 7ae6bc055..9d77e07b4 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -66,16 +66,19 @@ impl KmsManager { } /// Encrypt data with a master key + #[hotpath::measure] pub async fn encrypt(&self, request: EncryptRequest) -> Result<EncryptResponse> { self.backend.encrypt(request).await } /// Decrypt data with a master key + #[hotpath::measure] pub async fn decrypt(&self, request: DecryptRequest) -> Result<DecryptResponse> { self.backend.decrypt(request).await } /// Generate a data encryption key + #[hotpath::measure] pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> { self.backend.generate_data_key(request).await } diff --git a/crates/lifecycle/Cargo.toml b/crates/lifecycle/Cargo.toml index 43dfac6aa..8f5f63556 100644 --- a/crates/lifecycle/Cargo.toml +++ b/crates/lifecycle/Cargo.toml @@ -25,7 +25,35 @@ keywords = ["lifecycle", "storage", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-lifecycle/latest/rustfs_lifecycle/" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-replication/hotpath", + "rustfs-storage-api/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-replication/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-replication/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true async-trait.workspace = true metrics.workspace = true rustfs-common.workspace = true diff --git a/crates/lock/Cargo.toml b/crates/lock/Cargo.toml index 7b0993adb..dcaccf4a6 100644 --- a/crates/lock/Cargo.toml +++ b/crates/lock/Cargo.toml @@ -28,7 +28,21 @@ documentation = "https://docs.rs/rustfs-lock/latest/rustfs_lock/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "rustfs-io-metrics/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu", "rustfs-utils/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-io-metrics = { workspace = true } rustfs-utils = { workspace = true } async-trait.workspace = true diff --git a/crates/lock/src/distributed_lock.rs b/crates/lock/src/distributed_lock.rs index 97b8af06d..e3465d732 100644 --- a/crates/lock/src/distributed_lock.rs +++ b/crates/lock/src/distributed_lock.rs @@ -540,6 +540,7 @@ impl DistributedLock { } /// Acquire a lock and return a RAII guard + #[hotpath::measure] pub(crate) async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<DistributedLockGuard>> { if self.clients.is_empty() { return Err(LockError::internal("No lock clients available")); @@ -626,6 +627,7 @@ impl DistributedLock { } /// Convenience: acquire exclusive lock as a guard + #[hotpath::measure] pub async fn lock_guard( &self, resource: ObjectKey, @@ -640,6 +642,7 @@ impl DistributedLock { } /// Convenience: acquire exclusive lock with expected contention logs suppressed + #[hotpath::measure] pub async fn lock_guard_quiet( &self, resource: ObjectKey, @@ -655,6 +658,7 @@ impl DistributedLock { } /// Convenience: acquire shared lock as a guard + #[hotpath::measure] pub async fn rlock_guard( &self, resource: ObjectKey, diff --git a/crates/log-analyzer/Cargo.toml b/crates/log-analyzer/Cargo.toml index b8830e4d0..2c0eebcb4 100644 --- a/crates/log-analyzer/Cargo.toml +++ b/crates/log-analyzer/Cargo.toml @@ -32,7 +32,14 @@ doctest = false name = "la-dump-anchors" path = "src/bin/la_dump_anchors.rs" +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true chrono = { workspace = true, features = ["serde"] } flate2 = { workspace = true } regex = { workspace = true } diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index e5b395735..2d704d2d4 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -28,7 +28,14 @@ documentation = "https://docs.rs/rustfs-madmin/latest/rustfs_madmin/" [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true chrono = { workspace = true, features = ["serde"] } humantime.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index e6808f364..ddbb5d6de 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -26,9 +26,40 @@ categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/" [features] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-config/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-s3-ops/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-targets/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-s3-ops/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-targets/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-s3-ops/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-targets/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] demo-examples = [] [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["notify", "server-config-model"] } rustfs-ecstore = { workspace = true } rustfs-s3-types = { workspace = true } diff --git a/crates/object-capacity/Cargo.toml b/crates/object-capacity/Cargo.toml index 0ae8161df..fd17815f4 100644 --- a/crates/object-capacity/Cargo.toml +++ b/crates/object-capacity/Cargo.toml @@ -34,7 +34,33 @@ harness = false [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-config/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["constants"] } rustfs-io-metrics = { workspace = true } rustfs-utils = { workspace = true, features = ["os"] } diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 876fa9be6..ed3107b9b 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -1035,6 +1035,7 @@ impl HybridCapacityManager { /// /// Joiners subscribe to the watch channel *before* releasing the mutex, which guarantees /// they cannot miss the completion notification even if the leader finishes very quickly. + #[hotpath::measure] pub async fn refresh_or_join<F, Fut>(&self, source: DataSource, refresh_fn: F) -> Result<CapacityUpdate, String> where F: FnOnce() -> Fut, @@ -1142,6 +1143,7 @@ impl HybridCapacityManager { } /// Start a background refresh if one is not already in flight. + #[hotpath::measure] pub async fn spawn_refresh_if_needed<F, Fut>(self: Arc<Self>, source: DataSource, refresh_fn: F) -> bool where F: FnOnce() -> Fut + Send + 'static, diff --git a/crates/object-capacity/src/scan.rs b/crates/object-capacity/src/scan.rs index 2021dd84b..62bb01b93 100644 --- a/crates/object-capacity/src/scan.rs +++ b/crates/object-capacity/src/scan.rs @@ -338,6 +338,7 @@ pub async fn select_capacity_refresh_disks( } } +#[hotpath::measure] pub async fn refresh_capacity_with_scope(disks: Vec<CapacityDiskRef>, dirty_subset: bool) -> Result<CapacityUpdate, String> { let scan_started_at = Instant::now(); let report = calculate_data_dir_used_capacity_report(&disks) diff --git a/crates/object-data-cache/Cargo.toml b/crates/object-data-cache/Cargo.toml index a89ce6a50..53a7ad5d3 100644 --- a/crates/object-data-cache/Cargo.toml +++ b/crates/object-data-cache/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools"] [lib] doctest = false +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } metrics = { workspace = true } moka = { workspace = true, features = ["future"] } diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 6f42090e5..8b627c353 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -27,6 +27,50 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/" [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/crossbeam", + "rustfs-audit/hotpath", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-iam/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-notify/hotpath", + "rustfs-security-governance/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-audit/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-iam/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-notify/hotpath-alloc", + "rustfs-security-governance/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-audit/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-iam/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-notify/hotpath-cpu", + "rustfs-security-governance/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] # Tokio runtime-level telemetry. Requires a `--cfg tokio_unstable` build; the # build script fails the compile when that flag is missing. Off by default so # ordinary builds neither pay for nor depend on Tokio's unstable API. @@ -58,6 +102,7 @@ required-features = ["dial9"] workspace = true [dependencies] +hotpath.workspace = true rustfs-audit = { workspace = true } rustfs-common = { workspace = true } rustfs-config = { workspace = true, features = ["observability"] } diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml index b85ded8b5..1732045fb 100644 --- a/crates/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -30,9 +30,27 @@ workspace = true [features] default = [] -hotpath = ["hotpath/hotpath", "hotpath/reqwest-0-13"] -hotpath-alloc = ["hotpath/hotpath-alloc"] -hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] +hotpath = [ + "hotpath/hotpath", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-crypto/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", +] [dependencies] rustfs-credentials = { workspace = true } diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 2d53eca84..91da8efce 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -30,6 +30,55 @@ workspace = true [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-ecstore?/hotpath", + "rustfs-iam/hotpath", + "rustfs-keystone?/hotpath", + "rustfs-policy/hotpath", + "rustfs-rio?/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-tls-runtime?/hotpath", + "rustfs-trusted-proxies?/hotpath", + "rustfs-utils/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-ecstore?/hotpath-alloc", + "rustfs-iam/hotpath-alloc", + "rustfs-keystone?/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-rio?/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-tls-runtime?/hotpath-alloc", + "rustfs-trusted-proxies?/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-ecstore?/hotpath-cpu", + "rustfs-iam/hotpath-cpu", + "rustfs-keystone?/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-rio?/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-tls-runtime?/hotpath-cpu", + "rustfs-trusted-proxies?/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"] swift = [ "dep:rustfs-keystone", @@ -62,6 +111,7 @@ webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"] [dependencies] +hotpath.workspace = true # Core RustFS dependencies rustfs-iam = { workspace = true } rustfs-credentials = { workspace = true } diff --git a/crates/protos/Cargo.toml b/crates/protos/Cargo.toml index 3f57bbc6b..27d723a48 100644 --- a/crates/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -32,7 +32,38 @@ workspace = true name = "gproto" path = "src/main.rs" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-common.workspace = true rustfs-io-metrics.workspace = true rustfs-config.workspace = true diff --git a/crates/replication/Cargo.toml b/crates/replication/Cargo.toml index 5dc9a1108..e5554b922 100644 --- a/crates/replication/Cargo.toml +++ b/crates/replication/Cargo.toml @@ -25,7 +25,14 @@ keywords = ["replication", "storage", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-replication/latest/rustfs_replication/" +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } byteorder.workspace = true regex.workspace = true diff --git a/crates/rio-v2/Cargo.toml b/crates/rio-v2/Cargo.toml index b03ac2b88..b941ed7fc 100644 --- a/crates/rio-v2/Cargo.toml +++ b/crates/rio-v2/Cargo.toml @@ -28,7 +28,32 @@ documentation = "https://docs.rs/rustfs-rio-v2/latest/rustfs_rio_v2/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-rio/hotpath", + "rustfs-utils/hotpath", + "rustfs-filemeta/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-rio/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-rio/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true aes-gcm = { workspace = true, features = ["rand_core"] } bytes = { workspace = true, features = ["serde"] } chacha20poly1305.workspace = true diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index 4d0f3916e..72bb71af0 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -30,9 +30,32 @@ workspace = true [features] default = [] -hotpath = ["hotpath/hotpath", "hotpath/tokio", "hotpath/futures", "hotpath/reqwest-0-13"] -hotpath-alloc = ["hotpath/hotpath-alloc"] -hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] [dependencies] hotpath.workspace = true diff --git a/crates/s3-ops/Cargo.toml b/crates/s3-ops/Cargo.toml index 170f423c3..23dbd432a 100644 --- a/crates/s3-ops/Cargo.toml +++ b/crates/s3-ops/Cargo.toml @@ -27,7 +27,14 @@ categories = ["data-structures"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "rustfs-s3-types/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3-types/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3-types/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-s3-types = { workspace = true } [lib] diff --git a/crates/s3-types/Cargo.toml b/crates/s3-types/Cargo.toml index 0d25caf4b..76c408594 100644 --- a/crates/s3-types/Cargo.toml +++ b/crates/s3-types/Cargo.toml @@ -27,7 +27,14 @@ categories = ["data-structures"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/s3select-api/Cargo.toml b/crates/s3select-api/Cargo.toml index 78edb9daf..4e850b62b 100644 --- a/crates/s3select-api/Cargo.toml +++ b/crates/s3select-api/Cargo.toml @@ -28,7 +28,37 @@ documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/ [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "rustfs-common/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true metrics = { workspace = true } async-trait.workspace = true bytes = { workspace = true, features = ["serde"] } diff --git a/crates/s3select-query/Cargo.toml b/crates/s3select-query/Cargo.toml index 5ce13b9fd..619997edb 100644 --- a/crates/s3select-query/Cargo.toml +++ b/crates/s3select-query/Cargo.toml @@ -28,7 +28,20 @@ documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_qu [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "rustfs-s3select-api/hotpath", +] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3select-api/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3select-api/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-s3select-api = { workspace = true } async-recursion = { workspace = true } async-trait.workspace = true diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 33dde7715..127604853 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -29,7 +29,48 @@ documentation = "https://docs.rs/rustfs-scanner/latest/rustfs_scanner/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-data-usage/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-filemeta/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["server-config-model"] } rustfs-common = { workspace = true } rustfs-credentials = { workspace = true } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 91f9e1597..95c5fb194 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -2531,6 +2531,7 @@ where } #[instrument(skip_all)] +#[hotpath::measure] async fn run_data_scanner_cycle( ctx: &CancellationToken, storeapi: &Arc<ECStore>, diff --git a/crates/security-governance/Cargo.toml b/crates/security-governance/Cargo.toml index f71e1b032..da0b48470 100644 --- a/crates/security-governance/Cargo.toml +++ b/crates/security-governance/Cargo.toml @@ -30,5 +30,12 @@ doctest = false [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true thiserror = { workspace = true } diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 24cb667eb..b13c03275 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -25,7 +25,14 @@ keywords = ["digital-signature", "verification", "integrity", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "cryptography"] documentation = "https://docs.rs/rustfs-signer/latest/rustfs_signer/" +[features] +default = [] +hotpath = ["hotpath/hotpath", "rustfs-utils/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-utils/hotpath-cpu"] + [dependencies] +hotpath.workspace = true tracing.workspace = true bytes = { workspace = true, features = ["serde"] } http.workspace = true diff --git a/crates/storage-api/Cargo.toml b/crates/storage-api/Cargo.toml index f67d34331..74b519d3d 100644 --- a/crates/storage-api/Cargo.toml +++ b/crates/storage-api/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools", "filesystem"] [lib] doctest = false +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-filemeta/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-filemeta/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"] + [dependencies] +hotpath.workspace = true async-trait.workspace = true # Storage-facing replication contracts are isolated in src/replication.rs until # the underlying wire types can move without creating a replication/storage-api cycle. diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index bd0671a40..ae759eaf7 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -11,7 +11,41 @@ keywords = ["file-system", "notification", "target", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-extension-schema/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-extension-schema/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-extension-schema/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["notify", "audit", "server-config-model"] } rustfs-extension-schema = { workspace = true } rustfs-tls-runtime = { workspace = true } diff --git a/crates/targets/src/runtime/mod.rs b/crates/targets/src/runtime/mod.rs index 3a235e069..7999a33ce 100644 --- a/crates/targets/src/runtime/mod.rs +++ b/crates/targets/src/runtime/mod.rs @@ -522,6 +522,7 @@ fn snapshot_from_delivery(target_id: TargetID, delivery: TargetDeliverySnapshot) } } +#[hotpath::measure] pub async fn init_target_and_optionally_start_replay<E, F, G>( target: Box<dyn Target<E> + Send + Sync>, on_replay_start: F, @@ -557,6 +558,7 @@ where Some((shared, cancel)) } +#[hotpath::measure] pub(crate) async fn prepare_target<E>( target: Box<dyn Target<E> + Send + Sync>, cancellation: Option<&CancellationToken>, @@ -598,6 +600,7 @@ where type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>); +#[hotpath::measure] pub async fn activate_targets_with_replay<E, F, Fut>( targets: Vec<Box<dyn Target<E> + Send + Sync>>, mut activate_one: F, @@ -670,6 +673,7 @@ fn seed_interval_start(now: tokio::time::Instant, interval: Duration) -> tokio:: now.checked_sub(interval).unwrap_or(now) } +#[hotpath::measure] async fn stream_replay_worker<E>( store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send), target: SharedTarget<E>, @@ -805,6 +809,7 @@ async fn stream_replay_worker<E>( /// Returns `true` if a cancel signal was observed while processing (e.g. during /// retry backoff), so the caller can stop promptly instead of continuing to /// drain a store that a replacement worker may already own. +#[hotpath::measure] async fn process_replay_batch<E>( store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send), batch_keys: &mut Vec<Key>, diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index e39dc9d49..b7ac97f0b 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -85,6 +85,7 @@ fn classify_probe_error(err: &reqwest::Error) -> TargetHealthReason { TargetHealthReason::Unreachable } +#[hotpath::measure] async fn probe_health_url(client: &Client, health_check_url: &Url) -> TargetHealth { match tokio::time::timeout(WEBHOOK_HEALTH_TIMEOUT, client.head(health_check_url.as_str()).send()).await { Ok(Ok(_)) => TargetHealth::online(TargetHealthReason::Reachable), @@ -480,6 +481,7 @@ where build_queued_payload(event) } + #[hotpath::measure] async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { debug!( event = EVENT_WEBHOOK_DELIVERY_STATE, diff --git a/crates/test-utils/Cargo.toml b/crates/test-utils/Cargo.toml index 81d2a53c0..bf56d069a 100644 --- a/crates/test-utils/Cargo.toml +++ b/crates/test-utils/Cargo.toml @@ -25,7 +25,32 @@ keywords = ["testing", "storage", "rustfs", "Minio"] categories = ["development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-test-utils/latest/rustfs_test_utils/" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-ecstore/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-data-usage/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-ecstore = { workspace = true } rustfs-storage-api = { workspace = true } tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } diff --git a/crates/tls-runtime/Cargo.toml b/crates/tls-runtime/Cargo.toml index fba5f3fc2..fb8920547 100644 --- a/crates/tls-runtime/Cargo.toml +++ b/crates/tls-runtime/Cargo.toml @@ -27,7 +27,14 @@ categories = ["network-programming", "web-programming", "development-tools"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-common/hotpath", "rustfs-config/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-common/hotpath-alloc", "rustfs-config/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-common/hotpath-cpu", "rustfs-config/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-common.workspace = true rustfs-config.workspace = true arc-swap.workspace = true diff --git a/crates/trusted-proxies/Cargo.toml b/crates/trusted-proxies/Cargo.toml index 0a15e5a7f..1494d8a35 100644 --- a/crates/trusted-proxies/Cargo.toml +++ b/crates/trusted-proxies/Cargo.toml @@ -24,7 +24,20 @@ description = " RustFS Trusted Proxies module provides secure and efficient mana keywords = ["trusted-proxies", "network-security", "rustfs", "proxy-management"] categories = ["network-programming", "security", "web-programming"] +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-config/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-config/hotpath-cpu", "rustfs-utils/hotpath-cpu"] + [dependencies] +hotpath.workspace = true async-trait = { workspace = true } axum = { workspace = true } http = { workspace = true } diff --git a/crates/trusted-proxies/src/cloud/ranges.rs b/crates/trusted-proxies/src/cloud/ranges.rs index cd3b8e4e0..dbf814674 100644 --- a/crates/trusted-proxies/src/cloud/ranges.rs +++ b/crates/trusted-proxies/src/cloud/ranges.rs @@ -227,6 +227,7 @@ pub struct GoogleCloudIpRanges; impl GoogleCloudIpRanges { /// Fetches the latest Google Cloud IP ranges from their official source. + #[hotpath::measure] pub async fn fetch() -> Result<Vec<IpNetwork>, AppError> { let client = Client::builder() .timeout(Duration::from_secs(10)) diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 5ba901965..2e3c7c6d8 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -25,6 +25,7 @@ keywords = ["utilities", "hashing", "compression", "network", "rustfs"] categories = ["web-programming", "development-tools", "cryptography"] [dependencies] +hotpath.workspace = true base64-simd = { workspace = true, optional = true } blake2 = { workspace = true, optional = true } brotli = { workspace = true, optional = true } @@ -76,6 +77,9 @@ workspace = true [features] default = ["ip"] # features that are enabled by default +hotpath = ["hotpath/hotpath", "hotpath/tokio", "hotpath/futures", "hotpath/reqwest-0-13"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] ip = ["dep:local-ip-address"] # ip characteristics and their dependencies net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:hyper", "dep:tokio"] # network features with DNS resolver egress = ["ip", "dep:reqwest", "dep:tokio", "dep:url"] diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 8348d3189..54cdd9547 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -32,7 +32,14 @@ doctest = false name = "zip_benchmark" harness = false +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true async-compression = { workspace = true, features = [ "tokio", "bzip2", diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 53db5ca19..24a18a938 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -60,25 +60,137 @@ hotpath = [ "hotpath/crossbeam", "hotpath/parking_lot", "hotpath/reqwest-0-13", + "rustfs-audit/hotpath", + "rustfs-common/hotpath", + "rustfs-concurrency/hotpath", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-crypto/hotpath", + "rustfs-data-usage/hotpath", "rustfs-ecstore/hotpath", + "rustfs-extension-schema/hotpath", "rustfs-filemeta/hotpath", + "rustfs-heal/hotpath", + "rustfs-iam/hotpath", + "rustfs-io-core/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-keystone/hotpath", + "rustfs-kms/hotpath", + "rustfs-lock/hotpath", + "rustfs-log-analyzer/hotpath", + "rustfs-madmin/hotpath", + "rustfs-notify/hotpath", + "rustfs-object-capacity/hotpath", + "rustfs-object-data-cache/hotpath", + "rustfs-obs/hotpath", "rustfs-policy/hotpath", + "rustfs-protocols/hotpath", + "rustfs-protos/hotpath", "rustfs-rio/hotpath", + "rustfs-s3-ops/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-s3select-api/hotpath", + "rustfs-s3select-query/hotpath", + "rustfs-scanner/hotpath", + "rustfs-security-governance/hotpath", + "rustfs-signer/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-targets/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-trusted-proxies/hotpath", + "rustfs-utils/hotpath", + "rustfs-zip/hotpath", + "rustfs-test-utils/hotpath", ] hotpath-alloc = [ + "hotpath", "hotpath/hotpath-alloc", + "rustfs-audit/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-concurrency/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", "rustfs-ecstore/hotpath-alloc", + "rustfs-extension-schema/hotpath-alloc", "rustfs-filemeta/hotpath-alloc", + "rustfs-heal/hotpath-alloc", + "rustfs-iam/hotpath-alloc", + "rustfs-io-core/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-keystone/hotpath-alloc", + "rustfs-kms/hotpath-alloc", + "rustfs-lock/hotpath-alloc", + "rustfs-log-analyzer/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-notify/hotpath-alloc", + "rustfs-object-capacity/hotpath-alloc", + "rustfs-object-data-cache/hotpath-alloc", + "rustfs-obs/hotpath-alloc", "rustfs-policy/hotpath-alloc", + "rustfs-protocols/hotpath-alloc", + "rustfs-protos/hotpath-alloc", "rustfs-rio/hotpath-alloc", + "rustfs-s3-ops/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-s3select-api/hotpath-alloc", + "rustfs-s3select-query/hotpath-alloc", + "rustfs-scanner/hotpath-alloc", + "rustfs-security-governance/hotpath-alloc", + "rustfs-signer/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-targets/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-trusted-proxies/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-zip/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", ] hotpath-cpu = [ "hotpath", "hotpath/hotpath-cpu", + "rustfs-audit/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-concurrency/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", "rustfs-ecstore/hotpath-cpu", + "rustfs-extension-schema/hotpath-cpu", "rustfs-filemeta/hotpath-cpu", + "rustfs-heal/hotpath-cpu", + "rustfs-iam/hotpath-cpu", + "rustfs-io-core/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-keystone/hotpath-cpu", + "rustfs-kms/hotpath-cpu", + "rustfs-lock/hotpath-cpu", + "rustfs-log-analyzer/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-notify/hotpath-cpu", + "rustfs-object-capacity/hotpath-cpu", + "rustfs-object-data-cache/hotpath-cpu", + "rustfs-obs/hotpath-cpu", "rustfs-policy/hotpath-cpu", + "rustfs-protocols/hotpath-cpu", + "rustfs-protos/hotpath-cpu", "rustfs-rio/hotpath-cpu", + "rustfs-s3-ops/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-s3select-api/hotpath-cpu", + "rustfs-s3select-query/hotpath-cpu", + "rustfs-scanner/hotpath-cpu", + "rustfs-security-governance/hotpath-cpu", + "rustfs-signer/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-targets/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-trusted-proxies/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-zip/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", ] [lints] From 76b3c085b58000ff2cc544a80ff7f4cd747b6831 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 16:31:21 +0800 Subject: [PATCH 38/52] refactor(kms): fold the KmsClient layer into KmsBackend and complete lifecycle overrides (#5501) --- crates/kms/src/backends/contract_tests.rs | 81 +++-- crates/kms/src/backends/local.rs | 63 ++-- crates/kms/src/backends/mod.rs | 183 ----------- ...tests__vault_kv2_backend_capabilities.snap | 4 +- crates/kms/src/backends/static_kms.rs | 219 +++++-------- crates/kms/src/backends/vault.rs | 305 ++++++++++++------ crates/kms/src/backends/vault_transit.rs | 137 +++++--- crates/kms/src/deletion_worker.rs | 1 - 8 files changed, 470 insertions(+), 523 deletions(-) diff --git a/crates/kms/src/backends/contract_tests.rs b/crates/kms/src/backends/contract_tests.rs index a436a17e2..52d9beb18 100644 --- a/crates/kms/src/backends/contract_tests.rs +++ b/crates/kms/src/backends/contract_tests.rs @@ -27,11 +27,11 @@ //! server, so they are `#[ignore]`d in CI. Static is covered by its own //! stateless contract below. +use super::KmsBackend; use super::local::LocalKmsBackend; use super::static_kms::StaticKmsBackend; use super::vault::VaultKmsBackend; use super::vault_transit::VaultTransitKmsBackend; -use super::{KmsBackend, KmsClient}; use crate::config::KmsConfig; use crate::error::{KmsError, Result}; use crate::manager::KmsManager; @@ -46,6 +46,24 @@ use rand::RngExt as _; use std::collections::HashMap; use std::sync::Arc; +fn expect_unsupported<T: std::fmt::Debug>(result: Result<T>) { + match result { + Err(KmsError::UnsupportedCapability { .. }) => {} + other => panic!("expected UnsupportedCapability, got {other:?}"), + } +} + +/// Rotation while not Enabled: backends with rotation support must reject it +/// through the state machine; backends without it report the capability gap. +async fn expect_rotate_rejected(backend: &dyn KmsBackend, key_id: &str) { + let result = backend.rotate_key(key_id).await; + if backend.capabilities().rotate { + expect_invalid_key_state(result, ""); + } else { + expect_unsupported(result); + } +} + fn expect_invalid_key_state<T: std::fmt::Debug>(result: Result<T>, expected_fragment: &str) { match result { Err(KmsError::InvalidOperation { message }) => assert!( @@ -117,11 +135,9 @@ async fn assert_key_state(backend: &dyn KmsBackend, key_id: &str, expected: KeyS assert_eq!(described.key_metadata.key_state, expected, "unexpected state for key {key_id}"); } -/// Drives one freshly created (Enabled) key through the full state matrix. -/// -/// `backend` is the product surface; `client` drives the lifecycle -/// transitions not yet exposed through `KmsBackend`. -async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn KmsClient, key_id: &str) { +/// Drives one freshly created (Enabled) key through the full state matrix, +/// entirely through the `KmsBackend` product surface. +async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) { // Enabled: cryptographic use is allowed. Keep an envelope around to prove // decryption keeps working in later states. let data_key = backend @@ -134,16 +150,13 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn Km .expect("Enabled key must encrypt"); // Enabled -> Disabled. - client - .disable_key(key_id, None) - .await - .expect("disable from Enabled must succeed"); + backend.disable_key(key_id).await.expect("disable from Enabled must succeed"); assert_key_state(backend, key_id, KeyState::Disabled).await; // Disabled: new cryptographic use and rotation are rejected... expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "disabled"); expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "disabled"); - expect_invalid_key_state(client.rotate_key(key_id, None).await, ""); + expect_rotate_rejected(backend, key_id).await; // ...but decryption of existing data keeps working (explicit AWS deviation)... let decrypted = backend .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) @@ -151,17 +164,14 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn Km .expect("decrypt with a disabled key must keep working"); assert_eq!(decrypted.plaintext, data_key.plaintext_key, "decrypt must recover the original data key"); // ...disable stays idempotent, cancel has nothing to cancel, and enable recovers. - client.disable_key(key_id, None).await.expect("disable must be idempotent"); + backend.disable_key(key_id).await.expect("disable must be idempotent"); expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion"); - client - .enable_key(key_id, None) - .await - .expect("enable from Disabled must succeed"); + backend.enable_key(key_id).await.expect("enable from Disabled must succeed"); assert_key_state(backend, key_id, KeyState::Enabled).await; // Disabled keys may still be scheduled for deletion. - client - .disable_key(key_id, None) + backend + .disable_key(key_id) .await .expect("disable before scheduling must succeed"); backend @@ -173,10 +183,9 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn Km // PendingDeletion: everything except decryption and cancellation is rejected. expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "pending deletion"); expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "pending deletion"); - expect_invalid_key_state(client.enable_key(key_id, None).await, "pending deletion"); - expect_invalid_key_state(client.disable_key(key_id, None).await, "pending deletion"); - expect_invalid_key_state(client.rotate_key(key_id, None).await, ""); - expect_invalid_key_state(client.schedule_key_deletion(key_id, 7, None).await, "pending deletion"); + expect_invalid_key_state(backend.enable_key(key_id).await, "pending deletion"); + expect_invalid_key_state(backend.disable_key(key_id).await, "pending deletion"); + expect_rotate_rejected(backend, key_id).await; expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "pending deletion"); let decrypted = backend .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) @@ -215,7 +224,7 @@ async fn local_fixture() -> (tempfile::TempDir, KmsConfig, LocalKmsBackend, Stri #[tokio::test] async fn local_backend_state_machine_contract() { let (_temp_dir, _config, backend, key_id) = local_fixture().await; - assert_state_machine_contract(&backend, backend.lifecycle_client(), &key_id).await; + assert_state_machine_contract(&backend, &key_id).await; } /// SSE-shaped regression: disabling a key must not break decryption of data @@ -256,10 +265,7 @@ async fn static_backend_stateless_contract() { rand::rng().fill(&mut raw_key[..]); let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key)); let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build"); - // StaticKmsBackend implements both traits with overlapping method names, - // so pin each surface once instead of qualifying every call. let backend: &dyn KmsBackend = &static_backend; - let client: &dyn KmsClient = &static_backend; let data_key = backend .generate_data_key(generate_request(key_id)) @@ -275,9 +281,11 @@ async fn static_backend_stateless_contract() { expect_invalid_key_state(backend.create_key(create_request("another-key".to_string())).await, "read-only"); expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "read-only"); expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "read-only"); - expect_invalid_key_state(client.disable_key(key_id, None).await, "read-only"); - expect_invalid_key_state(client.schedule_key_deletion(key_id, 7, None).await, "read-only"); - expect_invalid_key_state(client.rotate_key(key_id, None).await, "read-only"); + // Enable/disable and rotation are capability gaps at the product + // surface, not state-machine rejections. + expect_unsupported(backend.enable_key(key_id).await); + expect_unsupported(backend.disable_key(key_id).await); + expect_unsupported(backend.rotate_key(key_id).await); } fn vault_dev_config(constructor: fn(url::Url, String) -> KmsConfig) -> KmsConfig { @@ -298,7 +306,15 @@ async fn vault_kv2_backend_state_machine_contract() { .await .expect("key should be created"); - assert_state_machine_contract(&backend, backend.lifecycle_client(), &created.key_id).await; + assert_state_machine_contract(&backend, &created.key_id).await; + + // KV2 additionally supports version-retaining rotation, which must only + // work while the key is Enabled (the shared matrix covered the + // rejections). + backend + .rotate_key(&created.key_id) + .await + .expect("rotation of an Enabled KV2 key must succeed"); // Cleanup: leave the key pending deletion so repeated runs stay tidy. let _ = backend.delete_key(schedule_request(&created.key_id)).await; @@ -316,13 +332,12 @@ async fn vault_transit_backend_state_machine_contract() { .await .expect("key should be created"); - assert_state_machine_contract(&backend, backend.lifecycle_client(), &created.key_id).await; + assert_state_machine_contract(&backend, &created.key_id).await; // Transit additionally supports rotation, which must only work while the // key is Enabled (the shared matrix already covered the rejections). backend - .lifecycle_client() - .rotate_key(&created.key_id, None) + .rotate_key(&created.key_id) .await .expect("rotation of an Enabled transit key must succeed"); diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 2d410f6d7..e1e347cca 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -14,9 +14,7 @@ //! Local file-based KMS backend implementation -use crate::backends::{ - BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_status_permits, -}; +use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits}; use crate::config::KmsConfig; use crate::config::LocalConfig; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -937,9 +935,12 @@ impl LocalKmsClient { } } -#[async_trait] -impl KmsClient for LocalKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKeyInfo> { +impl LocalKmsClient { + pub(crate) async fn generate_data_key( + &self, + request: &GenerateKeyRequest, + context: Option<&OperationContext>, + ) -> Result<DataKeyInfo> { debug!("Generating data key for master key: {}", request.master_key_id); let key_info = self.describe_key(&request.master_key_id, context).await?; @@ -980,7 +981,7 @@ impl KmsClient for LocalKmsClient { Ok(data_key) } - async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result<EncryptResponse> { + pub(crate) async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result<EncryptResponse> { debug!("Encrypting data with key: {}", request.key_id); // Verify key exists and its state allows encryption @@ -997,7 +998,7 @@ impl KmsClient for LocalKmsClient { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { + pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { debug!("Decrypting data"); // Parse the data key envelope from ciphertext @@ -1031,7 +1032,14 @@ impl KmsClient for LocalKmsClient { Ok(plaintext) } - async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn create_key( + &self, + key_id: &str, + algorithm: &str, + context: Option<&OperationContext>, + ) -> Result<MasterKeyInfo> { debug!("Creating master key: {}", key_id); // Check if key already exists @@ -1060,14 +1068,18 @@ impl KmsClient for LocalKmsClient { Ok(master_key) } - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { + pub(crate) async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { debug!("Describing key: {}", key_id); let master_key = self.load_master_key(key_id).await?; Ok(master_key.into()) } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { + pub(crate) async fn list_keys( + &self, + request: &ListKeysRequest, + _context: Option<&OperationContext>, + ) -> Result<ListKeysResponse> { debug!("Listing keys"); let mut keys = Vec::new(); @@ -1111,7 +1123,7 @@ impl KmsClient for LocalKmsClient { }) } - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; @@ -1129,7 +1141,7 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; @@ -1146,7 +1158,9 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn schedule_key_deletion( + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn schedule_key_deletion( &self, key_id: &str, pending_window_days: u32, @@ -1170,7 +1184,9 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Canceling deletion for key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; @@ -1190,7 +1206,9 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { if !fs::try_exists(self.master_key_path(key_id)?).await? { return Err(KmsError::key_not_found(key_id)); } @@ -1199,7 +1217,7 @@ impl KmsClient for LocalKmsClient { )) } - async fn health_check(&self) -> Result<()> { + pub(crate) async fn health_check(&self) -> Result<()> { // Check if key directory is accessible if !self.config.key_dir.exists() { return Err(KmsError::backend_error("Key directory does not exist")); @@ -1210,17 +1228,6 @@ impl KmsClient for LocalKmsClient { Ok(()) } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new( - "local".to_string(), - env!("CARGO_PKG_VERSION").to_string(), - self.config.key_dir.to_string_lossy().to_string(), - true, // We'll assume healthy for now - ) - .with_metadata("key_dir".to_string(), self.config.key_dir.to_string_lossy().to_string()) - .with_metadata("encrypted_at_rest".to_string(), self.master_cipher.is_some().to_string()) - } } /// LocalKmsBackend wraps LocalKmsClient and implements the KmsBackend trait diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index a74ed9a38..bf9a54e55 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -19,7 +19,6 @@ use crate::types::*; use async_trait::async_trait; use jiff::Zoned; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; #[cfg(test)] mod contract_tests; @@ -99,136 +98,6 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat ensure_key_state_permits(key_id, &state, operation) } -/// Abstract KMS client interface that all backends must implement -#[async_trait] -pub trait KmsClient: Send + Sync { - /// Generate a new data encryption key (DEK) - /// - /// Creates a new data key using the specified master key. The returned DataKey - /// contains both the plaintext and encrypted versions of the key. - /// - /// # Arguments - /// * `request` - The key generation request - /// * `context` - Optional operation context for auditing - /// - /// # Returns - /// Returns a DataKey containing both plaintext and encrypted key material - async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKeyInfo>; - - /// Encrypt data directly using a master key - /// - /// Encrypts the provided plaintext using the specified master key. - /// This is different from generate_data_key as it encrypts user data directly. - /// - /// # Arguments - /// * `request` - The encryption request containing plaintext and key ID - /// * `context` - Optional operation context for auditing - async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result<EncryptResponse>; - - /// Decrypt data using a master key - /// - /// Decrypts the provided ciphertext. The KMS automatically determines - /// which key was used for encryption based on the ciphertext metadata. - /// - /// # Arguments - /// * `request` - The decryption request containing ciphertext - /// * `context` - Optional operation context for auditing - async fn decrypt(&self, request: &DecryptRequest, context: Option<&OperationContext>) -> Result<Vec<u8>>; - - /// Create a new master key - /// - /// Creates a new master key in the KMS with the specified ID. - /// Returns an error if a key with the same ID already exists. - /// - /// # Arguments - /// * `key_id` - Unique identifier for the new key - /// * `algorithm` - Key algorithm (e.g., "AES_256") - /// * `context` - Optional operation context for auditing - async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo>; - - /// Get information about a specific key - /// - /// Returns metadata and information about the specified key. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn describe_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<KeyInfo>; - - /// List available keys - /// - /// Returns a paginated list of keys available in the KMS. - /// - /// # Arguments - /// * `request` - List request parameters (pagination, filters) - /// * `context` - Optional operation context for auditing - async fn list_keys(&self, request: &ListKeysRequest, context: Option<&OperationContext>) -> Result<ListKeysResponse>; - - /// Enable a key - /// - /// Enables a previously disabled key, allowing it to be used for cryptographic operations. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn enable_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>; - - /// Disable a key - /// - /// Disables a key, preventing it from being used for new cryptographic operations. - /// Existing encrypted data can still be decrypted. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn disable_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>; - - /// Schedule key deletion - /// - /// Schedules a key for deletion after a specified number of days. - /// This allows for a grace period to recover the key if needed. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `pending_window_days` - Number of days before actual deletion - /// * `context` - Optional operation context for auditing - async fn schedule_key_deletion( - &self, - key_id: &str, - pending_window_days: u32, - context: Option<&OperationContext>, - ) -> Result<()>; - - /// Cancel key deletion - /// - /// Cancels a previously scheduled key deletion. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn cancel_key_deletion(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>; - - /// Rotate a key - /// - /// Creates a new version of the specified key. Previous versions remain - /// available for decryption but new operations will use the new version. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn rotate_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo>; - - /// Health check - /// - /// Performs a health check on the KMS backend to ensure it's operational. - async fn health_check(&self) -> Result<()>; - - /// Get backend information - /// - /// Returns information about the KMS backend (type, version, etc.). - fn backend_info(&self) -> BackendInfo; -} - /// Simplified KMS backend interface for manager #[async_trait] pub trait KmsBackend: Send + Sync { @@ -327,58 +196,6 @@ pub enum ExpiredKeyRemoval { NotExpired, } -/// Information about a KMS backend -#[derive(Debug, Clone)] -pub struct BackendInfo { - /// Backend type name (e.g., "local", "vault") - pub backend_type: String, - /// Backend version - pub version: String, - /// Backend endpoint or location - pub endpoint: String, - /// Whether the backend is currently healthy - pub healthy: bool, - /// Additional metadata about the backend - pub metadata: HashMap<String, String>, -} - -impl BackendInfo { - /// Create a new backend info - /// - /// # Arguments - /// * `backend_type` - The type of the backend - /// * `version` - The version of the backend - /// * `endpoint` - The endpoint or location of the backend - /// * `healthy` - Whether the backend is healthy - /// - /// # Returns - /// A new BackendInfo instance - /// - pub fn new(backend_type: String, version: String, endpoint: String, healthy: bool) -> Self { - Self { - backend_type, - version, - endpoint, - healthy, - metadata: HashMap::new(), - } - } - - /// Add metadata to the backend info - /// - /// # Arguments - /// * `key` - Metadata key - /// * `value` - Metadata value - /// - /// # Returns - /// Updated BackendInfo instance - /// - pub fn with_metadata(mut self, key: String, value: String) -> Self { - self.metadata.insert(key, value); - self - } -} - /// Set of operations a KMS backend supports. /// /// Reported by [`KmsBackend::capabilities`] so callers (manager, admin API) diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap index 57be08b27..2c0bd7fce 100644 --- a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap @@ -8,7 +8,7 @@ expression: capabilities_snapshot(backend.capabilities()) "encrypt": true, "generate_data_key": true, "physical_delete": true, - "rotate": false, + "rotate": true, "schedule_deletion": true, - "versioning": false + "versioning": true } diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 73b988eeb..f3fc065dc 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -21,7 +21,7 @@ //! //! encrypted_data(plaintext_len+16) || nonce (12 bytes) -use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{BackendCapabilities, KmsBackend}; use crate::config::{BackendConfig, KmsConfig}; use crate::encryption::DataKeyEnvelope; use crate::error::{KmsError, Result}; @@ -98,9 +98,10 @@ impl StaticKmsBackend { } } -#[async_trait] -impl KmsClient for StaticKmsBackend { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> { +impl StaticKmsBackend { + /// Generate a fresh data key and wrap it in the standard KMS envelope, + /// authenticated against the canonical encryption context. + pub(crate) fn generate_data_key_envelope(&self, request: &GenerateKeyRequest) -> Result<DataKeyInfo> { if request.master_key_id != self.key_id { return Err(KmsError::key_not_found(&request.master_key_id)); } @@ -151,7 +152,8 @@ impl KmsClient for StaticKmsBackend { )) } - async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { + /// Encrypt caller-provided plaintext into the standard KMS envelope. + pub(crate) fn encrypt_to_envelope(&self, request: &EncryptRequest) -> Result<EncryptResponse> { if request.key_id != self.key_id { return Err(KmsError::key_not_found(&request.key_id)); } @@ -196,7 +198,8 @@ impl KmsClient for StaticKmsBackend { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { + /// Open a KMS envelope produced by this backend. + pub(crate) fn decrypt_envelope(&self, request: &DecryptRequest) -> Result<Vec<u8>> { let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext) .map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?; if envelope.master_key_id != self.key_id { @@ -235,14 +238,8 @@ impl KmsClient for StaticKmsBackend { Ok(plaintext) } - async fn create_key(&self, key_id: &str, _algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { - if key_id == self.key_id { - return Err(KmsError::key_already_exists(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot create new keys")) - } - - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { + /// Describe the single configured key. + pub(crate) fn configured_key_info(&self, key_id: &str) -> Result<KeyInfo> { if key_id != self.key_id { return Err(KmsError::key_not_found(key_id)); } @@ -263,7 +260,8 @@ impl KmsClient for StaticKmsBackend { }) } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { + /// List the single configured key, honouring the pagination marker. + pub(crate) fn list_configured_key(&self, request: &ListKeysRequest) -> Result<ListKeysResponse> { let key_info = KeyInfo { key_id: self.key_id.clone(), description: Some("Static single-key KMS backend".to_string()), @@ -295,57 +293,6 @@ impl KmsClient for StaticKmsBackend { truncated: false, }) } - - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - // Static KMS key is always enabled - Ok(()) - } - - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot disable keys")) - } - - async fn schedule_key_deletion( - &self, - key_id: &str, - _pending_window_days: u32, - _context: Option<&OperationContext>, - ) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot schedule key deletion")) - } - - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot cancel key deletion")) - } - - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot rotate keys")) - } - - async fn health_check(&self) -> Result<()> { - // Static KMS is always healthy if it was successfully initialized - Ok(()) - } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new("static".to_string(), env!("CARGO_PKG_VERSION").to_string(), "local".to_string(), true) - .with_metadata("key_id".to_string(), self.key_id.clone()) - } } #[async_trait] @@ -359,12 +306,12 @@ impl KmsBackend for StaticKmsBackend { } async fn encrypt(&self, request: EncryptRequest) -> Result<EncryptResponse> { - <Self as KmsClient>::encrypt(self, &request, None).await + self.encrypt_to_envelope(&request) } async fn decrypt(&self, request: DecryptRequest) -> Result<DecryptResponse> { let key_id = self.key_id.clone(); - let plaintext = <Self as KmsClient>::decrypt(self, &request, None).await?; + let plaintext = self.decrypt_envelope(&request)?; Ok(DecryptResponse { plaintext, key_id, @@ -380,7 +327,7 @@ impl KmsBackend for StaticKmsBackend { encryption_context: request.encryption_context, grant_tokens: Vec::new(), }; - let data_key = <Self as KmsClient>::generate_data_key(self, &gen_req, None).await?; + let data_key = self.generate_data_key_envelope(&gen_req)?; let plaintext_key = data_key .plaintext @@ -395,7 +342,7 @@ impl KmsBackend for StaticKmsBackend { } async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> { - let key_info = <Self as KmsClient>::describe_key(self, &request.key_id, None).await?; + let key_info = self.configured_key_info(&request.key_id)?; let key_metadata = KeyMetadata { key_id: key_info.key_id.clone(), key_state: if key_info.status == KeyStatus::Active { @@ -415,7 +362,7 @@ impl KmsBackend for StaticKmsBackend { } async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> { - <Self as KmsClient>::list_keys(self, &request, None).await + self.list_configured_key(&request) } async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> { @@ -446,7 +393,7 @@ impl KmsBackend for StaticKmsBackend { #[cfg(test)] mod tests { use super::*; - use crate::backends::{KmsBackend as KmsBackendTrait, KmsClient}; + use crate::backends::KmsBackend as KmsBackendTrait; use crate::config::{BackendConfig, KmsBackend, StaticConfig}; use crate::encryption::is_data_key_envelope; use base64::Engine as _; @@ -491,8 +438,8 @@ mod tests { // Generate data key let request = GenerateKeyRequest::new(key_id.clone(), "AES_256".to_string()) .with_context("bucket".to_string(), "test-bucket".to_string()); - let data_key = KmsClient::generate_data_key(&backend, &request, None) - .await + let data_key = backend + .generate_data_key_envelope(&request) .expect("Failed to generate data key"); assert_eq!(data_key.key_id, key_id); @@ -509,9 +456,7 @@ mod tests { // Decrypt the data key let decrypt_request = DecryptRequest::new(data_key.ciphertext.clone()).with_context("bucket".to_string(), "test-bucket".to_string()); - let decrypted = KmsClient::decrypt(&backend, &decrypt_request, None) - .await - .expect("Failed to decrypt"); + let decrypted = backend.decrypt_envelope(&decrypt_request).expect("Failed to decrypt"); assert_eq!(decrypted.as_slice(), data_key.plaintext.as_deref().expect("plaintext should exist")); } @@ -523,8 +468,8 @@ mod tests { .with_context("bucket".to_string(), "source-bucket".to_string()) .with_context("object".to_string(), "source-object".to_string()); - let data_key = KmsClient::generate_data_key(&backend, &request, None) - .await + let data_key = backend + .generate_data_key_envelope(&request) .expect("generate static KMS data key"); assert!( @@ -568,8 +513,8 @@ mod tests { let (backend, key_id, _key) = create_test_backend().await; let request = GenerateKeyRequest::new(key_id, "AES_256".to_string()) .with_context("bucket".to_string(), "source-bucket".to_string()); - let generated = KmsClient::generate_data_key(&backend, &request, None) - .await + let generated = backend + .generate_data_key_envelope(&request) .expect("generate context-bound data key"); let mut envelope: DataKeyEnvelope = serde_json::from_slice(&generated.ciphertext).expect("parse static KMS envelope"); envelope @@ -578,8 +523,8 @@ mod tests { let decrypt_request = DecryptRequest::new(serde_json::to_vec(&envelope).expect("serialize tampered envelope")) .with_context("bucket".to_string(), "different-bucket".to_string()); - let error = KmsClient::decrypt(&backend, &decrypt_request, None) - .await + let error = backend + .decrypt_envelope(&decrypt_request) .expect_err("tampering with authenticated envelope context must fail"); assert!(matches!(error, KmsError::CryptographicError { .. })); @@ -590,7 +535,7 @@ mod tests { let (backend, _key_id, _key) = create_test_backend().await; let request = GenerateKeyRequest::new("wrong-key-id".to_string(), "AES_256".to_string()); - let result = KmsClient::generate_data_key(&backend, &request, None).await; + let result = backend.generate_data_key_envelope(&request); assert!(result.is_err()); assert!(result.expect_err("should be Err").to_string().contains("wrong-key-id")); } @@ -602,7 +547,7 @@ mod tests { // Ciphertext too short let short = vec![0u8; 10]; let request = DecryptRequest::new(short); - let result = KmsClient::decrypt(&backend, &request, None).await; + let result = backend.decrypt_envelope(&request); assert!(result.is_err()); } @@ -612,9 +557,7 @@ mod tests { // Generate a valid ciphertext first let gen_request = GenerateKeyRequest::new(key_id, "AES_256".to_string()); - let data_key = KmsClient::generate_data_key(&backend, &gen_request, None) - .await - .expect("generate"); + let data_key = backend.generate_data_key_envelope(&gen_request).expect("generate"); // Tamper with the ciphertext (flip a bit in the encrypted portion) let mut tampered = data_key.ciphertext.clone(); @@ -623,7 +566,7 @@ mod tests { } let request = DecryptRequest::new(tampered); - let result = KmsClient::decrypt(&backend, &request, None).await; + let result = backend.decrypt_envelope(&request); assert!(result.is_err()); } @@ -632,7 +575,14 @@ mod tests { let (backend, key_id, _key) = create_test_backend().await; // Creating the pre-configured key should return KeyAlreadyExists - let result = KmsClient::create_key(&backend, &key_id, "AES_256", None).await; + let result = KmsBackendTrait::create_key( + &backend, + CreateKeyRequest { + key_name: Some(key_id.clone()), + ..Default::default() + }, + ) + .await; assert!(result.is_err()); assert!(result.expect_err("should be Err").to_string().contains("already exists")); } @@ -642,7 +592,14 @@ mod tests { let (backend, _key_id, _key) = create_test_backend().await; // Creating any other key should return invalid operation (read-only) - let result = KmsClient::create_key(&backend, "other-key", "AES_256", None).await; + let result = KmsBackendTrait::create_key( + &backend, + CreateKeyRequest { + key_name: Some("other-key".to_string()), + ..Default::default() + }, + ) + .await; assert!(result.is_err()); let err_msg = result.expect_err("should be Err").to_string(); assert!(err_msg.contains("read-only") || err_msg.contains("cannot create")); @@ -652,15 +609,13 @@ mod tests { async fn test_describe_key() { let (backend, key_id, _key) = create_test_backend().await; - let key_info = KmsClient::describe_key(&backend, &key_id, None) - .await - .expect("describe_key should succeed"); + let key_info = backend.configured_key_info(&key_id).expect("describe_key should succeed"); assert_eq!(key_info.key_id, key_id); assert_eq!(key_info.status, KeyStatus::Active); assert_eq!(key_info.algorithm, "AES_256"); // Wrong key ID - let result = KmsClient::describe_key(&backend, "nonexistent", None).await; + let result = backend.configured_key_info("nonexistent"); assert!(result.is_err()); } @@ -668,8 +623,8 @@ mod tests { async fn test_list_keys() { let (backend, key_id, _key) = create_test_backend().await; - let response = KmsClient::list_keys(&backend, &ListKeysRequest::default(), None) - .await + let response = backend + .list_configured_key(&ListKeysRequest::default()) .expect("list_keys should succeed"); assert_eq!(response.keys.len(), 1); assert_eq!(response.keys[0].key_id, key_id); @@ -677,61 +632,45 @@ mod tests { } #[tokio::test] - async fn test_disable_key_returns_error() { + async fn lifecycle_mutations_are_unsupported_at_the_product_surface() { let (backend, key_id, _key) = create_test_backend().await; - let result = KmsClient::disable_key(&backend, &key_id, None).await; - assert!(result.is_err()); - assert!(result.expect_err("should be Err").to_string().contains("read-only")); - } - - #[tokio::test] - async fn test_enable_key_is_noop() { - let (backend, key_id, _key) = create_test_backend().await; - - // Enable should succeed (no-op for static KMS) - KmsClient::enable_key(&backend, &key_id, None) - .await - .expect("enable_key should be no-op"); - - // Wrong key should still fail - let result = KmsClient::enable_key(&backend, "wrong", None).await; - assert!(result.is_err()); + // The static backend advertises no enable/disable or rotation + // capability, so the shared KmsBackend defaults reject all three. + for result in [ + KmsBackendTrait::enable_key(&backend, &key_id).await, + KmsBackendTrait::disable_key(&backend, &key_id).await, + KmsBackendTrait::rotate_key(&backend, &key_id).await, + ] { + let error = result.expect_err("static lifecycle mutations must be rejected"); + assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "got {error:?}"); + } } #[tokio::test] async fn test_delete_key_returns_error() { let (backend, key_id, _key) = create_test_backend().await; - let result = KmsClient::schedule_key_deletion(&backend, &key_id, 7, None).await; + let result = KmsBackendTrait::delete_key( + &backend, + DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + }, + ) + .await; assert!(result.is_err()); assert!(result.expect_err("should be Err").to_string().contains("read-only")); } - #[tokio::test] - async fn test_rotate_key_returns_error() { - let (backend, key_id, _key) = create_test_backend().await; - - let result = KmsClient::rotate_key(&backend, &key_id, None).await; - assert!(result.is_err()); - } - #[tokio::test] async fn test_health_check() { let (backend, _key_id, _key) = create_test_backend().await; - KmsClient::health_check(&backend).await.expect("health_check should succeed"); - } - - #[tokio::test] - async fn test_backend_info() { - let (backend, key_id, _key) = create_test_backend().await; - - let info = KmsClient::backend_info(&backend); - assert_eq!(info.backend_type, "static"); - assert_eq!(info.endpoint, "local"); - assert!(info.healthy); - assert_eq!(info.metadata.get("key_id"), Some(&key_id)); + KmsBackendTrait::health_check(&backend) + .await + .expect("health_check should succeed"); } #[tokio::test] @@ -740,17 +679,13 @@ mod tests { let plaintext = b"Hello, static KMS world!"; let enc_request = EncryptRequest::new(key_id.clone(), plaintext.to_vec()); - let enc_response = KmsClient::encrypt(&backend, &enc_request, None) - .await - .expect("encrypt should succeed"); + let enc_response = backend.encrypt_to_envelope(&enc_request).expect("encrypt should succeed"); assert_eq!(enc_response.key_id, key_id); assert!(!enc_response.ciphertext.is_empty()); let dec_request = DecryptRequest::new(enc_response.ciphertext); - let decrypted = KmsClient::decrypt(&backend, &dec_request, None) - .await - .expect("decrypt should succeed"); + let decrypted = backend.decrypt_envelope(&dec_request).expect("decrypt should succeed"); assert_eq!(decrypted, plaintext); } diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index ebc417833..24d9c9c39 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -19,8 +19,7 @@ use crate::backends::vault_credentials::{ token_source_for, }; use crate::backends::{ - BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits, - ensure_key_status_permits, + BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits, }; use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -42,7 +41,6 @@ use vaultrs::{api::kv2::requests::SetSecretRequestOptions, error::ClientError, k /// Vault KMS client implementation pub struct VaultKmsClient { credentials: Arc<VaultCredentialProvider>, - config: VaultConfig, /// Mount path for the KV engine (typically "kv" or "secret") kv_mount: String, /// Path prefix for storing keys @@ -200,7 +198,6 @@ impl VaultKmsClient { credentials, kv_mount: config.kv_mount.clone(), key_path_prefix: config.key_path_prefix.clone(), - config, dek_crypto: AesDekCrypto::new(), retry: RetryPolicy::from_config(kms_config), cancel: CancellationToken::new(), @@ -254,13 +251,6 @@ impl VaultKmsClient { Ok(general_purpose::STANDARD.encode(key_material)) } - /// Decode key material from KV2 storage (plain Base64, see `encrypt_key_material`). - async fn decrypt_key_material(&self, encrypted_material: &str) -> Result<Vec<u8>> { - general_purpose::STANDARD - .decode(encrypted_material) - .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string())) - } - /// Read the immutable material record of one key version. /// /// A missing record fails closed with [`KmsError::KeyVersionNotFound`]; falling @@ -589,9 +579,12 @@ impl VaultKmsClient { } } -#[async_trait] -impl KmsClient for VaultKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> { +impl VaultKmsClient { + pub(crate) async fn generate_data_key( + &self, + request: &GenerateKeyRequest, + _context: Option<&OperationContext>, + ) -> Result<DataKeyInfo> { debug!("Generating data key for master key: {}", request.master_key_id); let key_data = self.get_key_data(&request.master_key_id).await?; @@ -632,20 +625,32 @@ impl KmsClient for VaultKmsClient { Ok(data_key) } - async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { + pub(crate) async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { debug!("Encrypting data with key: {}", request.key_id); - // Get the master key and verify its state allows encryption + // Single read of the key record: the material we wrap with and the + // version stamped into the envelope must come from the same snapshot + // (see generate_data_key). let key_data = self.get_key_data(&request.key_id).await?; ensure_key_status_permits(&request.key_id, &key_data.status, StateGatedOperation::Encrypt)?; - let key_material = self.decrypt_key_material(&key_data.encrypted_key_material).await?; + let key_material = decode_stored_key_material(&request.key_id, &key_data.encrypted_key_material) + .inspect_err(|error| warn!(key_id = %request.key_id, %error, "Vault KMS key material failed validation"))?; + let (encrypted_key, nonce) = self.dek_crypto.encrypt(&key_material, &request.plaintext).await?; - // For simplicity, we'll use a basic encryption approach - // In practice, you'd use proper AEAD encryption - let mut ciphertext = request.plaintext.clone(); - for (i, byte) in ciphertext.iter_mut().enumerate() { - *byte ^= key_material[i % key_material.len()]; - } + // Wrap the ciphertext in the same authenticated envelope that + // generate_data_key emits, so decrypt() round-trips it and resolves + // the wrapping master key version after rotations. + let envelope = DataKeyEnvelope { + key_id: uuid::Uuid::new_v4().to_string(), + master_key_id: request.key_id.clone(), + key_spec: "AES_256".to_string(), + encrypted_key, + nonce, + encryption_context: request.encryption_context.clone(), + created_at: Zoned::now(), + master_key_version: Some(key_data.version), + }; + let ciphertext = serde_json::to_vec(&envelope)?; Ok(EncryptResponse { ciphertext, @@ -655,7 +660,7 @@ impl KmsClient for VaultKmsClient { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { + pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { debug!("Decrypting data"); // Parse the data key envelope from ciphertext @@ -697,7 +702,12 @@ impl KmsClient for VaultKmsClient { Ok(plaintext) } - async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + pub(crate) async fn create_key( + &self, + key_id: &str, + algorithm: &str, + _context: Option<&OperationContext>, + ) -> Result<MasterKeyInfo> { debug!("Creating master key: {} with algorithm: {}", key_id, algorithm); // Existence pre-check with read-confirm recovery: a create whose @@ -779,7 +789,7 @@ impl KmsClient for VaultKmsClient { Ok(master_key) } - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { + pub(crate) async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { debug!("Describing key: {}", key_id); let key_data = self.get_key_data(key_id).await?; @@ -799,7 +809,11 @@ impl KmsClient for VaultKmsClient { }) } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { + pub(crate) async fn list_keys( + &self, + request: &ListKeysRequest, + _context: Option<&OperationContext>, + ) -> Result<ListKeysResponse> { debug!("Listing keys with limit: {:?}", request.limit); let all_keys = self.list_vault_keys().await?; @@ -836,7 +850,7 @@ impl KmsClient for VaultKmsClient { }) } - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; @@ -848,7 +862,7 @@ impl KmsClient for VaultKmsClient { Ok(()) } - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; @@ -860,39 +874,6 @@ impl KmsClient for VaultKmsClient { Ok(()) } - async fn schedule_key_deletion( - &self, - key_id: &str, - pending_window_days: u32, - _context: Option<&OperationContext>, - ) -> Result<()> { - debug!("Scheduling key deletion: {}", key_id); - - let mut key_data = self.get_key_data(key_id).await?; - ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?; - key_data.status = KeyStatus::PendingDeletion; - key_data.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); - self.store_key_data(key_id, &key_data).await?; - - debug!(key_id, "Vault KMS key deletion scheduled"); - Ok(()) - } - - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - debug!("Canceling key deletion: {}", key_id); - - let mut key_data = self.get_key_data(key_id).await?; - if key_data.status != KeyStatus::PendingDeletion { - return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); - } - key_data.status = KeyStatus::Active; - key_data.deletion_date = None; - self.store_key_data(key_id, &key_data).await?; - - debug!(key_id, "Vault KMS key deletion canceled"); - Ok(()) - } - /// Rotate the master key while keeping every historical version decryptable. /// /// Commit protocol (all writes check-and-set, in this order): @@ -908,10 +889,11 @@ impl KmsClient for VaultKmsClient { /// or interrupted rotation never exposes half-committed material. Concurrent /// rotations are serialized by the check-and-set writes: at most one caller /// commits each version and the losers fail without side effects on current. - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { debug!("Rotating master key: {}", key_id); let (mut cas, mut key_data) = self.get_key_data_versioned(key_id).await?; + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Rotate)?; // The material about to be frozen must be decodable: freezing poisoned // material would give legacy envelopes a permanently broken baseline. This @@ -992,7 +974,7 @@ impl KmsClient for VaultKmsClient { }) } - async fn health_check(&self) -> Result<()> { + pub(crate) async fn health_check(&self) -> Result<()> { debug!("Performing Vault health check"); // Use list_vault_keys but handle the case where no keys exist (which is normal) @@ -1014,15 +996,6 @@ impl KmsClient for VaultKmsClient { } } } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new("vault-kv2".to_string(), "0.1.0".to_string(), self.config.address.clone(), true) - .with_metadata("kv_mount".to_string(), self.kv_mount.clone()) - .with_metadata("key_prefix".to_string(), self.key_path_prefix.clone()) - // Master key material is protected only by Vault ACLs and KV2 at-rest - // encryption; there is no additional cryptographic wrapping. - .with_metadata("at_rest_protection".to_string(), "vault-kv2-acl".to_string()) - } } /// VaultKmsBackend wraps VaultKmsClient and implements the KmsBackend trait @@ -1031,12 +1004,6 @@ pub struct VaultKmsBackend { } impl VaultKmsBackend { - /// Lifecycle driver for the shared state-machine contract tests. - #[cfg(test)] - pub(crate) fn lifecycle_client(&self) -> &VaultKmsClient { - &self.client - } - /// Create a new VaultKmsBackend pub async fn new(config: KmsConfig) -> Result<Self> { config.validate()?; @@ -1296,17 +1263,32 @@ impl KmsBackend for VaultKmsBackend { }) } + async fn enable_key(&self, key_id: &str) -> Result<()> { + self.client.enable_key(key_id, None).await + } + + async fn disable_key(&self, key_id: &str) -> Result<()> { + self.client.disable_key(key_id, None).await + } + + async fn rotate_key(&self, key_id: &str) -> Result<()> { + self.client.rotate_key(key_id, None).await.map(|_| ()) + } + async fn health_check(&self) -> Result<bool> { self.client.health_check().await.map(|_| true) } fn capabilities(&self) -> BackendCapabilities { - // Rotation is unadvertised: the KV2 backend cannot rotate without - // replacing key material in place, and no historical versions are - // retained, so versioning is unsupported as well. + // Rotation freezes the outgoing material as an immutable version + // record before switching the current pointer, and envelopes resolve + // their wrapping version on decrypt, so every historical version + // stays decryptable after a rotation. BackendCapabilities::minimal() + .with_rotate(true) .with_enable_disable(true) .with_schedule_deletion(true) + .with_versioning(true) .with_physical_delete(true) } @@ -1720,19 +1702,6 @@ mod tests { assert!(!is_cas_conflict(¬_found)); } - #[tokio::test] - async fn test_vault_kv2_backend_info_reports_at_rest_protection() { - let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) - .await - .expect("client"); - - let info = client.backend_info(); - assert_eq!(info.backend_type, "vault-kv2"); - assert_eq!(info.metadata.get("at_rest_protection").map(String::as_str), Some("vault-kv2-acl")); - // The KV2 backend must not present itself as Transit-backed. - assert!(!format!("{info:?}").contains("Transit")); - } - fn integration_generate_request(key_id: &str) -> GenerateKeyRequest { GenerateKeyRequest { master_key_id: key_id.to_string(), @@ -2081,4 +2050,150 @@ mod tests { let legacy: VaultKeyData = serde_json::from_value(value).expect("legacy record must deserialize"); assert!(legacy.deletion_date.is_none()); } + + /// KV2 write acknowledgement (`SecretVersionMetadata`) for `kv2::set`. + fn kv2_write_ack() -> serde_json::Value { + serde_json::json!({ + "created_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "deletion_time": "", + "destroyed": false, + "version": 2, + }) + } + + #[tokio::test] + async fn wired_kv2_encrypt_round_trips_through_decrypt() { + // One key-record read for the encrypt, one for the decrypt. + let (_vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ]) + .await; + let context = HashMap::from([("bucket".to_string(), "kv2".to_string())]); + + let encrypted = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"kv2-direct-encrypt".to_vec(), + encryption_context: context.clone(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect("encrypt must produce an envelope"); + + // The ciphertext is a real KMS envelope wrapping AEAD output that + // decrypt() can open, not an XOR of the plaintext with the master key + // material. + let envelope: DataKeyEnvelope = serde_json::from_slice(&encrypted.ciphertext).expect("envelope must parse"); + assert_eq!(envelope.master_key_id, "wired-key"); + assert_eq!(envelope.master_key_version, Some(1)); + + let decrypted = client + .decrypt( + &DecryptRequest { + ciphertext: encrypted.ciphertext.clone(), + encryption_context: context, + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect("decrypt must round-trip the envelope"); + assert_eq!(decrypted, b"kv2-direct-encrypt".to_vec()); + + // A different object context must not decrypt (checked before any + // Vault read, so no scripted response is consumed). + let error = client + .decrypt( + &DecryptRequest { + ciphertext: encrypted.ciphertext, + encryption_context: HashMap::from([("bucket".to_string(), "other".to_string())]), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("a different context must not decrypt"); + assert!(matches!(error, KmsError::ContextMismatch { .. }), "got {error:?}"); + } + + /// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the + /// current secret version used as the rotation check-and-set base. + fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value { + serde_json::json!({ + "cas_required": false, + "created_time": "2026-01-01T00:00:00Z", + "current_version": current_version, + "delete_version_after": "0s", + "max_versions": 0, + "oldest_version": 0, + "updated_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "versions": {}, + }) + } + + #[tokio::test] + async fn wired_kv2_rotate_rejected_while_disabled() { + let mut key_data = healthy_key_data(); + key_data.status = KeyStatus::Disabled; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&key_data)), + ]) + .await; + + let error = client + .rotate_key("wired-key", None) + .await + .expect_err("rotation of a disabled key must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!( + requests.len(), + 2, + "the state gate must reject after the versioned read, before any write: {requests:?}" + ); + assert!(requests.iter().all(|line| line.starts_with("GET ")), "{requests:?}"); + } + + #[tokio::test] + async fn wired_backend_lifecycle_overrides_reach_the_client() { + let mut disabled = healthy_key_data(); + disabled.status = KeyStatus::Disabled; + let vault = ScriptedVault::serve(vec![ + // disable: read the Active record, persist it Disabled. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_write_ack()), + // enable: read the Disabled record, persist it Active. + ScriptedResponse::ok(kv2_read_data(&disabled)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + backend + .disable_key("wired-key") + .await + .expect("KmsBackend::disable_key must persist through the client"); + backend + .enable_key("wired-key") + .await + .expect("KmsBackend::enable_key must persist through the client"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 4, "each transition is one read plus one write: {requests:?}"); + assert!(requests[0].starts_with("GET ") && requests[2].starts_with("GET "), "{requests:?}"); + assert!(requests[1].starts_with("POST ") && requests[3].starts_with("POST "), "{requests:?}"); + } } diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 79d5219b7..2b01e8478 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -18,9 +18,7 @@ use crate::backends::vault_credentials::{ CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider, token_source_for, }; -use crate::backends::{ - BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits, -}; +use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits}; use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; use crate::error::{KmsError, Result}; @@ -506,9 +504,12 @@ impl VaultTransitKmsClient { } } -#[async_trait] -impl KmsClient for VaultTransitKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> { +impl VaultTransitKmsClient { + pub(crate) async fn generate_data_key( + &self, + request: &GenerateKeyRequest, + _context: Option<&OperationContext>, + ) -> Result<DataKeyInfo> { self.ensure_key_state_allows(&request.master_key_id, StateGatedOperation::GenerateDataKey) .await?; @@ -540,7 +541,7 @@ impl KmsClient for VaultTransitKmsClient { )) } - async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { + pub(crate) async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> { let metadata = self .ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt) .await?; @@ -556,7 +557,7 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { + pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> { let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext) .map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?; @@ -578,7 +579,14 @@ impl KmsClient for VaultTransitKmsClient { .await } - async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn create_key( + &self, + key_id: &str, + algorithm: &str, + _context: Option<&OperationContext>, + ) -> Result<MasterKeyInfo> { if algorithm != "AES_256" { return Err(KmsError::unsupported_algorithm(algorithm)); } @@ -645,11 +653,17 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> { self.key_info(key_id).await } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> { + pub(crate) async fn list_keys( + &self, + request: &ListKeysRequest, + _context: Option<&OperationContext>, + ) -> Result<ListKeysResponse> { let all_keys = self .run("vault_transit_list_keys", OpClass::ReadIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; @@ -692,7 +706,7 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { // A pending deletion must be reverted through cancel_key_deletion, not // silently by enabling, so the gate rejects PendingDeletion here. let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Enable).await?; @@ -701,13 +715,15 @@ impl KmsClient for VaultTransitKmsClient { self.store_key_metadata(key_id, &metadata).await } - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Disable).await?; metadata.key_state = KeyState::Disabled; self.store_key_metadata(key_id, &metadata).await } - async fn schedule_key_deletion( + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn schedule_key_deletion( &self, key_id: &str, pending_window_days: u32, @@ -721,17 +737,7 @@ impl KmsClient for VaultTransitKmsClient { self.store_key_metadata(key_id, &metadata).await } - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - let mut metadata = self.get_key_metadata(key_id).await?; - if metadata.key_state != KeyState::PendingDeletion { - return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); - } - metadata.key_state = KeyState::Enabled; - metadata.deletion_date = None; - self.store_key_metadata(key_id, &metadata).await - } - - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { + pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { self.ensure_key_state_allows(key_id, StateGatedOperation::Rotate).await?; // Single attempt, never retried: replaying a rotate whose response was @@ -768,7 +774,7 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn health_check(&self) -> Result<()> { + pub(crate) async fn health_check(&self) -> Result<()> { self.run("vault_transit_health_check", OpClass::ReadIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; key::list(&vault.client, &self.config.mount_path) @@ -780,11 +786,6 @@ impl KmsClient for VaultTransitKmsClient { }) .await } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new("vault-transit".to_string(), "0.1.0".to_string(), self.config.address.clone(), true) - .with_metadata("mount_path".to_string(), self.config.mount_path.clone()) - } } pub struct VaultTransitKmsBackend { @@ -792,14 +793,6 @@ pub struct VaultTransitKmsBackend { } impl VaultTransitKmsBackend { - /// Lifecycle driver for the shared state-machine contract tests. Using the - /// backend's own client keeps its in-process metadata cache coherent with - /// the transitions the tests perform. - #[cfg(test)] - pub(crate) fn lifecycle_client(&self) -> &VaultTransitKmsClient { - &self.client - } - pub async fn new(config: KmsConfig) -> Result<Self> { config.validate()?; @@ -1000,6 +993,18 @@ impl KmsBackend for VaultTransitKmsBackend { }) } + async fn enable_key(&self, key_id: &str) -> Result<()> { + self.client.enable_key(key_id, None).await + } + + async fn disable_key(&self, key_id: &str) -> Result<()> { + self.client.disable_key(key_id, None).await + } + + async fn rotate_key(&self, key_id: &str) -> Result<()> { + self.client.rotate_key(key_id, None).await.map(|_| ()) + } + async fn health_check(&self) -> Result<bool> { self.client.health_check().await.map(|_| true) } @@ -1437,4 +1442,58 @@ mod tests { assert_eq!(metadata.key_state, KeyState::Enabled); assert!(metadata.deletion_date.is_none()); } + + /// KV2 write acknowledgement (`SecretVersionMetadata`) for `kv2::set`. + fn kv2_write_ack() -> serde_json::Value { + serde_json::json!({ + "created_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "deletion_time": "", + "destroyed": false, + "version": 2, + }) + } + + #[tokio::test] + async fn wired_backend_lifecycle_overrides_reach_the_client() { + let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // disable: metadata cache miss reads KV, then persists Disabled. + ScriptedResponse::ok(metadata_read_data(&metadata)), + ScriptedResponse::ok(kv2_write_ack()), + // enable: the state gate hits the metadata cache, so only the + // persisting write goes out. + ScriptedResponse::ok(kv2_write_ack()), + // rotate: the gate hits the cache again; the single rotate + // attempt fails and must not be retried. + ScriptedResponse::error(503, "standby"), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + backend + .disable_key("wired-key") + .await + .expect("KmsBackend::disable_key must persist through the client"); + backend + .enable_key("wired-key") + .await + .expect("KmsBackend::enable_key must persist through the client"); + let error = backend + .rotate_key("wired-key") + .await + .expect_err("the scripted 503 must fail the rotation"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 4, "gated reads, two writes and one rotate attempt: {requests:?}"); + assert_eq!(requests[3], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); + } } diff --git a/crates/kms/src/deletion_worker.rs b/crates/kms/src/deletion_worker.rs index fba80b121..3fca15b25 100644 --- a/crates/kms/src/deletion_worker.rs +++ b/crates/kms/src/deletion_worker.rs @@ -182,7 +182,6 @@ impl DeletionWorker { #[cfg(test)] mod tests { use super::*; - use crate::backends::KmsClient as _; use crate::backends::local::LocalKmsBackend; use crate::config::KmsConfig; use crate::error::KmsError; From 40eee6177a74470b3cb554f39fabe7f30daeb554 Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Fri, 31 Jul 2026 19:33:10 +0800 Subject: [PATCH 39/52] test: add formal hotpath ABBA runner (#5507) Co-authored-by: heihutu <heihutu@gmail.com> --- crates/kms/tests/vault_fault_injection.rs | 39 +- docs/operations/hotpath-warp-abba-runbook.md | 277 ++++++++++++ rustfs/src/admin/router.rs | 12 +- scripts/run_hotpath_warp_abba.sh | 428 +++++++++++++++++++ 4 files changed, 733 insertions(+), 23 deletions(-) create mode 100644 docs/operations/hotpath-warp-abba-runbook.md create mode 100755 scripts/run_hotpath_warp_abba.sh diff --git a/crates/kms/tests/vault_fault_injection.rs b/crates/kms/tests/vault_fault_injection.rs index 14176fdb9..f7f86113a 100644 --- a/crates/kms/tests/vault_fault_injection.rs +++ b/crates/kms/tests/vault_fault_injection.rs @@ -33,9 +33,11 @@ use std::time::Duration; use metrics_util::MetricKind; use metrics_util::debugging::{DebugValue, DebuggingRecorder}; -use rustfs_kms::backends::KmsClient; -use rustfs_kms::backends::vault::VaultKmsClient; -use rustfs_kms::{KmsConfig, KmsError, VaultAuthMethod, VaultConfig}; +use rustfs_kms::backends::KmsBackend as KmsBackendTrait; +use rustfs_kms::backends::vault::VaultKmsBackend; +use rustfs_kms::{ + BackendConfig, DescribeKeyRequest, KmsBackend as KmsBackendKind, KmsConfig, KmsError, VaultAuthMethod, VaultConfig, +}; const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; @@ -54,14 +56,23 @@ fn vault_config(address: &str, token: &str) -> VaultConfig { } } -fn kms_config(attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig { +fn kms_config(vault_config: VaultConfig, attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig { KmsConfig { + backend: KmsBackendKind::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(vault_config)), + allow_insecure_dev_defaults: true, timeout: attempt_timeout, retry_attempts, ..KmsConfig::default() } } +fn describe_key_request(key_id: &str) -> DescribeKeyRequest { + DescribeKeyRequest { + key_id: key_id.to_string(), + } +} + type MetricEntry = ( metrics_util::CompositeKey, Option<metrics::Unit>, @@ -118,11 +129,10 @@ fn connection_refused_is_retried_within_budget() { let snapshot = record_metrics(|| { Box::pin(async move { - let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_secs(2), 2)) + let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_secs(2), 2)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-refused", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-refused")) .await .expect_err("a refused connection must fail the operation"); assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); @@ -166,11 +176,10 @@ fn stalled_connection_is_cut_off_by_the_attempt_timeout() { } }); - let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_millis(250), 1)) + let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_millis(250), 1)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-stalled", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled")) .await .expect_err("a stalled request must be cut off by the attempt timeout"); assert!( @@ -201,11 +210,10 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() { let snapshot = record_metrics(|| { Box::pin(async { let config = vault_config(&real_vault_address(), "fault-injection-invalid-token"); - let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-forbidden", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden")) .await .expect_err("an invalid token must be rejected"); assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); @@ -235,11 +243,10 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() { let snapshot = record_metrics(|| { Box::pin(async move { let config = vault_config(&real_vault_address(), &token); - let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-definitely-missing", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-definitely-missing")) .await .expect_err("a missing key must resolve to key-not-found"); assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}"); diff --git a/docs/operations/hotpath-warp-abba-runbook.md b/docs/operations/hotpath-warp-abba-runbook.md new file mode 100644 index 000000000..5d73b59d1 --- /dev/null +++ b/docs/operations/hotpath-warp-abba-runbook.md @@ -0,0 +1,277 @@ +# Hotpath warp ABBA validation runbook + +This runbook describes how to collect formal Linux or production-cluster +evidence for hotpath performance changes. Use it when a short local A/B smoke +run is too noisy to decide whether a regression is real. + +The ABBA runner executes each workload and drive-sync cell as: + +```text +A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline +``` + +`B1` and `B2` are compared with `A1` to measure the candidate delta. `A2` is +also compared with `A1` to measure baseline drift. Treat a candidate regression +as actionable only when the `A2` drift is passing or materially smaller than +the `B1` and `B2` delta for the same workload. + +## Scope + +Use this runbook for hotpath profiling and performance validation of RustFS +object I/O changes, especially when CPU, memory allocation, lock/channel wait +time, request throughput, or tail latency is the review question. + +The script validates the same workload matrix as the hotpath warp A/B gate: + +| Workload | mode | size | +| --- | --- | --- | +| `put-4kib` | put | 4KiB | +| `put-4mib` | put | 4MiB | +| `get-4kib` | get | 4KiB | +| `get-4mib` | get | 4MiB | +| `get-10mib` | get | 10MiB | +| `mixed-256k` | mixed | 256KiB | + +Each workload runs with `RUSTFS_DRIVE_SYNC_ENABLE=true` and +`RUSTFS_DRIVE_SYNC_ENABLE=false`, so a full ABBA pass produces 48 measurement +cells: 6 workloads x 2 drive-sync modes x 4 ABBA legs. + +## Prerequisites + +Run the formal pass on Linux, not on a laptop smoke environment. + +Required tools on the bench host: + +- `bash`, `curl`, `git`, and core GNU userland. +- `warp` on `PATH`, or pass `--warp-bin`. +- Two RustFS Linux binaries: one baseline and one candidate. +- Enough isolated disks or directories for the local runner, or an externally + managed RustFS cluster for production-like validation. +- Stable host telemetry collection such as `pidstat`, `mpstat`, `iostat`, + `sar`, `perf`, `heaptrack`, or the platform's equivalent observability stack. + +Cluster-mode requirements: + +- A deploy hook that can replace the RustFS binary on every node. +- The hook must apply `RUSTFS_DRIVE_SYNC_ENABLE` for the current ABBA leg. +- The hook must restart RustFS and return only after the rollout command has + been accepted. The ABBA script performs the HTTP readiness wait. +- The benchmark client should run outside the RustFS nodes when possible. +- Do not run against a production data set unless the workload bucket and test + credentials are isolated and approved for destructive benchmark traffic. + +## Build the binaries + +Build the baseline from the comparison commit, usually `origin/main` or the +previous accepted release: + +```bash +git fetch origin main +git switch --detach origin/main +cargo build --release -p rustfs --bins +cp target/release/rustfs /tmp/rustfs-baseline +``` + +Build the candidate from the PR commit: + +```bash +git switch <candidate-branch> +cargo build --release -p rustfs --bins +cp target/release/rustfs /tmp/rustfs-candidate +``` + +For cross-compiled cluster binaries, keep both outputs on the bench host and +make the deploy hook copy the selected binary to the cluster. The ABBA runner +passes the selected binary path through `HOTPATH_ABBA_BINARY`. + +## Local Linux runner + +Use local mode for a dedicated Linux runner with disposable data paths. This is +not a substitute for a production-like cluster, but it is useful before spending +cluster time. + +```bash +scripts/run_hotpath_warp_abba.sh \ + --baseline-bin /tmp/rustfs-baseline \ + --candidate-bin /tmp/rustfs-candidate \ + --address 127.0.0.1:9000 \ + --data-root /var/tmp/rustfs-hotpath-abba \ + --disks 4 \ + --duration 120s \ + --rounds 3 \ + --cooldown 30 \ + --concurrency 16 \ + --out-dir target/hotpath-abba/linux-local +``` + +The script starts and stops RustFS for each ABBA leg. The data root is +throwaway and should not contain important data. + +## Production-like cluster runner + +Use external mode when RustFS lifecycle is managed by ansible, systemd, a +cluster scheduler, or a dedicated deployment harness. In this mode the ABBA +script does not start RustFS directly; it calls `--deploy-hook` before each leg +and then waits for `http://<endpoint><health-path>`. + +The deploy hook receives: + +| Environment variable | Value | +| --- | --- | +| `HOTPATH_ABBA_LEG` | `A1`, `B1`, `B2`, or `A2` | +| `HOTPATH_ABBA_PHASE` | `baseline` or `candidate` | +| `HOTPATH_ABBA_BINARY` | selected baseline or candidate binary path | +| `HOTPATH_ABBA_DRIVE_SYNC` | `true` or `false` | + +Example ansible-shaped command: + +```bash +scripts/run_hotpath_warp_abba.sh \ + --baseline-bin /srv/rustfs-binaries/rustfs-baseline \ + --candidate-bin /srv/rustfs-binaries/rustfs-candidate \ + --endpoint rustfs-bench.example.internal:9000 \ + --deploy-hook ' + set -euo pipefail + cd /srv/rustfs-ansible + cp "${HOTPATH_ABBA_BINARY:?}" roles/rustfs/files/rustfs + export RUSTFS_DRIVE_SYNC_ENABLE="${HOTPATH_ABBA_DRIVE_SYNC:?}" + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags stop + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags config + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags binary-copy + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags start + ' \ + --duration 180s \ + --rounds 5 \ + --cooldown 45 \ + --concurrency 32 \ + --out-dir target/hotpath-abba/cluster-pr-XXXX +``` + +For formal evidence, prefer `--rounds 5` or higher when the cluster budget +allows it. The script enforces `--rounds >= 3`. + +## CPU and memory evidence + +ABBA warp output answers whether the candidate changed throughput or latency. +Collect host telemetry at the same time to explain why. + +Recommended minimum: + +```bash +mkdir -p target/hotpath-abba/cluster-pr-XXXX/telemetry + +pidstat -durh 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/pidstat.txt & +PIDSTAT_PID=$! + +mpstat 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/mpstat.txt & +MPSTAT_PID=$! + +iostat -xz 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/iostat.txt & +IOSTAT_PID=$! +``` + +Stop the collectors after the ABBA script exits: + +```bash +kill "$PIDSTAT_PID" "$MPSTAT_PID" "$IOSTAT_PID" +``` + +For deeper CPU attribution, run `perf record` around one representative +workload after the ABBA gate identifies a candidate regression or improvement: + +```bash +perf record -F 99 -g -- sleep 180 +perf report --stdio > target/hotpath-abba/cluster-pr-XXXX/telemetry/perf-report.txt +``` + +For allocation profiling, build the candidate with: + +```bash +cargo build --release -p rustfs --bins --features hotpath-alloc +``` + +Then run the same ABBA command with that binary. Compare allocation-heavy +function sections only within the same build mode. Do not compare +`hotpath-alloc` binaries directly with default release binaries for throughput +acceptance, because allocation instrumentation intentionally changes what is +measured. + +For CPU hotpath sections emitted by hotpath, build with: + +```bash +cargo build --release -p rustfs --bins --features hotpath-cpu +``` + +Use the CPU-enabled report to explain hotspots after the default or plain +`hotpath` ABBA gate shows a real effect. + +## Output layout + +The ABBA runner writes: + +```text +<out-dir>/ + manifest.env + abba_schedule.csv + candidate_gate.md + baseline_drift_gate.md + summary.md + <workload>/<sync>/<leg>/median_summary.csv + <workload>/<sync>/<leg>/baseline_compare.csv +``` + +Attach or link at least these files in the issue or PR: + +- `summary.md` +- `candidate_gate.md` +- `baseline_drift_gate.md` +- `abba_schedule.csv` +- every `median_summary.csv` and `baseline_compare.csv` for a failed or + borderline workload +- host telemetry files used to explain CPU, memory, or disk saturation + +## Interpretation + +Use this decision table: + +| Candidate gate | A2 drift gate | Interpretation | +| --- | --- | --- | +| PASS | PASS | Candidate is acceptable for the measured matrix. | +| WARN | PASS | Candidate has a small measurable signal; inspect telemetry and decide if it is expected. | +| FAIL | PASS | Candidate likely regressed the affected workload; investigate before merge. | +| FAIL | FAIL on the same workload | Environment drift is high; rerun on a quieter runner or increase duration and rounds. | +| PASS | FAIL | Candidate did not exceed the budget, but the rig was unstable; avoid using the numbers as proof of improvement. | + +When `B1` and `B2` disagree, treat the result as inconclusive even if the gate +passes. Increase duration, rounds, cooldown, or runner isolation before drawing +a conclusion. + +## AI execution checklist + +When delegating the run to an AI agent or an automation runner, provide these +inputs explicitly: + +- repository checkout and candidate branch or commit; +- baseline commit or binary path; +- candidate binary path; +- runner type: local Linux or external cluster; +- endpoint, access key, secret key source, and region; +- deploy hook path or exact command for cluster mode; +- output directory; +- required duration, rounds, cooldown, concurrency, and fail/warn budgets; +- where to upload artifacts after the run. + +The AI agent should execute this sequence: + +1. Confirm `uname -a`, RustFS commits, binary SHA256 sums, `warp --version`, + CPU model, memory size, disk layout, and whether the run is local or cluster. +2. Run `scripts/run_hotpath_warp_abba.sh --dry-run` with the final arguments. +3. Run the real ABBA command with `--rounds >= 3`. +4. Preserve the full output directory without editing generated CSV files. +5. Read `summary.md`, `candidate_gate.md`, and `baseline_drift_gate.md`. +6. Summarize only measured facts: candidate deltas, baseline drift, CPU or + memory saturation, and any failed workloads. +7. Post the summary and artifact location to the tracking issue or PR. + +Do not report a performance win or loss when the baseline drift gate failed on +the same workload and no rerun was collected. diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index a72858a8a..7bec6c88a 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -2708,18 +2708,16 @@ impl<T: Operation> S3Router<T> { pub fn insert(&mut self, method: Method, path: &str, operation: T) -> std::io::Result<()> { let path = Self::make_route_str(method, path); + #[cfg(test)] + let registered_path = path.clone(); // warn!("set uri {}", &path); - #[cfg(test)] - { - self.router.insert(path.clone(), operation).map_err(std::io::Error::other)?; - self.registered_routes.push(path); - } - - #[cfg(not(test))] self.router.insert(path, operation).map_err(std::io::Error::other)?; + #[cfg(test)] + self.registered_routes.push(registered_path); + Ok(()) } diff --git a/scripts/run_hotpath_warp_abba.sh b/scripts/run_hotpath_warp_abba.sh new file mode 100755 index 000000000..ccaff476d --- /dev/null +++ b/scripts/run_hotpath_warp_abba.sh @@ -0,0 +1,428 @@ +#!/usr/bin/env bash +# Formal Linux / production-cluster ABBA runner for the hotpath warp matrix. +# +# This script is intentionally a thin orchestrator around the existing +# run_object_batch_bench_enhanced.sh load driver and hotpath_warp_ab_gate.sh +# relative-budget gate. It runs each durability/workload cell as: +# +# A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline +# +# Candidate legs are compared against A1. The final A2 leg is also compared +# against A1 to quantify baseline drift separately from candidate deltas. + +set -euo pipefail + +PROJECT_ROOT="$(git rev-parse --show-toplevel)" +ENHANCED_BENCH="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh" +GATE="${PROJECT_ROOT}/scripts/hotpath_warp_ab_gate.sh" + +BASELINE_BIN="" +CANDIDATE_BIN="" +ENDPOINT="" +DEPLOY_HOOK="" +HEALTH_PATH="/health" +ADDRESS="127.0.0.1:9000" +DATA_ROOT="/tmp/rustfs-hotpath-abba" +DISKS=4 +ACCESS_KEY="rustfsadmin" +SECRET_KEY="rustfsadmin" +REGION="us-east-1" +WARP_BIN="warp" +CONCURRENCY=8 +DURATION="60s" +ROUNDS=3 +COOLDOWN_SECS=20 +HEALTH_TIMEOUT_SECS=180 +FAIL_PCT=10 +WARN_PCT=5 +ALLOW_REGRESSION=false +EXEMPTION_REASON="deliberate correctness tradeoff" +OUT_DIR="${PROJECT_ROOT}/target/hotpath-abba/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)" +DRY_RUN=false + +WORKLOADS=( + "put-4kib|put|4KiB" + "put-4mib|put|4MiB" + "get-4kib|get|4KiB" + "get-4mib|get|4MiB" + "get-10mib|get|10MiB" + "mixed-256k|mixed|256KiB" +) +DRIVE_SYNC_MATRIX=("sync-on|true" "sync-off|false") + +usage() { + cat <<'USAGE' +Usage: scripts/run_hotpath_warp_abba.sh --baseline-bin <path> --candidate-bin <path> [options] + +Formal ABBA mode for Linux runners or production-like clusters. The schedule is +A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline for every workload +and drive-sync cell. + +Required: + --baseline-bin <path> Baseline RustFS binary. + --candidate-bin <path> Candidate RustFS binary. + +Local Linux runner mode: + --address <host:port> Local RustFS address (default 127.0.0.1:9000). + --disks <n> Throwaway local disks per node (default 4). + --data-root <path> Local disk root (default /tmp/rustfs-hotpath-abba). + +Production / cluster mode: + --endpoint <host:port> Existing cluster endpoint. Enables external mode. + --deploy-hook <cmd> Command run before each ABBA leg. It receives: + HOTPATH_ABBA_LEG=A1|B1|B2|A2 + HOTPATH_ABBA_PHASE=baseline|candidate + HOTPATH_ABBA_BINARY=<baseline/candidate binary> + HOTPATH_ABBA_DRIVE_SYNC=true|false + --health-path <path> Readiness path (default /health). + +Benchmark: + --duration <dur> warp duration per cell (default 60s). + --rounds <n> rounds per cell; must be >= 3 (default 3). + --cooldown <n> cooldown seconds between rounds/sizes (default 20). + --concurrency <n> warp concurrency (default 8). + --warp-bin <path> warp binary (default warp). + +Credentials: + --access-key <value> S3 access key (default rustfsadmin). + --secret-key <value> S3 secret key (default rustfsadmin). + --region <value> S3 region (default us-east-1). + +Gate: + --fail-pct <n> Regression budget that fails gate (default 10). + --warn-pct <n> Regression budget that warns (default 5). + --allow-regression Downgrade candidate gate FAIL to WARN. + --exemption-reason <s> Reason recorded when allow-regression is used. + +Output: + --out-dir <path> Output dir (default target/hotpath-abba/<ts>). + --dry-run Print commands without starting servers or warp. + -h, --help + +Outputs: + <out-dir>/abba_schedule.csv + <out-dir>/candidate_gate.md + <out-dir>/baseline_drift_gate.md + <out-dir>/summary.md + <out-dir>/<workload>/<sync>/<leg>/{median_summary.csv,baseline_compare.csv} +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +log() { + printf '[hotpath-abba] %s\n' "$*" >&2 +} + +run() { + if [[ "$DRY_RUN" == "true" ]]; then + { printf 'DRY-RUN:'; printf ' %q' "$@"; printf '\n'; } >&2 + return 0 + fi + "$@" +} + +validate_positive_int() { + local value="$1" name="$2" + [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --baseline-bin) BASELINE_BIN="$2"; shift 2 ;; + --candidate-bin) CANDIDATE_BIN="$2"; shift 2 ;; + --endpoint) ENDPOINT="$2"; shift 2 ;; + --deploy-hook) DEPLOY_HOOK="$2"; shift 2 ;; + --health-path) HEALTH_PATH="$2"; shift 2 ;; + --address) ADDRESS="$2"; shift 2 ;; + --data-root) DATA_ROOT="$2"; shift 2 ;; + --disks) DISKS="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --cooldown) COOLDOWN_SECS="$2"; shift 2 ;; + --health-timeout) HEALTH_TIMEOUT_SECS="$2"; shift 2 ;; + --fail-pct) FAIL_PCT="$2"; shift 2 ;; + --warn-pct) WARN_PCT="$2"; shift 2 ;; + --allow-regression) ALLOW_REGRESSION=true; shift ;; + --exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +validate_positive_int "$DISKS" "--disks" +validate_positive_int "$CONCURRENCY" "--concurrency" +validate_positive_int "$ROUNDS" "--rounds" +validate_positive_int "$COOLDOWN_SECS" "--cooldown" +validate_positive_int "$HEALTH_TIMEOUT_SECS" "--health-timeout" +validate_positive_int "$FAIL_PCT" "--fail-pct" +validate_positive_int "$WARN_PCT" "--warn-pct" +[[ "$ROUNDS" -ge 3 ]] || die "--rounds must be >= 3 for formal ABBA evidence" +[[ -n "$BASELINE_BIN" ]] || die "--baseline-bin is required" +[[ -n "$CANDIDATE_BIN" ]] || die "--candidate-bin is required" +[[ "$DRY_RUN" == "true" || -x "$BASELINE_BIN" ]] || die "baseline binary is not executable: $BASELINE_BIN" +[[ "$DRY_RUN" == "true" || -x "$CANDIDATE_BIN" ]] || die "candidate binary is not executable: $CANDIDATE_BIN" +[[ -x "$ENHANCED_BENCH" ]] || die "missing load driver: $ENHANCED_BENCH" +[[ -x "$GATE" ]] || die "missing gate: $GATE" +if [[ "$DRY_RUN" != "true" ]] && ! command -v "$WARP_BIN" >/dev/null 2>&1; then + die "warp not found on PATH; install warp or pass --warp-bin" +fi + +EXTERNAL=false +if [[ -n "$ENDPOINT" ]]; then + EXTERNAL=true + ADDRESS="$ENDPOINT" + [[ -n "$DEPLOY_HOOK" ]] || log "warning: external mode without --deploy-hook; binaries must be swapped out of band" +fi + +mkdir -p "$OUT_DIR" +SERVER_LOG_DIR="$OUT_DIR/server-logs" +run mkdir -p "$SERVER_LOG_DIR" + +SERVER_PID="" +SERVER_LOG="" +tear_down() { + [[ "$EXTERNAL" == "true" ]] && return 0 + [[ -n "$SERVER_PID" ]] || return 0 + run kill "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" +} +trap tear_down EXIT INT TERM + +dump_server_log() { + [[ -n "$SERVER_LOG" && -f "$SERVER_LOG" ]] || return 0 + echo "----- last 80 lines of $SERVER_LOG -----" >&2 + tail -n 80 "$SERVER_LOG" >&2 || true + echo "----------------------------------------" >&2 +} + +wait_health() { + [[ "$DRY_RUN" == "true" ]] && return 0 + local i + for ((i = 0; i < HEALTH_TIMEOUT_SECS; i++)); do + if [[ "$EXTERNAL" != "true" && -n "$SERVER_PID" ]] && ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo "error: rustfs server (pid $SERVER_PID) exited before becoming healthy after ${i}s" >&2 + dump_server_log + return 1 + fi + if curl -fsS "http://${ADDRESS}${HEALTH_PATH}" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "error: endpoint http://${ADDRESS}${HEALTH_PATH} did not become healthy within ${HEALTH_TIMEOUT_SECS}s" >&2 + dump_server_log + return 1 +} + +binary_for_leg() { + case "$1" in + A1|A2) echo "$BASELINE_BIN" ;; + B1|B2) echo "$CANDIDATE_BIN" ;; + *) die "unknown ABBA leg: $1" ;; + esac +} + +phase_for_leg() { + case "$1" in + A1|A2) echo "baseline" ;; + B1|B2) echo "candidate" ;; + *) die "unknown ABBA leg: $1" ;; + esac +} + +bring_up() { + local leg="$1" drive_sync="$2" + local phase bin + phase="$(phase_for_leg "$leg")" + bin="$(binary_for_leg "$leg")" + + if [[ "$EXTERNAL" == "true" ]]; then + if [[ -n "$DEPLOY_HOOK" ]]; then + log "deploy hook: leg=$leg phase=$phase drive_sync=$drive_sync" + HOTPATH_ABBA_LEG="$leg" HOTPATH_ABBA_PHASE="$phase" HOTPATH_ABBA_BINARY="$bin" HOTPATH_ABBA_DRIVE_SYNC="$drive_sync" \ + run bash -c "$DEPLOY_HOOK" + fi + wait_health + return 0 + fi + + local node_dir="$DATA_ROOT/$leg-sync-$drive_sync" + local disks=() d + for ((d = 1; d <= DISKS; d++)); do + disks+=("$node_dir/d$d") + done + run mkdir -p "${disks[@]}" + SERVER_LOG="$SERVER_LOG_DIR/$leg-sync-$drive_sync.log" + + if [[ "$DRY_RUN" == "true" ]]; then + { printf 'DRY-RUN: RUSTFS_DRIVE_SYNC_ENABLE=%s %q server' "$drive_sync" "$bin" + printf ' %q' "${disks[@]}"; printf '\n'; } >&2 + SERVER_PID="dry-run" + return 0 + fi + + cat >"$SERVER_LOG_DIR/$leg-sync-$drive_sync.env" <<EOF +leg=$leg +phase=$phase +drive_sync=$drive_sync +binary=$bin +address=$ADDRESS +disks=${disks[*]} +health_url=http://${ADDRESS}${HEALTH_PATH} +health_timeout_secs=$HEALTH_TIMEOUT_SECS +uname=$(uname -a 2>/dev/null || echo unknown) +warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown) +EOF + RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \ + RUSTFS_ADDRESS="$ADDRESS" \ + RUSTFS_ACCESS_KEY="$ACCESS_KEY" \ + RUSTFS_SECRET_KEY="$SECRET_KEY" \ + RUSTFS_REGION="$REGION" \ + RUSTFS_CONSOLE_ENABLE=false \ + RUSTFS_DRIVE_SYNC_ENABLE="$drive_sync" \ + "$bin" server "${disks[@]}" >"$SERVER_LOG" 2>&1 & + SERVER_PID=$! + wait_health +} + +measure() { + local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" baseline_csv="${6:-}" + local cell="$OUT_DIR/$workload/$sync_label/$leg" + local args=( + --tool warp --warp-bin "$WARP_BIN" --warp-mode "$mode" + --endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" + --region "$REGION" --sizes "$size" --concurrency "$CONCURRENCY" + --duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS" + --out-dir "$cell" + ) + [[ -n "$baseline_csv" ]] && args+=(--baseline-csv "$baseline_csv") + run "$ENHANCED_BENCH" "${args[@]}" >&2 + echo "$cell" +} + +write_schedule_header() { + echo "sync_label,drive_sync,workload,mode,size,leg,phase,binary,out_dir" >"$OUT_DIR/abba_schedule.csv" +} + +append_schedule() { + local sync_label="$1" drive_sync="$2" workload="$3" mode="$4" size="$5" leg="$6" + local phase bin + phase="$(phase_for_leg "$leg")" + bin="$(binary_for_leg "$leg")" + echo "$sync_label,$drive_sync,$workload,$mode,$size,$leg,$phase,$bin,$OUT_DIR/$workload/$sync_label/$leg" >>"$OUT_DIR/abba_schedule.csv" +} + +write_manifest() { + cat >"$OUT_DIR/manifest.env" <<EOF +generated_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown) +runner=$(uname -srm 2>/dev/null || echo unknown) +schedule=ABBA +rounds=$ROUNDS +duration=$DURATION +cooldown_secs=$COOLDOWN_SECS +concurrency=$CONCURRENCY +baseline_bin=$BASELINE_BIN +candidate_bin=$CANDIDATE_BIN +external=$EXTERNAL +endpoint=$ADDRESS +warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown) +EOF +} + +declare -a CANDIDATE_COMPARE_CSVS=() +declare -a DRIFT_COMPARE_CSVS=() + +write_manifest +write_schedule_header + +for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do + IFS='|' read -r sync_label drive_sync <<<"$ds_spec" + + for leg in A1 B1 B2 A2; do + log "=== $sync_label leg $leg ($(phase_for_leg "$leg")) ===" + bring_up "$leg" "$drive_sync" + + for wl_spec in "${WORKLOADS[@]}"; do + IFS='|' read -r workload mode size <<<"$wl_spec" + append_schedule "$sync_label" "$drive_sync" "$workload" "$mode" "$size" "$leg" + + baseline_csv="" + if [[ "$leg" != "A1" ]]; then + baseline_csv="$OUT_DIR/$workload/$sync_label/A1/median_summary.csv" + fi + + cell="$(measure "$leg" "$workload" "$mode" "$size" "$sync_label" "$baseline_csv")" + case "$leg" in + B1|B2) CANDIDATE_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;; + A2) DRIFT_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;; + esac + done + + tear_down + done +done + +gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/candidate_gate.md") +for csv in "${CANDIDATE_COMPARE_CSVS[@]}"; do + gate_args+=(--compare-csv "$csv") +done +[[ "$ALLOW_REGRESSION" == "true" ]] && gate_args+=(--allow-regression --exemption-reason "$EXEMPTION_REASON") + +drift_gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/baseline_drift_gate.md") +for csv in "${DRIFT_COMPARE_CSVS[@]}"; do + drift_gate_args+=(--compare-csv "$csv") +done + +if [[ "$DRY_RUN" == "true" ]]; then + log "dry-run complete; candidate compare CSVs=${#CANDIDATE_COMPARE_CSVS[@]} baseline drift CSVs=${#DRIFT_COMPARE_CSVS[@]}" + { printf 'DRY-RUN:'; printf ' %q' "$GATE" "${gate_args[@]}"; printf '\n'; } >&2 + { printf 'DRY-RUN:'; printf ' %q' "$GATE" "${drift_gate_args[@]}"; printf '\n'; } >&2 + exit 0 +fi + +log "applying candidate relative-budget gate" +set +e +"$GATE" "${gate_args[@]}" +candidate_status=$? +set -e + +log "applying A2-vs-A1 baseline drift gate" +set +e +"$GATE" "${drift_gate_args[@]}" +drift_status=$? +set -e + +cat >"$OUT_DIR/summary.md" <<EOF +# Hotpath Warp ABBA Summary + +- schedule: A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline +- runner: $(uname -srm 2>/dev/null || echo unknown) +- warp: $("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown) +- matrix: duration=$DURATION rounds=$ROUNDS cooldown=$COOLDOWN_SECS disks=$DISKS concurrency=$CONCURRENCY +- endpoint: $ADDRESS +- baseline binary: $BASELINE_BIN +- candidate binary: $CANDIDATE_BIN +- candidate gate: $OUT_DIR/candidate_gate.md (exit $candidate_status) +- baseline drift gate: $OUT_DIR/baseline_drift_gate.md (exit $drift_status) + +Interpretation: + +- Treat candidate gate failures as actionable only when the A2-vs-A1 drift gate is PASS or the affected workload's A2 drift is materially smaller than the B1/B2 candidate delta. +- If both candidate and baseline drift fail on the same workload, rerun with longer duration, more rounds, or a quieter runner before assigning causality. +EOF + +log "summary written to $OUT_DIR/summary.md" +if [[ "$candidate_status" -ne 0 || "$drift_status" -ne 0 ]]; then + exit 1 +fi From db8039dece301279226ea7b035af7d8890321cc6 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Fri, 31 Jul 2026 23:57:39 +0800 Subject: [PATCH 40/52] feat(kms): export local backend key material as sealed backup bundles (#5499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): export local backend key material as sealed backup bundles Adds the producer side of the Local backup series on top of the #5483 contract: a directory-wide export fence gives the snapshot a single consistent generation, every artifact is AEAD-wrapped under a caller-supplied backup KEK that is separate from the business trust hierarchy, and the sealed manifest with completeness marker is written last so an interrupted export can never be mistaken for a restorable bundle. Restore and the admin API land in follow-up changes. * fix(kms): verify manifest digest against raw bytes, not re-serialized fields The decode path recomputed the digest by re-serializing the parsed manifest, which silently assumes every field's stored spelling survives a parse-and-reprint round trip. Timestamps do not guarantee that: the time zone annotation jiff emits depends on the host (IANA name, POSIX TZ string, Etc/Unknown), and the legacy-compat parser rewrites bracket-less spellings to +00:00[UTC]. On CI this made freshly written bundles fail digest verification while passing locally. Digest verification now operates on the raw stored bytes, normalized only through the JSON value layer with the digest slot emptied in place; parsed typed fields are never re-serialized on the decode path. Sealing uses the same value-layer canonical form, and the export additionally pins created_at to UTC so bundles are host-independent. A regression test seals a manifest whose created_at spelling cannot round-trip and proves decoding still verifies. One behavior sharpens: inserting an explicit null reserved slot after sealing is now rejected as a digest mismatch instead of being tolerated. * fix(kms): make manifest digest canonicalization independent of map ordering The canonical digest form serialized serde_json values directly, which inherits the key order of serde_json's map type: sorted by default, but insertion-ordered when any crate in the unified build graph enables the preserve_order feature. The workspace-wide CI build unified that feature while a per-crate local build did not, so the frozen fixture digest matched in one environment and not the other — and a bundle sealed by one build flavor would fail verification in the other. Canonicalization now rebuilds every JSON object with bytewise-sorted keys (array order preserved) before hashing, so the digest bytes are identical regardless of feature unification. Reproduced by enabling preserve_order in dev-dependencies (fixture test red), then verified green with the fix under both map flavors. --- crates/kms/src/backends/local.rs | 53 +- crates/kms/src/backup/local_export.rs | 999 ++++++++++++++++++++++++++ crates/kms/src/backup/manifest.rs | 159 +++- crates/kms/src/backup/mod.rs | 17 +- 4 files changed, 1195 insertions(+), 33 deletions(-) create mode 100644 crates/kms/src/backup/local_export.rs diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index e1e347cca..1a12b8593 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -397,6 +397,20 @@ pub struct LocalKmsClient { /// Per-key write locks serializing read-modify-write updates within this /// process (see [`Self::lock_key_for_write`]). key_write_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>, + /// Directory-wide writer fence for backup export (see + /// [`Self::acquire_export_fence`]). Writers hold the read side; an export + /// snapshot holds the write side so it observes a single-generation view. + export_fence: Arc<tokio::sync::RwLock<()>>, +} + +/// Guard pairing the export-fence read lock with a per-key write mutex. +/// +/// Dropping it releases both, so every existing `lock_key_for_write` call +/// site participates in the export fence without changes. +#[must_use] +struct KeyWriteGuard { + _fence: tokio::sync::OwnedRwLockReadGuard<()>, + _key: tokio::sync::OwnedMutexGuard<()>, } // pub(crate) so the backup contract tests can anchor the manifest's @@ -463,6 +477,7 @@ impl LocalKmsClient { legacy_master_cipher, dek_crypto: AesDekCrypto::new(), key_write_locks: Mutex::new(HashMap::new()), + export_fence: Arc::new(tokio::sync::RwLock::new(())), }; client.validate_existing_keys().await?; Ok(client) @@ -505,6 +520,7 @@ impl LocalKmsClient { legacy_master_cipher, dek_crypto: AesDekCrypto::new(), key_write_locks: Mutex::new(HashMap::new()), + export_fence: Arc::new(tokio::sync::RwLock::new(())), }) } @@ -515,12 +531,40 @@ impl LocalKmsClient { /// delete with a rewrite. Cross-process writers sharing a key directory /// remain unsupported. Entries live for the client's lifetime; the table /// is bounded by the number of distinct key ids this process touches. - async fn lock_key_for_write(&self, key_id: &str) -> tokio::sync::OwnedMutexGuard<()> { + async fn lock_key_for_write(&self, key_id: &str) -> KeyWriteGuard { + // Fence first, per-key mutex second: the ordering is uniform across + // all writers, so an export waiting on the write side can never + // deadlock with a writer holding a key mutex. + let fence = Arc::clone(&self.export_fence).read_owned().await; let lock = { let mut locks = self.key_write_locks.lock().expect("Local KMS key write lock table poisoned"); Arc::clone(locks.entry(key_id.to_string()).or_default()) }; - lock.lock_owned().await + KeyWriteGuard { + _fence: fence, + _key: lock.lock_owned().await, + } + } + + /// Block every key-directory writer while a backup export collects its + /// snapshot, so all records belong to one generation. + /// + /// Mutating operations hold the read side (via [`Self::lock_key_for_write`] + /// or [`Self::save_new_master_key`]); the export holds the write side only + /// for the collection phase, never while encrypting or writing the bundle. + pub(crate) async fn acquire_export_fence(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { + Arc::clone(&self.export_fence).write_owned().await + } + + /// Key directory root, exposed for the backup export module. + pub(crate) fn key_directory(&self) -> &Path { + &self.config.key_dir + } + + /// Absolute path of the master-key KDF salt file, exposed for the backup + /// export module. + pub(crate) fn master_key_salt_file(&self) -> PathBuf { + Self::master_key_salt_path(&self.config) } /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. @@ -797,6 +841,11 @@ impl LocalKmsClient { } async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> { + // Creates never take the per-key write lock (`NoClobber` publishing + // already linearizes them), so they join the export fence here. This + // must stay the only fence acquisition on the create path: the fence + // read lock is not reentrant while an export waits for the write side. + let _fence = Arc::clone(&self.export_fence).read_owned().await; let key_path = self.master_key_path(&master_key.key_id)?; let content = self.encode_master_key(master_key, key_material)?; let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs new file mode 100644 index 000000000..d3905b31f --- /dev/null +++ b/crates/kms/src/backup/local_export.rs @@ -0,0 +1,999 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Local backend backup export: sealed, KEK-protected bundle production. +//! +//! This is the producer side only; restore lives in a follow-up change. The +//! admin API is not wired here either — callers construct the request and +//! supply the backup KEK explicitly. +//! +//! # Bundle layout +//! +//! A bundle is a directory (simple to produce, artifacts stream one file at a +//! time, and partial output is trivially recognizable because the manifest is +//! written last): +//! +//! ```text +//! <destination>/ +//! manifest.json # sealed BackupManifest, written last +//! artifacts/keys/<key_id>.key.enc # one per stored key record +//! artifacts/master-key.salt.enc # present when the salt file exists +//! ``` +//! +//! # Artifact payload framing +//! +//! Every artifact payload is `nonce (12 bytes) || AES-256-GCM ciphertext`, +//! encrypted under the caller-supplied backup KEK with an AAD binding of +//! `(context, backup_id, snapshot_generation, artifact path)`, so an artifact +//! cannot be swapped into another bundle or renamed within its own bundle. +//! Records already encrypted at rest stay encrypted inside the wrap; +//! plaintext-dev-only records become ciphertext-only in the bundle, which is +//! their mandatory re-wrap under the backup KEK. +//! +//! # Write protocol +//! +//! Artifacts are written and fsynced first, re-read and digest-verified, and +//! only then is the sealed manifest (completeness marker plus final digest) +//! published. A crash at any earlier point leaves a bundle without a +//! manifest, which decodes as an incomplete bundle and can never be restored. + +use crate::backends::local::{LocalKmsClient, StoredKeyProtection}; +use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; +use crate::backup::error::BackupError; +use crate::backup::manifest::{ + AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, + DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, +}; +use crate::error::{KmsError, Result}; +use aes_gcm::{ + Aes256Gcm, Key, Nonce, + aead::{Aead, KeyInit, Payload}, +}; +use jiff::Zoned; +use rand::RngExt; +use serde::Deserialize; +use std::path::{Path, PathBuf}; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use zeroize::Zeroizing; + +/// File name of the sealed manifest inside a bundle directory. +pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json"; +const ARTIFACTS_DIR: &str = "artifacts"; +const KEYS_DIR: &str = "artifacts/keys"; +const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc"; +const AEAD_NONCE_LEN: usize = 12; +/// Domain-separation context for the artifact AAD binding. +const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1"; + +/// Caller-supplied backup KEK: a trust root deliberately separate from the +/// business KMS hierarchy (it must not be a key that is itself part of the +/// state being backed up). Where the KEK comes from is the admin layer's +/// concern; this module only consumes it. +pub struct BackupKek { + kek_id: String, + kek_version: u32, + key: Zeroizing<[u8; 32]>, +} + +impl BackupKek { + /// Wrap 32 bytes of KEK material. The material is zeroized on drop; + /// callers should zeroize their own copy of the input. + pub fn new(kek_id: impl Into<String>, kek_version: u32, key: [u8; 32]) -> Result<Self> { + let kek_id = kek_id.into(); + if kek_id.is_empty() { + return Err(KmsError::validation_error("backup KEK id must not be empty")); + } + Ok(Self { + kek_id, + kek_version, + key: Zeroizing::new(key), + }) + } + + /// Manifest descriptor for this KEK. + pub fn descriptor(&self) -> BackupKekDescriptor { + BackupKekDescriptor { + kek_id: self.kek_id.clone(), + kek_version: self.kek_version, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + } + } + + fn cipher(&self) -> Aes256Gcm { + Aes256Gcm::new(&Key::<Aes256Gcm>::from(*self.key)) + } +} + +/// Parameters of one export run. +/// +/// `snapshot_generation` is injected by the caller: the contract only +/// requires it to be monotonic per deployment, and the source (persisted +/// counter, coordinated clock) is decided by the admin layer, which keeps +/// this module free of ambient time or state lookups. +#[derive(Debug, Clone)] +pub struct LocalBackupExportRequest { + /// Unique identifier for this backup. + pub backup_id: String, + /// Opaque identity of the producing deployment. + pub deployment_identity: String, + /// RustFS version string recorded in the manifest. + pub rustfs_version: String, + /// Monotonic snapshot generation this bundle belongs to. + pub snapshot_generation: u64, + /// Bundle output directory; must not exist yet or must be empty. + pub destination: PathBuf, +} + +impl LocalBackupExportRequest { + fn validate(&self) -> Result<()> { + for (field, value) in [ + ("backup_id", &self.backup_id), + ("deployment_identity", &self.deployment_identity), + ("rustfs_version", &self.rustfs_version), + ] { + if value.is_empty() { + return Err(KmsError::validation_error(format!("backup export {field} must not be empty"))); + } + } + Ok(()) + } +} + +/// Minimal projection of a stored key record: only the fields the exporter +/// needs. Unknown fields are ignored on purpose — the record travels into the +/// bundle byte-identical, so the exporter must not constrain its schema. +#[derive(Deserialize)] +struct StoredRecordProbe { + key_id: String, + #[serde(default)] + at_rest_protection: StoredKeyProtection, +} + +struct CollectedRecord { + key_id: String, + protection: StoredKeyProtection, + /// Raw record bytes exactly as stored. Zeroized on drop because + /// plaintext-dev-only records embed key material. + raw: Zeroizing<Vec<u8>>, +} + +struct CollectedSnapshot { + records: Vec<CollectedRecord>, + salt: Option<Vec<u8>>, +} + +/// Export the Local backend's key directory as a sealed backup bundle. +/// +/// The directory scan runs under the export fence, so concurrent +/// create/update/delete operations are either fully included or fully +/// excluded — never half a record. Encryption and bundle writing happen after +/// the fence is released to keep it short. +/// +/// Returns the sealed manifest that was written to the bundle. +pub async fn export_local_backup( + client: &LocalKmsClient, + kek: &BackupKek, + request: &LocalBackupExportRequest, +) -> Result<BackupManifest> { + request.validate()?; + prepare_destination(&request.destination).await?; + + let snapshot = collect_snapshot(client).await?; + if snapshot.records.is_empty() { + return Err(KmsError::invalid_operation( + "Local backup export found no key records; refusing to publish an empty bundle", + )); + } + + let has_encrypted = snapshot + .records + .iter() + .any(|record| record.protection == StoredKeyProtection::EncryptedMasterKey); + if has_encrypted && snapshot.salt.is_none() { + return Err(KmsError::invalid_operation( + "key directory contains encrypted-master-key records but the master key salt file is missing; \ + the bundle would be unrestorable", + )); + } + + let manifest = build_and_write_bundle(kek, request, &snapshot).await?; + Ok(manifest) +} + +/// Read and fully validate the manifest of a local bundle directory. +/// +/// A directory without a manifest is an interrupted export: the manifest is +/// written last, so its absence means the bundle never sealed. +pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> { + let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE); + let bytes = match fs::read(&manifest_path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(BackupError::incomplete_bundle("bundle has no manifest; the export never sealed it").into()); + } + Err(error) => return Err(error.into()), + }; + let manifest = BackupManifest::decode(&bytes)?; + if manifest.backend != BackupBackendKind::Local { + return Err( + BackupError::corrupted(format!("bundle manifest declares backend {:?}, expected Local", manifest.backend)).into(), + ); + } + Ok(manifest) +} + +/// Read, verify, and decrypt one artifact of a local bundle. +/// +/// Fail-closed order: KEK identity, artifact presence, declared length, +/// encrypted digest, then AEAD authentication. The returned plaintext is +/// zeroized on drop. +pub async fn decrypt_bundle_artifact( + bundle_dir: &Path, + manifest: &BackupManifest, + descriptor: &ArtifactDescriptor, + kek: &BackupKek, +) -> Result<Zeroizing<Vec<u8>>> { + manifest.backup_kek.ensure_matches(&kek.kek_id, kek.kek_version)?; + if descriptor.aead_algorithm != AeadAlgorithm::Aes256Gcm { + return Err(KmsError::unsupported_algorithm(format!( + "{:?} (local bundles are produced with AES-256-GCM)", + descriptor.aead_algorithm + ))); + } + + let artifact_path = bundle_dir.join(&descriptor.path); + let payload = match fs::read(&artifact_path).await { + Ok(payload) => payload, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(BackupError::missing_artifact(descriptor.path.clone()).into()); + } + Err(error) => return Err(error.into()), + }; + + if (payload.len() as u64) < descriptor.len { + return Err(BackupError::truncated(format!( + "artifact '{}' is {} bytes, manifest declares {}", + descriptor.path, + payload.len(), + descriptor.len + )) + .into()); + } + if payload.len() as u64 != descriptor.len { + return Err(BackupError::corrupted(format!( + "artifact '{}' is {} bytes, manifest declares {}", + descriptor.path, + payload.len(), + descriptor.len + )) + .into()); + } + if ContentDigest::sha256_of(&payload) != descriptor.encrypted_digest { + return Err(BackupError::corrupted(format!("artifact '{}' does not match its manifest digest", descriptor.path)).into()); + } + if payload.len() < AEAD_NONCE_LEN { + return Err(BackupError::corrupted(format!("artifact '{}' is too short to carry a nonce", descriptor.path)).into()); + } + + let (nonce_bytes, ciphertext) = payload.split_at(AEAD_NONCE_LEN); + let mut nonce = [0u8; AEAD_NONCE_LEN]; + nonce.copy_from_slice(nonce_bytes); + let aad = artifact_aad(&manifest.backup_id, manifest.snapshot_generation, &descriptor.path); + let plaintext = kek + .cipher() + .decrypt( + &Nonce::from(nonce), + Payload { + msg: ciphertext, + aad: &aad, + }, + ) + .map_err(|_| { + KmsError::from(BackupError::corrupted(format!( + "artifact '{}' failed authenticated decryption under the supplied backup KEK", + descriptor.path + ))) + })?; + Ok(Zeroizing::new(plaintext)) +} + +/// Scan the key directory under the export fence. +async fn collect_snapshot(client: &LocalKmsClient) -> Result<CollectedSnapshot> { + let _fence = client.acquire_export_fence().await; + + let mut records = Vec::new(); + let mut entries = fs::read_dir(client.key_directory()).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if !path.extension().is_some_and(|extension| extension == "key") { + continue; + } + let stem = path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))? + .to_string(); + + let raw = Zeroizing::new(fs::read(&path).await?); + // Any unreadable record aborts the export: a bundle silently missing + // one key is worse than no bundle at all. + let probe: StoredRecordProbe = serde_json::from_slice(&raw) + .map_err(|error| KmsError::material_corrupt(&stem, format!("stored key record does not deserialize: {error}")))?; + if probe.key_id != stem { + return Err(KmsError::invalid_key(format!( + "Local KMS key file identity mismatch: expected {stem:?}, found {:?}", + probe.key_id + ))); + } + + records.push(CollectedRecord { + key_id: stem, + protection: probe.at_rest_protection, + raw, + }); + } + + let salt_path = client.master_key_salt_file(); + let salt = match fs::read(&salt_path).await { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + + records.sort_by(|a, b| a.key_id.cmp(&b.key_id)); + Ok(CollectedSnapshot { records, salt }) +} + +async fn build_and_write_bundle( + kek: &BackupKek, + request: &LocalBackupExportRequest, + snapshot: &CollectedSnapshot, +) -> Result<BackupManifest> { + let mut artifacts = Vec::with_capacity(snapshot.records.len() + 1); + for record in &snapshot.records { + let artifact_path = format!("{KEYS_DIR}/{}.key.enc", record.key_id); + let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::KeyMaterial, &artifact_path, &record.raw).await?; + artifacts.push(descriptor); + } + if let Some(salt) = &snapshot.salt { + let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::MasterKeySalt, SALT_ARTIFACT_PATH, salt).await?; + artifacts.push(descriptor); + } + + // Make the artifact directory entries durable before sealing: the sealed + // manifest must never survive a crash that its artifacts did not. + fsync_dir(&request.destination.join(KEYS_DIR)).await?; + fsync_dir(&request.destination.join(ARTIFACTS_DIR)).await?; + + let manifest = BackupManifest { + format_version: BackupManifest::FORMAT_VERSION, + backup_id: request.backup_id.clone(), + // Normalized to UTC so the stored spelling is host-independent: the + // local zone's name (or a POSIX TZ string in minimal containers) has + // no business inside a portable bundle. + created_at: Zoned::now().with_time_zone(jiff::tz::TimeZone::UTC), + rustfs_version: request.rustfs_version.clone(), + deployment_identity: request.deployment_identity.clone(), + backend: BackupBackendKind::Local, + at_rest_protection: weakest_observed_protection(&snapshot.records), + responsibility: BackupResponsibility::FullMaterial, + snapshot_generation: request.snapshot_generation, + backup_kek: kek.descriptor(), + artifacts, + local_kdf: Some(local_kdf_descriptor(snapshot)), + key_versions: None, + capability_discovery: None, + completeness: CompletenessState::InProgress, + manifest_digest: ContentDigest { + algorithm: DigestAlgorithm::Sha256, + hex: String::new(), + }, + }; + let manifest = manifest.seal()?; + let manifest_bytes = manifest.encode()?; + + write_new_file(&request.destination.join(LOCAL_BUNDLE_MANIFEST_FILE), &manifest_bytes).await?; + fsync_dir(&request.destination).await?; + Ok(manifest) +} + +/// Encrypt one artifact, write it durably, and re-read it to verify the +/// digest before it is allowed into the manifest. +async fn encrypt_and_write_artifact( + kek: &BackupKek, + request: &LocalBackupExportRequest, + kind: ArtifactKind, + artifact_path: &str, + plaintext: &[u8], +) -> Result<ArtifactDescriptor> { + let mut nonce = [0u8; AEAD_NONCE_LEN]; + rand::rng().fill(&mut nonce[..]); + let aad = artifact_aad(&request.backup_id, request.snapshot_generation, artifact_path); + let ciphertext = kek + .cipher() + .encrypt( + &Nonce::from(nonce), + Payload { + msg: plaintext, + aad: &aad, + }, + ) + .map_err(|error| KmsError::cryptographic_error("backup_artifact_encrypt", error.to_string()))?; + + let mut payload = Vec::with_capacity(AEAD_NONCE_LEN + ciphertext.len()); + payload.extend_from_slice(&nonce); + payload.extend_from_slice(&ciphertext); + + let absolute_path = request.destination.join(artifact_path); + write_new_file(&absolute_path, &payload).await?; + + // Verify what actually landed on disk, not the in-memory buffer. + let written = fs::read(&absolute_path).await?; + let digest = ContentDigest::sha256_of(&written); + if written != payload { + return Err(KmsError::internal_error(format!( + "bundle artifact '{artifact_path}' read back differently than written" + ))); + } + + Ok(ArtifactDescriptor { + kind, + path: artifact_path.to_string(), + len: payload.len() as u64, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + encrypted_digest: digest, + }) +} + +/// AAD binding an artifact to its bundle identity and path. A JSON tuple +/// gives unambiguous field boundaries without a hand-rolled framing format. +fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> { + serde_json::to_vec(&(BUNDLE_AAD_CONTEXT, backup_id, snapshot_generation, artifact_path)) + .expect("AAD tuple of strings and integers always serializes") +} + +/// The bundle-level protection label is the weakest state observed across +/// records: any plaintext-dev-only record marks the whole bundle, then any +/// legacy-unspecified marker (unknown until read), and only a uniformly +/// encrypted directory is labeled encrypted-master-key. +fn weakest_observed_protection(records: &[CollectedRecord]) -> AtRestProtection { + let mut has_legacy = false; + for record in records { + match record.protection { + StoredKeyProtection::PlaintextDevOnly => return AtRestProtection::PlaintextDevOnly, + StoredKeyProtection::LegacyUnspecified => has_legacy = true, + StoredKeyProtection::EncryptedMasterKey => {} + } + } + if has_legacy { + AtRestProtection::LegacyUnspecified + } else { + AtRestProtection::EncryptedMasterKey + } +} + +fn local_kdf_descriptor(snapshot: &CollectedSnapshot) -> LocalKdfDescriptor { + let mut modes = Vec::new(); + for (marker, mode) in [ + (StoredKeyProtection::EncryptedMasterKey, AtRestProtection::EncryptedMasterKey), + (StoredKeyProtection::PlaintextDevOnly, AtRestProtection::PlaintextDevOnly), + (StoredKeyProtection::LegacyUnspecified, AtRestProtection::LegacyUnspecified), + ] { + if snapshot.records.iter().any(|record| record.protection == marker) { + modes.push(mode); + } + } + + // With a salt on disk the backend derives via Argon2id; without one only + // the pre-beta.9 SHA-256 derivation can apply. For plaintext-only + // directories the derivation is informational. + let derivation = if snapshot.salt.is_some() { + LocalKeyDerivation::current_argon2id() + } else { + LocalKeyDerivation::LegacySha256 + }; + + LocalKdfDescriptor { + derivation, + protection_modes: modes, + // The verifier shape is left to the restore change; the schema keeps + // it optional so bundles without one stay valid. + master_key_verifier: None, + } +} + +async fn prepare_destination(destination: &Path) -> Result<()> { + if fs::try_exists(destination).await? { + let mut entries = fs::read_dir(destination) + .await + .map_err(|error| KmsError::invalid_operation(format!("backup destination is not a readable directory: {error}")))?; + if entries.next_entry().await?.is_some() { + return Err(KmsError::invalid_operation( + "backup destination directory is not empty; refusing to mix bundles", + )); + } + } + fs::create_dir_all(destination.join(KEYS_DIR)).await?; + Ok(()) +} + +async fn write_new_file(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await + .map_err(|error| KmsError::io_error(format!("failed to create bundle file {}: {error}", path.display())))?; + file.write_all(bytes).await?; + file.sync_all().await?; + Ok(()) +} + +/// Fsync a directory so freshly created bundle entries survive power loss. +/// No-op on non-Unix platforms where directories cannot be opened for +/// syncing (mirrors the local backend's durable commit helper). +async fn fsync_dir(path: &Path) -> Result<()> { + #[cfg(unix)] + { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || std::fs::File::open(&path)?.sync_all()) + .await + .map_err(|error| KmsError::io_error(error.to_string()))??; + } + #[cfg(not(unix))] + let _ = path; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backends::KmsClient; + use crate::config::LocalConfig; + use std::sync::Arc; + use tempfile::TempDir; + + async fn encrypted_client() -> (LocalKmsClient, TempDir) { + let temp = TempDir::new().expect("temp dir"); + let client = LocalKmsClient::new(LocalConfig { + key_dir: temp.path().to_path_buf(), + master_key: Some("test-master-key".to_string()), + file_permissions: Some(0o600), + }) + .await + .expect("client should initialize"); + (client, temp) + } + + async fn dev_client() -> (LocalKmsClient, TempDir) { + let temp = TempDir::new().expect("temp dir"); + let client = LocalKmsClient::new(LocalConfig { + key_dir: temp.path().to_path_buf(), + master_key: None, + file_permissions: Some(0o600), + }) + .await + .expect("client should initialize"); + (client, temp) + } + + fn test_kek() -> BackupKek { + BackupKek::new("backup-kek-test", 1, [0x42; 32]).expect("kek") + } + + fn export_request(destination: PathBuf) -> LocalBackupExportRequest { + LocalBackupExportRequest { + backup_id: "backup-0001".to_string(), + deployment_identity: "deployment-test".to_string(), + rustfs_version: "1.0.0-test".to_string(), + snapshot_generation: 7, + destination, + } + } + + fn walk_files(dir: &Path, out: &mut Vec<PathBuf>) { + for entry in std::fs::read_dir(dir).expect("read dir") { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + walk_files(&path, out); + } else { + out.push(path); + } + } + } + + fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|window| window == needle) + } + + #[tokio::test] + async fn export_round_trips_and_decrypts_to_source_records() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("alpha", "AES_256", None).await.expect("create alpha"); + client.create_key("beta", "AES_256", None).await.expect("create beta"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + + assert_eq!(manifest.backend, BackupBackendKind::Local); + assert_eq!(manifest.responsibility, BackupResponsibility::FullMaterial); + assert_eq!(manifest.at_rest_protection, AtRestProtection::EncryptedMasterKey); + assert_eq!(manifest.snapshot_generation, 7); + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.derivation, LocalKeyDerivation::current_argon2id()); + assert_eq!(kdf.protection_modes, vec![AtRestProtection::EncryptedMasterKey]); + + // alpha, beta (sorted), then the salt artifact. + assert_eq!(manifest.artifacts.len(), 3); + assert_eq!(manifest.artifacts[0].path, "artifacts/keys/alpha.key.enc"); + assert_eq!(manifest.artifacts[1].path, "artifacts/keys/beta.key.enc"); + assert_eq!(manifest.artifacts[2].kind, ArtifactKind::MasterKeySalt); + + let reread = read_local_bundle_manifest(&destination) + .await + .expect("manifest should decode"); + assert_eq!(reread, manifest); + + for (artifact, key_id) in [(&manifest.artifacts[0], "alpha"), (&manifest.artifacts[1], "beta")] { + let decrypted = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect("artifact should decrypt"); + let source = fs::read(client.key_directory().join(format!("{key_id}.key"))) + .await + .expect("source record"); + assert_eq!(decrypted.as_slice(), source.as_slice(), "record {key_id} must round-trip verbatim"); + } + + let salt = decrypt_bundle_artifact(&destination, &manifest, &manifest.artifacts[2], &kek) + .await + .expect("salt should decrypt"); + let source_salt = fs::read(client.master_key_salt_file()).await.expect("source salt"); + assert_eq!(salt.as_slice(), source_salt.as_slice()); + } + + #[tokio::test] + async fn plaintext_dev_only_material_is_rewrapped_and_absent_from_bundle() { + let (client, _key_dir) = dev_client().await; + client.create_key("dev-key", "AES_256", None).await.expect("create key"); + + let material = client + .decrypt_key_material_for_export("dev-key") + .await + .expect("material should be readable"); + let source_record = fs::read(client.key_directory().join("dev-key.key")).await.expect("record"); + let record_json: serde_json::Value = serde_json::from_slice(&source_record).expect("record parses"); + let material_base64 = record_json + .get("encrypted_key_material") + .and_then(|value| value.as_str()) + .expect("material field") + .to_string(); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + + assert_eq!(manifest.at_rest_protection, AtRestProtection::PlaintextDevOnly); + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.protection_modes, vec![AtRestProtection::PlaintextDevOnly]); + assert_eq!(kdf.derivation, LocalKeyDerivation::LegacySha256); + assert!( + !manifest.artifacts.iter().any(|a| a.kind == ArtifactKind::MasterKeySalt), + "dev-mode directory has no salt to bundle" + ); + + // Byte-level: neither the raw material nor its base64 form may appear + // anywhere in the bundle. The mandatory KEK re-wrap is what hides it. + let mut files = Vec::new(); + walk_files(&destination, &mut files); + assert!(!files.is_empty()); + for file in files { + let bytes = std::fs::read(&file).expect("bundle file"); + assert!( + !contains_subslice(&bytes, material.as_ref()), + "raw key material leaked into {}", + file.display() + ); + assert!( + !contains_subslice(&bytes, material_base64.as_bytes()), + "base64 key material leaked into {}", + file.display() + ); + } + + // The wrapped record still round-trips for restore. + let decrypted = decrypt_bundle_artifact(&destination, &manifest, &manifest.artifacts[0], &kek) + .await + .expect("artifact should decrypt"); + assert_eq!(decrypted.as_slice(), source_record.as_slice()); + } + + #[tokio::test] + async fn export_fence_blocks_writers_until_released() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("existing", "AES_256", None).await.expect("create key"); + let client = Arc::new(client); + + let fence = client.acquire_export_fence().await; + + let writer = { + let client = Arc::clone(&client); + tokio::spawn(async move { + client.create_key("new-key", "AES_256", None).await.expect("create"); + client.disable_key("existing", None).await.expect("disable"); + }) + }; + + for _ in 0..64 { + tokio::task::yield_now().await; + } + assert!(!writer.is_finished(), "writers must stay blocked while the export fence is held"); + + drop(fence); + writer.await.expect("writer should finish after fence release"); + assert!( + fs::try_exists(client.key_directory().join("new-key.key")) + .await + .expect("exists") + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_writers_yield_complete_records() { + let (client, _key_dir) = encrypted_client().await; + for index in 0..5 { + client + .create_key(&format!("seed-{index}"), "AES_256", None) + .await + .expect("seed key"); + } + let client = Arc::new(client); + + let writer = { + let client = Arc::clone(&client); + tokio::spawn(async move { + for index in 0..30 { + client + .create_key(&format!("concurrent-{index}"), "AES_256", None) + .await + .expect("create"); + let target = format!("seed-{}", index % 5); + if index % 2 == 0 { + client.disable_key(&target, None).await.expect("disable"); + } else { + client.enable_key(&target, None).await.expect("enable"); + } + } + }) + }; + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed under concurrent writers"); + writer.await.expect("writer task"); + + // Whatever subset of writers landed before the fence, every record in + // the bundle must be complete: parseable, self-identifying, and with + // non-empty material. No torn records, no half-updates. + let reread = read_local_bundle_manifest(&destination).await.expect("manifest decodes"); + assert_eq!(reread, manifest); + for artifact in manifest.artifacts.iter().filter(|a| a.kind == ArtifactKind::KeyMaterial) { + let record = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect("record decrypts"); + let value: serde_json::Value = serde_json::from_slice(&record).expect("record is complete JSON"); + let key_id = value.get("key_id").and_then(|v| v.as_str()).expect("key_id present"); + assert_eq!(artifact.path, format!("artifacts/keys/{key_id}.key.enc")); + let material = value + .get("encrypted_key_material") + .and_then(|v| v.as_str()) + .expect("material present"); + assert!(!material.is_empty()); + } + } + + #[tokio::test] + async fn tampered_and_truncated_bundles_fail_closed() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("victim", "AES_256", None).await.expect("create key"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + let artifact = &manifest.artifacts[0]; + let artifact_file = destination.join(&artifact.path); + let original_artifact = std::fs::read(&artifact_file).expect("artifact bytes"); + + // Tampered artifact byte: digest verification rejects it. + let mut tampered = original_artifact.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0x01; + std::fs::write(&artifact_file, &tampered).expect("write tampered"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect_err("tampered artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + + // Truncated artifact: typed truncation error. + std::fs::write(&artifact_file, &original_artifact[..original_artifact.len() - 4]).expect("truncate"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect_err("truncated artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); + std::fs::write(&artifact_file, &original_artifact).expect("restore artifact"); + + // Tampered manifest (generation flip): sealed digest mismatch. + let manifest_file = destination.join(LOCAL_BUNDLE_MANIFEST_FILE); + let original_manifest = std::fs::read(&manifest_file).expect("manifest bytes"); + let tampered_manifest = String::from_utf8(original_manifest.clone()) + .expect("manifest is utf-8") + .replace("\"snapshot_generation\":7", "\"snapshot_generation\":8"); + assert_ne!(tampered_manifest.as_bytes(), original_manifest.as_slice(), "tamper must apply"); + std::fs::write(&manifest_file, tampered_manifest).expect("write tampered manifest"); + let error = read_local_bundle_manifest(&destination) + .await + .expect_err("tampered manifest must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + + // Truncated manifest. + std::fs::write(&manifest_file, &original_manifest[..original_manifest.len() / 2]).expect("truncate manifest"); + let error = read_local_bundle_manifest(&destination) + .await + .expect_err("truncated manifest must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); + + // Missing manifest: the bundle never sealed. + std::fs::remove_file(&manifest_file).expect("remove manifest"); + let error = read_local_bundle_manifest(&destination) + .await + .expect_err("bundle without manifest must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::IncompleteBundle { .. })), "got {error:?}"); + } + + #[tokio::test] + async fn wrong_kek_is_rejected_before_decryption() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("victim", "AES_256", None).await.expect("create key"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + let artifact = &manifest.artifacts[0]; + + let wrong_id = BackupKek::new("other-kek", 1, [0x42; 32]).expect("kek"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &wrong_id) + .await + .expect_err("mismatched KEK id must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); + + let wrong_version = BackupKek::new("backup-kek-test", 2, [0x42; 32]).expect("kek"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &wrong_version) + .await + .expect_err("mismatched KEK version must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); + + // Right identity, wrong material: AEAD authentication fails closed. + let wrong_material = BackupKek::new("backup-kek-test", 1, [0x24; 32]).expect("kek"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &wrong_material) + .await + .expect_err("wrong KEK material must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + } + + #[tokio::test] + async fn missing_salt_with_encrypted_records_fails_export() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("victim", "AES_256", None).await.expect("create key"); + fs::remove_file(client.master_key_salt_file()).await.expect("remove salt"); + + let bundle = TempDir::new().expect("bundle dir"); + let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect_err("export without salt must fail"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("salt"), "got {error}"); + } + + #[tokio::test] + async fn refuses_empty_key_dir_and_nonempty_destination() { + let (client, _key_dir) = dev_client().await; + let bundle = TempDir::new().expect("bundle dir"); + let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect_err("empty key dir must not produce a bundle"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + client.create_key("dev-key", "AES_256", None).await.expect("create key"); + let occupied = bundle.path().join("occupied"); + std::fs::create_dir_all(&occupied).expect("mkdir"); + std::fs::write(occupied.join("stale"), b"leftover").expect("occupy"); + let error = export_local_backup(&client, &test_kek(), &export_request(occupied)) + .await + .expect_err("non-empty destination must be refused"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + } + + #[tokio::test] + async fn legacy_records_export_verbatim_with_weakest_protection_label() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("modern", "AES_256", None).await.expect("create key"); + client.create_key("legacy-key", "AES_256", None).await.expect("create key"); + + // Strip the protection marker to fabricate a pre-beta.9 record, the + // same way the local backend's own legacy-compat tests do. + let legacy_path = client.key_directory().join("legacy-key.key"); + let mut record: serde_json::Value = + serde_json::from_slice(&fs::read(&legacy_path).await.expect("record")).expect("record parses"); + record + .as_object_mut() + .expect("record is an object") + .remove("at_rest_protection"); + let legacy_bytes = serde_json::to_vec_pretty(&record).expect("record serializes"); + fs::write(&legacy_path, &legacy_bytes).await.expect("write legacy record"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + + assert_eq!(manifest.at_rest_protection, AtRestProtection::LegacyUnspecified); + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!( + kdf.protection_modes, + vec![AtRestProtection::EncryptedMasterKey, AtRestProtection::LegacyUnspecified] + ); + + let legacy_artifact = manifest + .artifacts + .iter() + .find(|a| a.path == "artifacts/keys/legacy-key.key.enc") + .expect("legacy artifact"); + let decrypted = decrypt_bundle_artifact(&destination, &manifest, legacy_artifact, &kek) + .await + .expect("legacy artifact decrypts"); + assert_eq!(decrypted.as_slice(), legacy_bytes.as_slice(), "legacy record must travel verbatim"); + } + + #[tokio::test] + async fn record_identity_mismatch_aborts_export() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("good", "AES_256", None).await.expect("create key"); + std::fs::copy(client.key_directory().join("good.key"), client.key_directory().join("evil.key")) + .expect("plant mismatched record"); + + let bundle = TempDir::new().expect("bundle dir"); + let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect_err("identity mismatch must abort the export"); + assert!(matches!(error, KmsError::InvalidKey { .. }), "got {error:?}"); + } +} diff --git a/crates/kms/src/backup/manifest.rs b/crates/kms/src/backup/manifest.rs index 052642428..3eff4fad0 100644 --- a/crates/kms/src/backup/manifest.rs +++ b/crates/kms/src/backup/manifest.rs @@ -355,8 +355,10 @@ struct ManifestProbe { /// /// The manifest is the authoritative description of one backup bundle: what /// was captured, under which snapshot generation, protected by which backup -/// KEK, and which restore responsibility applies. Field order is part of the -/// canonical digest form and is frozen for this format version. +/// KEK, and which restore responsibility applies. The digest's canonical +/// form is the manifest's JSON value with the digest hex emptied (see +/// [`Self::compute_digest`]); decoders verify it against the raw stored +/// bytes and never re-serialize parsed fields. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BackupManifest { @@ -415,8 +417,14 @@ impl BackupManifest { /// /// Fail-closed order: truncated or malformed input, then unknown format /// version, then a missing completeness marker, then schema decoding - /// (unknown fields, duplicate fields, missing fields), then semantic - /// validation including digest verification. + /// (unknown fields, duplicate fields, missing fields), then digest + /// verification against the raw input bytes, then semantic validation. + /// + /// The digest is verified against the bytes as stored — parsed typed + /// fields are never re-serialized for verification, so a field whose + /// string form does not round-trip byte-identically through its parsed + /// representation (timestamps in environment-dependent time zone + /// spellings, for example) cannot produce a spurious mismatch. pub fn decode(bytes: &[u8]) -> Result<Self, BackupError> { let probe: ManifestProbe = serde_json::from_slice(bytes).map_err(map_serde_error)?; if probe.format_version != Self::FORMAT_VERSION { @@ -429,7 +437,9 @@ impl BackupManifest { return Err(BackupError::incomplete_bundle("manifest has no completeness marker")); } let manifest: Self = serde_json::from_slice(bytes).map_err(map_serde_error)?; - manifest.validate()?; + manifest.validate_pre_digest()?; + Self::verify_digest_in_bytes(bytes, &manifest.manifest_digest)?; + manifest.validate_content()?; Ok(manifest) } @@ -449,23 +459,30 @@ impl BackupManifest { Ok(self) } - /// Compute the digest over the canonical manifest bytes. + /// Compute the digest over the canonical manifest form. /// - /// Canonical form: compact JSON serialization of this manifest with the - /// digest hex emptied. Field order is struct declaration order and is - /// frozen for format version 1, so the same manifest content always - /// hashes to the same value. + /// Canonical form: the JSON *value* of the manifest with the digest hex + /// emptied, object keys rebuilt in bytewise-sorted order at every level + /// (see [`canonicalize_value`]), then serialized compactly. The value + /// layer is what makes sealing and decoding agree byte-for-byte — a + /// decoder recovers the identical value from the raw stored bytes + /// without round-tripping any typed field through parse-and-reprint — + /// and the explicit key sort makes the bytes independent of + /// `serde_json`'s map implementation (`preserve_order` on or off). pub fn compute_digest(&self) -> Result<ContentDigest, BackupError> { let mut unsealed = self.clone(); unsealed.manifest_digest = ContentDigest::placeholder(self.manifest_digest.algorithm); - let canonical = serde_json::to_vec(&unsealed) + let value = serde_json::to_value(&unsealed) .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?; - match self.manifest_digest.algorithm { - DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)), - } + Self::digest_of_canonical_value(value, self.manifest_digest.algorithm) } - /// Verify the sealed digest against the current manifest content. + /// Verify the sealed digest against the current in-memory content. + /// + /// This is the producer-side check (sealing and [`Self::encode`]). + /// Decoders must use the raw stored bytes instead (see [`Self::decode`]): + /// re-serializing parsed fields is not guaranteed to reproduce the + /// stored spelling byte-for-byte. pub fn verify_digest(&self) -> Result<(), BackupError> { if !self.manifest_digest.is_well_formed() { return Err(BackupError::corrupted("manifest digest is not a well-formed digest value")); @@ -478,6 +495,33 @@ impl BackupManifest { Ok(()) } + /// Verify a declared digest against raw manifest bytes, normalizing only + /// through the JSON value layer and emptying the digest slot in place. + fn verify_digest_in_bytes(bytes: &[u8], declared: &ContentDigest) -> Result<(), BackupError> { + if !declared.is_well_formed() { + return Err(BackupError::corrupted("manifest digest is not a well-formed digest value")); + } + let mut value: serde_json::Value = serde_json::from_slice(bytes).map_err(map_serde_error)?; + let Some(slot) = value.get_mut("manifest_digest").and_then(|digest| digest.get_mut("hex")) else { + return Err(BackupError::corrupted("manifest has no digest slot")); + }; + *slot = serde_json::Value::String(String::new()); + if Self::digest_of_canonical_value(value, declared.algorithm)? != *declared { + return Err(BackupError::corrupted( + "manifest digest mismatch: content does not match the sealed digest", + )); + } + Ok(()) + } + + fn digest_of_canonical_value(value: serde_json::Value, algorithm: DigestAlgorithm) -> Result<ContentDigest, BackupError> { + let canonical = serde_json::to_vec(&canonicalize_value(value)) + .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?; + match algorithm { + DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)), + } + } + /// Look up a required artifact by kind, failing closed when absent. pub fn require_artifact(&self, kind: ArtifactKind) -> Result<&ArtifactDescriptor, BackupError> { self.artifacts @@ -486,11 +530,21 @@ impl BackupManifest { .ok_or_else(|| BackupError::missing_artifact(artifact_kind_name(kind))) } - /// Validate the full manifest contract. + /// Validate the full manifest contract against the in-memory content. /// - /// This is decode-side validation and also guards [`Self::encode`], so a - /// producer cannot publish a manifest a decoder would reject. + /// This guards [`Self::encode`], so a producer cannot publish a manifest + /// a decoder would reject. [`Self::decode`] runs the same checks but + /// verifies the digest against the raw input bytes instead. pub fn validate(&self) -> Result<(), BackupError> { + self.validate_pre_digest()?; + self.verify_digest()?; + self.validate_content() + } + + /// Checks that must run before any digest verification: an unknown + /// version or an unsealed bundle is reported as its own typed error, not + /// as a digest mismatch. + fn validate_pre_digest(&self) -> Result<(), BackupError> { if self.format_version != Self::FORMAT_VERSION { return Err(BackupError::UnknownVersion { found: self.format_version, @@ -500,7 +554,12 @@ impl BackupManifest { if self.completeness != CompletenessState::Complete { return Err(BackupError::incomplete_bundle("completeness marker records an in-progress bundle")); } - self.verify_digest()?; + Ok(()) + } + + /// Semantic validation of everything except version, completeness, and + /// digest integrity. + fn validate_content(&self) -> Result<(), BackupError> { require_non_empty("backup_id", &self.backup_id)?; require_non_empty("rustfs_version", &self.rustfs_version)?; require_non_empty("deployment_identity", &self.deployment_identity)?; @@ -645,6 +704,29 @@ fn map_serde_error(error: serde_json::Error) -> BackupError { } } +/// Rebuild a JSON value with object keys in bytewise-sorted order at every +/// nesting level (array element order is preserved). +/// +/// `serde_json`'s map keeps keys sorted by default but preserves insertion +/// order when the `preserve_order` feature is unified into the build by any +/// other crate. Digest bytes must not depend on that, so the ordering is +/// imposed explicitly here instead of being inherited from the map type. +fn canonicalize_value(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut entries: Vec<(String, serde_json::Value)> = map.into_iter().collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + let mut sorted = serde_json::Map::with_capacity(entries.len()); + for (key, entry) in entries { + sorted.insert(key, canonicalize_value(entry)); + } + serde_json::Value::Object(sorted) + } + serde_json::Value::Array(items) => serde_json::Value::Array(items.into_iter().map(canonicalize_value).collect()), + other => other, + } +} + #[cfg(test)] mod tests { use super::*; @@ -736,7 +818,7 @@ mod tests { /// canonical form and frozen. If serialization layout or field order /// changes, this value changes and the fixture test fails — which is the /// point: that is a format-version bump, not a patch. - const FIXTURE_DIGEST_HEX: &str = "01accb3e2bc51e12d17d1efc52ad6ab9441c50bec52c3fb29e0a9f92b725cdaa"; + const FIXTURE_DIGEST_HEX: &str = "a8b104d61c358cd75cc9a7691d2f7d2d96f73c53653bef8940a49e96dbdf7775"; fn fixture() -> String { FIXTURE.replace("SEALED_DIGEST_HEX", FIXTURE_DIGEST_HEX) @@ -1010,12 +1092,39 @@ mod tests { let with_discovery = fixture().replace("\"completeness\"", "\"capability_discovery\": [1], \"completeness\""); expect_corrupted(BackupManifest::decode(with_discovery.as_bytes()), "reserved"); - // Explicit null carries no data and is tolerated as absence; digest - // verification still passes because null slots are skipped on - // serialization. + // An explicit null slot decodes as absence at the schema layer, but + // sealed bundles never contain the key (`skip_serializing_if`), so + // inserting one after sealing is a byte-level modification and the + // raw-bytes digest check rejects it. let with_null = fixture().replace("\"completeness\"", "\"key_versions\": null, \"completeness\""); - let decoded = BackupManifest::decode(with_null.as_bytes()).expect("null reserved slot should decode"); - assert_eq!(decoded.key_versions, None); + expect_corrupted(BackupManifest::decode(with_null.as_bytes()), "digest mismatch"); + } + + #[test] + fn digest_verification_survives_non_round_tripping_timestamp_spellings() { + // A legacy `created_at` spelling (no time zone annotation) parses via + // the compat fallback and re-serializes differently ("+00:00[UTC]"), + // and host-dependent zone spellings can do the same. Digest + // verification therefore operates on the raw stored bytes and must + // never re-serialize parsed fields. + let sealed = seal(local_manifest_unsealed()); + let mut value = serde_json::to_value(&sealed).expect("manifest should convert to a value"); + value["created_at"] = serde_json::Value::String("2026-07-30T00:00:00+00:00".to_string()); + value["manifest_digest"]["hex"] = serde_json::Value::String(String::new()); + let digest = BackupManifest::digest_of_canonical_value(value.clone(), DigestAlgorithm::Sha256) + .expect("canonical digest should compute"); + value["manifest_digest"]["hex"] = serde_json::Value::String(digest.hex); + let bytes = serde_json::to_vec(&value).expect("manifest bytes"); + + let decoded = BackupManifest::decode(&bytes).expect("a non-round-tripping timestamp spelling must not break decoding"); + + // Precondition: the spelling really does not survive a typed + // round-trip — otherwise this test is vacuous. + let reserialized = serde_json::to_value(&decoded).expect("decoded manifest should convert to a value"); + assert_ne!(reserialized["created_at"], value["created_at"]); + // Which is exactly why the producer-side (in-memory) digest check + // cannot be used on decoded manifests. + assert!(decoded.verify_digest().is_err()); } #[test] diff --git a/crates/kms/src/backup/mod.rs b/crates/kms/src/backup/mod.rs index 59cdba350..d3f9134e9 100644 --- a/crates/kms/src/backup/mod.rs +++ b/crates/kms/src/backup/mod.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Backup/restore contract types for KMS state. +//! Backup/restore contracts and backup production for KMS state. //! -//! This module is contract-only: it defines the versioned backup manifest, -//! the per-backend responsibility matrix, typed failure modes, and the -//! restore dry-run report. Nothing here is wired into handlers or backends; -//! backup export, restore orchestration, and the admin API build on these -//! types in follow-up changes. +//! The contract side defines the versioned backup manifest, the per-backend +//! responsibility matrix, typed failure modes, and the restore dry-run +//! report. [`local_export`] implements the producer side for the Local +//! backend as a crate-internal API; restore orchestration and the admin API +//! build on these pieces in follow-up changes. //! //! # Bundle model //! @@ -50,6 +50,7 @@ mod capability; mod dry_run; mod error; +pub mod local_export; mod manifest; pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; @@ -57,6 +58,10 @@ pub use dry_run::{ ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, }; pub use error::BackupError; +pub use local_export::{ + BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup, + read_local_bundle_manifest, +}; pub use manifest::{ AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, From 04c59218506e012824a0a7720528f379f59cb332 Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Sat, 1 Aug 2026 01:52:00 +0800 Subject: [PATCH 41/52] fix(s3): allow CopyObject COPY request metadata (#5514) * fix(s3): allow CopyObject COPY request metadata Co-Authored-By: heihutu <heihutu@gmail.com> * fix(kms): remove stale KmsClient import Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> --- .../e2e_test/src/copy_object_metadata_test.rs | 33 ++++++++++--------- crates/kms/src/backup/local_export.rs | 1 - rustfs/src/app/object_usecase.rs | 13 -------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/crates/e2e_test/src/copy_object_metadata_test.rs b/crates/e2e_test/src/copy_object_metadata_test.rs index cfd257f06..f56fe7f41 100644 --- a/crates/e2e_test/src/copy_object_metadata_test.rs +++ b/crates/e2e_test/src/copy_object_metadata_test.rs @@ -117,9 +117,14 @@ mod tests { .key("assets/explicit-copy.js") .copy_source(format!("{bucket}/{key}")) .metadata_directive(MetadataDirective::Copy) + .customize() + .mutate_request(|request| { + request.headers_mut().insert("content-type", "application/octet-stream"); + request.headers_mut().insert("x-amz-meta-request-only", "ignored"); + }) .send() .await - .expect("explicit COPY directive failed"); + .expect("explicit COPY directive with request metadata failed"); let explicit_copy_head = client .head_object() .bucket(bucket) @@ -128,6 +133,18 @@ mod tests { .await .expect("HEAD failed after explicit COPY"); assert_eq!(explicit_copy_head.cache_control(), Some("max-age=60")); + assert_eq!(explicit_copy_head.content_type(), Some("text/javascript; charset=utf-8")); + assert_eq!( + explicit_copy_head.metadata().and_then(|metadata| metadata.get("mtime")), + Some(&"1777992333".to_string()) + ); + assert_eq!( + explicit_copy_head + .metadata() + .and_then(|metadata| metadata.get("request-only")), + None, + "COPY must ignore request metadata" + ); assert_eq!( explicit_copy_head.website_redirect_location(), None, @@ -571,20 +588,6 @@ mod tests { Some("InvalidArgument") ); - let ignored_replacement = client - .copy_object() - .bucket(bucket) - .key(key) - .copy_source(format!("{bucket}/{key}")) - .content_type("application/ignored") - .send() - .await - .expect_err("Replacement fields without REPLACE should be rejected"); - assert_eq!( - ignored_replacement.as_service_error().and_then(|error| error.code()), - Some("InvalidRequest") - ); - let unchanged = client .get_object() .bucket(bucket) diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs index d3905b31f..00bb905e8 100644 --- a/crates/kms/src/backup/local_export.rs +++ b/crates/kms/src/backup/local_export.rs @@ -560,7 +560,6 @@ async fn fsync_dir(path: &Path) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::backends::KmsClient; use crate::config::LocalConfig; use std::sync::Arc; use tempfile::TempDir; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 8d46feadd..806b07ff9 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -6072,19 +6072,6 @@ impl DefaultObjectUsecase { )); } }; - let has_replacement_metadata = metadata.is_some() - || cache_control.is_some() - || content_disposition.is_some() - || content_encoding.is_some() - || content_language.is_some() - || content_type.is_some() - || expires.is_some(); - if has_replacement_metadata && !replaces_metadata { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Replacement metadata requires the REPLACE metadata directive".to_string(), - )); - } let replacement_metadata = if replaces_metadata { validate_archive_content_encoding(&key, content_type.as_deref(), content_encoding.as_deref())?; let mut replacement_metadata = metadata.unwrap_or_default(); From 74019845c44f04c2e0c44df672060f8a6ad45d07 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 04:24:04 +0800 Subject: [PATCH 42/52] fix(kms): drop stale KmsClient trait import in local_export tests (#5519) The KmsClient trait was folded into KmsBackend (#5501), but the backup export tests merged afterwards (#5499) still imported it, breaking the crate's test build; create_key is an inherent LocalKmsClient method, so the import is simply unused. From 792f2ef2047bc8e147aaec2225a1b8b0aef0f791 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 05:28:10 +0800 Subject: [PATCH 43/52] docs(kms): add observability dashboard, alert rules and runbook (#5517) Add a Grafana dashboard for the four KMS backend operation metrics emitted at the policy choke point, Prometheus alert rules with conservative default thresholds (pending staging baseline calibration), and an operations runbook documenting the metric contract and the response procedure for each alert. Register the new alert rules file in the observability READMEs. Refs rustfs/backlog#1584 (part of rustfs/backlog#1562) --- .docker/observability/README.md | 10 + .docker/observability/README_ZH.md | 10 + .../prometheus-rules/rustfs-kms-alerts.yml | 188 +++++ deploy/observability/README.md | 2 + .../grafana/rustfs-kms-observability.json | 793 ++++++++++++++++++ docs/operations/kms-observability-runbook.md | 130 +++ 6 files changed, 1133 insertions(+) create mode 100644 .docker/observability/prometheus-rules/rustfs-kms-alerts.yml create mode 100644 deploy/observability/grafana/rustfs-kms-observability.json create mode 100644 docs/operations/kms-observability-runbook.md diff --git a/.docker/observability/README.md b/.docker/observability/README.md index ff5e0bbdb..cb4b9a881 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -60,6 +60,16 @@ The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-con | `CodecStreamingFallbackSpike` | Warning | Codec streaming fallback > 10x baseline for 10m | | `IoQueueSaturation` | Warning | IO queue utilization > 90% for 5m | +The file `prometheus-rules/rustfs-kms-alerts.yml` contains alerting rules for the KMS backend operation metrics. Thresholds are conservative defaults pending staging baseline calibration; response procedures live in `docs/operations/kms-observability-runbook.md`, and the matching dashboard is `deploy/observability/grafana/rustfs-kms-observability.json`. + +| Alert | Severity | Condition | +|-------|----------|-----------| +| `KmsBackendFatalErrors` | Critical | Fatal (non-retryable) attempt failures > 0 for 5m | +| `KmsBackendHighErrorRate` | Critical | Non-success operation ratio > 5% for 10m (with traffic guard) | +| `KmsBackendP99LatencyHigh` | Warning | Operation p99 duration (incl. retries) > 2s for 10m | +| `KmsBackendAttemptFailureSpike` | Warning | Attempt failure rate > 0.5/s for 10m | +| `KmsBackendRetryBudgetExhausted` | Warning | budget_exhausted / deadline_exceeded outcomes > 0.05/s for 10m | + ### Enabling Alert Rules Add the alert rules file to your Prometheus configuration: diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md index 97253f20c..f42674f6c 100644 --- a/.docker/observability/README_ZH.md +++ b/.docker/observability/README_ZH.md @@ -60,6 +60,16 @@ | `CodecStreamingFallbackSpike` | 警告 | Codec streaming 回退 > 10x 基线，持续 10 分钟 | | `IoQueueSaturation` | 警告 | IO 队列利用率 > 90%，持续 5 分钟 | +文件 `prometheus-rules/rustfs-kms-alerts.yml` 包含 KMS 后端操作指标的告警规则。阈值为保守默认值，待 staging 基线校准；响应流程见 `docs/operations/kms-observability-runbook.md`，配套仪表盘为 `deploy/observability/grafana/rustfs-kms-observability.json`。 + +| 告警 | 级别 | 条件 | +|------|------|------| +| `KmsBackendFatalErrors` | 严重 | fatal（不可重试）尝试失败 > 0，持续 5 分钟 | +| `KmsBackendHighErrorRate` | 严重 | 非 success 操作占比 > 5%，持续 10 分钟（含流量下限保护） | +| `KmsBackendP99LatencyHigh` | 警告 | 操作 p99 耗时（含重试）> 2s，持续 10 分钟 | +| `KmsBackendAttemptFailureSpike` | 警告 | 尝试失败率 > 0.5/s，持续 10 分钟 | +| `KmsBackendRetryBudgetExhausted` | 警告 | budget_exhausted / deadline_exceeded 结果 > 0.05/s，持续 10 分钟 | + ### 启用告警规则 在 Prometheus 配置中添加告警规则文件： diff --git a/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml b/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml new file mode 100644 index 000000000..e4f1faf03 --- /dev/null +++ b/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml @@ -0,0 +1,188 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# RustFS KMS backend — Prometheus alerting rules +# ============================================================================= +# +# Metric source: the KMS operation-policy choke point in +# crates/kms/src/policy.rs. All label values are static enum strings +# (operation, op_class, outcome, error_class); key identifiers, key material, +# and tokens never appear in labels. +# +# Response procedures: docs/operations/kms-observability-runbook.md +# +# IMPORTANT — threshold status: every numeric threshold below is a +# conservative default chosen without a production baseline. Calibrate against +# a staging baseline before relying on these alerts for paging, and prefer +# loosening over tightening until the baseline exists. Formal SLO targets are +# deliberately not encoded here (see rustfs/backlog#1584). +# +# NOTE: prometheus.yml loads /etc/prometheus/rules/*.yml — keep the .yml +# extension or the file is silently ignored by the docker-compose stack. +# +# Validate: promtool check rules rustfs-kms-alerts.yml +# ============================================================================= + +groups: + # ========================================================================== + # Critical alerts — immediate action required + # ========================================================================== + - name: rustfs-kms-critical + interval: 30s + rules: + # ------------------------------------------------------------------ + # 1. KmsBackendFatalErrors + # Any attempt failure classified as fatal (non-retryable): auth + # or permission errors, malformed requests, missing keys. The + # policy never retries these, so even a low rate means real + # operations are failing right now. + # ------------------------------------------------------------------ + - alert: KmsBackendFatalErrors + expr: | + sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m])) > 0 + for: 5m + labels: + severity: critical + component: kms + annotations: + summary: "KMS backend fatal errors on operation {{ $labels.operation }}" + description: >- + Attempt failures classified as fatal are occurring at + {{ $value | printf "%.3f" }}/s on operation + {{ $labels.operation }}. Fatal failures are not retried: + each one is a KMS backend call that failed permanently + (authentication, permissions, malformed request, or a + missing key/version). + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendfatalerrors" + + # ------------------------------------------------------------------ + # 2. KmsBackendHighErrorRate + # Sustained share of operations terminating without success + # (fatal, budget_exhausted, deadline_exceeded). The cancelled + # outcome is excluded because shutdowns legitimately produce it. + # The traffic guard keeps a single failure on a near-idle + # cluster from firing the alert. + # Threshold: 5% for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendHighErrorRate + expr: | + ( + sum(rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m])) + / + clamp_min(sum(rate(rustfs_kms_backend_operations_total[5m])), 1e-9) + ) > 0.05 + and + sum(rate(rustfs_kms_backend_operations_total[5m])) > 0.02 + for: 10m + labels: + severity: critical + component: kms + annotations: + summary: "KMS backend non-success ratio above 5% for 10m" + description: >- + {{ $value | humanizePercentage }} of KMS backend operations + are terminating in fatal, budget_exhausted, or + deadline_exceeded. Object encryption and decryption paths + depending on the KMS are degraded or failing. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate" + + # ========================================================================== + # Warning alerts — investigation needed + # ========================================================================== + - name: rustfs-kms-warning + interval: 30s + rules: + # ------------------------------------------------------------------ + # 3. KmsBackendP99LatencyHigh + # p99 wall-clock duration of whole operations (attempts plus + # backoff) is sustained above 2 seconds. Because the histogram + # includes retries, a high p99 usually means the retry policy + # is absorbing backend failures, not that every call is slow. + # Threshold: 2s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendP99LatencyHigh + expr: | + histogram_quantile(0.99, + sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])) + ) > 2 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend operation p99 latency above 2s for 10m" + description: >- + The 99th-percentile KMS backend operation duration is + {{ $value | humanizeDuration }}, including retries and + backoff. Encryption and decryption latency is leaking into + S3 request latency. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendp99latencyhigh" + + # ------------------------------------------------------------------ + # 4. KmsBackendAttemptFailureSpike + # Aggregate attempt-failure rate (all error classes) sustained + # above an absolute floor. An absolute threshold is used instead + # of an offset-1d baseline ratio because fresh deployments have + # no baseline and an empty offset vector would keep a ratio + # alert from ever firing; switch to a baseline-relative form + # (see rustfs-get-optimization-alerts.yaml for the pattern) + # once a stable staging baseline exists. + # Threshold: 0.5/s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendAttemptFailureSpike + expr: | + sum(rate(rustfs_kms_backend_attempt_failures_total[5m])) > 0.5 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend attempt failures above 0.5/s for 10m" + description: >- + KMS backend attempts are failing at + {{ $value | printf "%.2f" }}/s across all error classes. + The retry policy may still be masking these from callers — + check the error-class breakdown before it stops absorbing + them. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendattemptfailurespike" + + # ------------------------------------------------------------------ + # 5. KmsBackendRetryBudgetExhausted + # Operations are running out of retry budget (budget_exhausted) + # or operation deadline (deadline_exceeded). These surface to + # callers as failed KMS operations even though every individual + # failure was retryable — the backend is unhealthy for longer + # than the policy can bridge. + # Threshold: 0.05/s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendRetryBudgetExhausted + expr: | + sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m])) > 0.05 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend operations exhausting retry budget ({{ $labels.outcome }})" + description: >- + KMS backend operations are terminating as + {{ $labels.outcome }} at {{ $value | printf "%.3f" }}/s. + Retryable failures are outlasting the retry budget, so + callers are seeing hard failures. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted" diff --git a/deploy/observability/README.md b/deploy/observability/README.md index d06deb7b3..9c19a9bd7 100644 --- a/deploy/observability/README.md +++ b/deploy/observability/README.md @@ -9,3 +9,5 @@ Import `grafana/rustfs-node-observability.json` into Grafana and select a Promet The dashboard uses the RustFS `server` metric label introduced with the node-local observability updates. `server` represents the RustFS node identity and is preferred for RustFS node comparisons. Prometheus `instance` still identifies the scrape target and remains useful for scrape/debugging views, but dashboards that compare RustFS nodes should group and filter by `server`. During a rolling upgrade, older nodes may still emit metrics without the `server` label. Complete the rollout before using this dashboard for node-by-node comparisons. + +`grafana/rustfs-kms-observability.json` covers the KMS backend operation metrics emitted at the operation-policy choke point. Unlike the node dashboard, KMS metrics do not carry the `server` label — use `job`/`instance` or promoted OTel resource attributes to split by node. Matching Prometheus alert rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`, and the alert response procedures are documented in `docs/operations/kms-observability-runbook.md`. diff --git a/deploy/observability/grafana/rustfs-kms-observability.json b/deploy/observability/grafana/rustfs-kms-observability.json new file mode 100644 index 000000000..0810a3e4a --- /dev/null +++ b/deploy/observability/grafana/rustfs-kms-observability.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "KMS backend operation metrics emitted at the operation-policy choke point (crates/kms/src/policy.rs). All label values are static enum strings; key identifiers, key material, and tokens never appear in labels. These metrics do not carry the RustFS `server` label — use your scrape topology (job/instance or promoted OTel resource attributes) to split by node. Alert response procedures: docs/operations/kms-observability-runbook.md.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Terminal outcomes of KMS backend operations. `fatal` means a non-retryable failure ended the operation on first observation; `budget_exhausted` and `deadline_exceeded` mean retries ran out; `cancelled` is normal during shutdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Backend Operation Rate by Outcome", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Operation names are static per-call-site identifiers (e.g. vault_kv2_read_key_version, vault_transit_encrypt, vault_login). `op_class` distinguishes read_idempotent, mutating_non_idempotent, and auth operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation, op_class) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{operation}} ({{op_class}})", + "range": true, + "refId": "A" + } + ], + "title": "Backend Operation Rate by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of operations that terminated in fatal, budget_exhausted, or deadline_exceeded. The cancelled outcome is plotted separately because shutdown windows legitimately spike it. The ratio is meaningless at near-zero traffic — read it together with the operation rate panels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome!~\"success|cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "non-success (excl. cancelled)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome=\"cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "cancelled", + "range": true, + "refId": "B" + } + ], + "title": "Non-Success Outcome Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-attempt failures by retry classification. retryable_conn covers connection-level failures, retryable_status covers retryable backend status codes, attempt_timeout covers attempts cut off by the per-attempt timeout, and fatal covers non-retryable failures (auth, permission, malformed request). Sustained fatal traffic is always actionable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{error_class}}", + "range": true, + "refId": "A" + } + ], + "title": "Attempt Failure Rate by Error Class", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Wall-clock duration of whole operations, including retries and backoff sleeps. A rising p99 with a flat p50 usually means a slow retry tail (backend degradation), not a uniform slowdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "refId": "B" + } + ], + "title": "Operation Duration p50 / p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "p99 duration split by operation. Auth operations (vault_login, vault_token_renew) and mutating writes are expected to sit higher than idempotent reads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operation Duration p99 by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Attempts one operation used before completing. Healthy operations average close to 1. A rising average or p99 means the retry policy is absorbing backend failures — read together with the attempt-failure panel to see the failure class.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(rustfs_kms_backend_operation_attempts_sum{operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum by (operation) (rate(rustfs_kms_backend_operation_attempts_count{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "{{operation}} avg", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_attempts_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p99 (all operations)", + "range": true, + "refId": "B" + } + ], + "title": "Operation Attempts Distribution", + "type": "timeseries" + }, + { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Placeholder for KMS metrics planned by sibling changes of rustfs/backlog#1584 that have **not landed yet**. Do not add panels for these names until the emitting code is merged, or the panels will render empty and mask real gaps.\n\n- TODO: key-cache effectiveness — hit/miss counters, entry gauge, eviction counter (replaces the former hardcoded-zero miss stat).\n- TODO: key lifecycle — pending-deletion/tombstone gauges from the deletion worker sweep, aggregate rotation-age gauge.\n- TODO: Vault credentials — token TTL remaining and fail-closed state gauges.\n- TODO: synthetic backend probe — probe outcome/failure-class metrics feeding the readiness cache.\n\nWhen a family lands, replace one bullet with a real panel and keep this list in sync with docs/operations/kms-observability-runbook.md (Coverage gaps).", + "mode": "markdown" + }, + "title": "Planned Panels (TODO — metrics not landed yet)", + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "rustfs", + "observability", + "kms" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_kms_backend_operations_total, operation)", + "hide": 0, + "includeAll": true, + "label": "KMS operation", + "multi": true, + "name": "operation", + "options": [], + "query": "label_values(rustfs_kms_backend_operations_total, operation)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RustFS KMS Observability", + "uid": "rustfs-kms-observability", + "version": 1, + "weekStart": "" +} diff --git a/docs/operations/kms-observability-runbook.md b/docs/operations/kms-observability-runbook.md new file mode 100644 index 000000000..310df6cbe --- /dev/null +++ b/docs/operations/kms-observability-runbook.md @@ -0,0 +1,130 @@ +# KMS observability runbook + +This runbook covers the KMS backend operation metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md). + +## Metric reference + +All four metrics are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through. Label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression. + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `rustfs_kms_backend_operations_total` | counter | `operation`, `op_class`, `outcome` | Operations executed under the operation policy, counted once per terminal outcome | +| `rustfs_kms_backend_attempt_failures_total` | counter | `operation`, `error_class` | Individual failed attempts, including attempts the retry policy later absorbed | +| `rustfs_kms_backend_operation_duration_seconds` | histogram | `operation`, `outcome` | Wall-clock duration of a whole operation, including retries and backoff sleeps | +| `rustfs_kms_backend_operation_attempts` | histogram | `operation`, `outcome` | Number of attempts one operation used before completing | + +Label values: + +- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `cancelled` (shutdown or caller cancellation). +- `op_class`: `read_idempotent` (safe to retry), `mutating_non_idempotent` (never replayed — a retryable failure terminates after a single attempt because the server may have processed the request), `auth` (login and token renewal). +- `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version). +- `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`. + +Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. These metrics do not carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`). + +Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of KMS series on a cluster using those backends is expected, not an outage. + +## Dashboard + +Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` (Prometheus data source) and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed there. + +The dashboard's "Planned Panels (TODO)" text panel lists metric families designed under rustfs/backlog#1584 but not yet landed (key-cache effectiveness, lifecycle gauges, token TTL, synthetic probe). Do not add panels or alerts for those names until the emitting code is merged; keep that panel and the [Coverage gaps](#coverage-gaps-and-planned-metrics) section below in sync as they land. + +## Alert rules + +The rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. The docker-compose Prometheus loads `/etc/prometheus/rules/*.yml`, so the `.yml` extension is load-bearing. Validate edits with `promtool check rules rustfs-kms-alerts.yml`. + +Every threshold in that file is a conservative default chosen without a production baseline; see [Threshold calibration](#threshold-calibration) before treating a firing alert as an SLO breach or a quiet one as health. + +## Alert response procedures + +### KmsBackendFatalErrors + +Meaning: attempts are failing with `error_class="fatal"` — failures the policy never retries. Each one is a KMS backend call that failed permanently (authentication, permissions, malformed request, or a missing key/version), so callers are seeing errors right now. This is the highest-signal KMS alert: fatal failures do not appear as background noise in a healthy system. + +Investigation: + +1. Break the rate down by operation: `sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m]))`. +2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs. +3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the minimal policy in [KMS backend security properties](kms-backend-security.md) (a policy that drifted or was re-scoped produces 403s that classify as fatal), and check the Vault audit log for the corresponding denied requests. +4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the rotation retention preconditions in [KMS backend security properties](kms-backend-security.md) and verify nobody destroyed version records under the key subtree. +5. Confirm blast radius with the outcome view: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="fatal"}[5m]))`. + +Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operation Rate by Outcome" dashboard panels; Vault server audit and server logs; S3-level 5xx on encrypted buckets. + +### KmsBackendHighErrorRate + +Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, or `deadline_exceeded`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page. + +Investigation: + +1. Break the failures down by outcome: `sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))`. +2. If `fatal` dominates, follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). +3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted) — the backend is unavailable or too slow for longer than the retry policy can bridge. +4. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured. + +Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts in this runbook. + +### KmsBackendP99LatencyHigh + +Meaning: the p99 wall-clock duration of KMS operations is sustained above 2s. The histogram includes retries and backoff sleeps, so a high p99 with a healthy p50 usually means a slow retry tail (a subset of calls failing and being retried), not a uniform slowdown. + +Investigation: + +1. Compare p50 and p99 on the "Operation Duration p50 / p99" panel. Flat p50 with elevated p99 points at retries; both elevated points at the backend or the network path being uniformly slow. +2. Split by operation with `histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])))` to see whether one backend call or all of them regressed. +3. Check the attempts histogram: an average meaningfully above 1 confirms the latency is retry-driven; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the failure classes. +4. If latency is not retry-driven, check the network path to Vault (TLS handshakes, DNS, proxies) and Vault's own telemetry (storage backend latency, load). +5. Remember that this latency sits inside S3 request latency for encrypted objects: sustained p99 near the operation deadline will start converting into `deadline_exceeded` outcomes. + +Related signals: the "Operation Duration p99 by Operation" and "Operation Attempts Distribution" panels; `KMS backend attempt failed with a retryable error; backing off before retry` warnings (fields: `operation`, `attempt`, `error_class`, `backoff`). + +### KmsBackendAttemptFailureSpike + +Meaning: individual attempts are failing at a sustained rate across all error classes. The retry policy may still be absorbing these — operations can keep succeeding while this alert fires — but the system is burning retry budget and running degraded, and a small further degradation will surface to callers. + +Investigation: + +1. Break the rate down by class: `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total[5m]))`. +2. `retryable_conn`: network-level failures — check connectivity, TLS, DNS, and whether Vault is down or restarting. +3. `retryable_status`: the backend answered with a retryable error — check Vault health and seal status (a sealed Vault returns 503, which lands here), and Vault-side rate limiting. +4. `attempt_timeout`: attempts are being cut off by the per-attempt timeout — either the backend is slow (correlate with [KmsBackendP99LatencyHigh](#kmsbackendp99latencyhigh)) or the configured attempt timeout is too tight for the deployment's network path. +5. `fatal`: follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). +6. Grep RustFS logs for `KMS backend attempt failed with a retryable error; backing off before retry` — the structured fields (`operation`, `attempt`, `error_class`, `backoff`) identify which call sites are cycling. + +Related signals: the "Attempt Failure Rate by Error Class" panel; the attempts histogram average rising above 1. + +### KmsBackendRetryBudgetExhausted + +Meaning: operations are terminating as `budget_exhausted` or `deadline_exceeded` — every individual failure was retryable, but the backend stayed unhealthy for longer than the retry policy could bridge, so callers received hard failures. + +Investigation: + +1. Identify the failing operations: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m]))`. +2. Establish how long the underlying failure has persisted from the attempt-failure rate history; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the class-specific diagnosis. +3. Note the by-design case: `mutating_non_idempotent` operations (e.g. `vault_kv2_cas_write_key`, `vault_transit_create_key`) are never replayed, so a single retryable failure terminates them as `budget_exhausted` after one attempt. A spike confined to mutating operations means write-path failures, not an exhausted retry loop. +4. `deadline_exceeded` clustering with duration p99 near the operation deadline means the budget is being spent on slow attempts rather than fast failures — treat as a latency problem first. +5. Confirm client impact and, if the backend outage is confirmed external (Vault down), coordinate recovery there; RustFS will resume without intervention once the backend recovers. + +Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings in RustFS logs; Vault availability monitoring. + +## Threshold calibration + +Every numeric threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584). + +## Coverage gaps and planned metrics + +The following families are designed under rustfs/backlog#1584 but land in separate changes; they are intentionally absent from the dashboard and alert rules, and referencing their names before the emitting code merges would produce permanently-empty panels and never-firing alerts: + +- Key-cache effectiveness: hit/miss counters, entry gauge, eviction counter (replacing the former hardcoded-zero miss statistic). +- Key lifecycle: pending-deletion/tombstone gauges from the deletion-worker sweep and an aggregate rotation-age gauge. +- Vault credentials: token TTL remaining and fail-closed state gauges (today only the log warnings quoted above exist). +- Synthetic backend probe: probe outcome and failure-class metrics feeding the readiness cache. + +When one of these lands, add its panels, extend the alert rules, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel, and update this section. + +## Related documents + +- [KMS backend security properties](kms-backend-security.md) — backend trust boundaries, minimal Vault policies, rotation retention preconditions. +- [Vault KMS authentication runbook](vault-kms-authentication.md) — credential sources, refresh behavior, and the fail-closed window. +- `deploy/observability/README.md` — dashboard import notes for all RustFS dashboards. From c09d11ff3be36070f9cc929bd19c030eb6c1ec5d Mon Sep 17 00:00:00 2001 From: cxymds <cxymds@gmail.com> Date: Sat, 1 Aug 2026 08:33:46 +0800 Subject: [PATCH 44/52] fix(config): fence persisted config updates and reloads (#5512) * fix(config): fence persisted config updates and reloads * fix(ci): unblock config and e2e checks Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com> --- crates/ecstore/src/api/mod.rs | 13 +- crates/ecstore/src/config/com.rs | 695 ++++++++++++++++-- crates/ecstore/src/set_disk/mod.rs | 225 +++++- crates/notify/src/config_manager.rs | 201 ++--- crates/notify/src/lib.rs | 5 +- crates/notify/src/storage_api.rs | 36 +- .../admin/handlers/audit_runtime_config.rs | 187 ++++- rustfs/src/admin/handlers/config_admin.rs | 343 ++++----- rustfs/src/admin/service/config.rs | 446 +++++++++-- rustfs/src/admin/storage_api.rs | 25 +- scripts/e2e-run.sh | 14 +- 11 files changed, 1711 insertions(+), 479 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 15aebe9d9..cfba93f88 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -267,12 +267,13 @@ pub mod config { pub mod com { pub use crate::config::com::{ COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS, - ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs, - read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate, - read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config, - save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock, - save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock, - with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock, + ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, + is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata, + read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock, + read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config, + save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation, + server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock, + with_server_config_read_lock, with_server_config_write_lock, }; } diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 539464173..c62aa74d8 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -53,12 +53,14 @@ use std::sync::LazyLock; use std::sync::{Arc, RwLock}; use tokio::sync::{OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; use tracing::{debug, error, info, instrument, warn}; +use uuid::Uuid; pub const CONFIG_PREFIX: &str = "config"; const SERVER_CONFIG_OBJECT: &str = "config/config.json"; +const CONFIG_TRANSACTION_LOCK_SUFFIX: &str = ".transaction.lock"; -// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock -// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order. +// Server-config lock order: SERVER_CONFIG_LOCK -> transaction lock -> +// SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order. static SERVER_CONFIG_LOCK: LazyLock<Arc<AsyncRwLock<()>>> = LazyLock::new(|| Arc::new(AsyncRwLock::new(()))); fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error { @@ -76,8 +78,11 @@ where T: Send + 'static, { tokio::spawn(async move { - // Lock order: SERVER_CONFIG_LOCK -> namespace write lock. + // Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock. let _local_guard = SERVER_CONFIG_LOCK.write().await; + let transaction_lock = server_config_transaction_lock_path(); + let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; + let _transaction_guard = transaction_lock.get_write_lock(get_lock_acquire_timeout()).await?; let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?; Ok(operation().await) @@ -96,8 +101,11 @@ where T: Send + 'static, { tokio::spawn(async move { - // Lock order: SERVER_CONFIG_LOCK -> namespace read lock. + // Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock. let _local_guard = SERVER_CONFIG_LOCK.read().await; + let transaction_lock = server_config_transaction_lock_path(); + let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; + let _transaction_guard = transaction_lock.get_read_lock(get_lock_acquire_timeout()).await?; let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?; Ok(operation().await) @@ -567,6 +575,21 @@ where } pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + save_config_with_opts_and_metadata(api, file, data, opts).await.map(|_| ()) +} + +async fn save_config_with_opts_and_metadata<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<ObjectInfo> where S: ObjectIO< Error = Error, @@ -579,11 +602,13 @@ where >, { let mut put_data = PutObjReader::from_vec(data); - if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await { - error!("save_config_with_opts: err: {:?}, file: {}", err, file); - return Err(err); + match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await { + Ok(object_info) => Ok(object_info), + Err(err) => { + error!("save_config_with_opts: err: {:?}, file: {}", err, file); + Err(err) + } } - Ok(()) } fn new_server_config() -> Config { @@ -594,8 +619,12 @@ async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config> where S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>, { + let snapshot = read_server_config_snapshot(api.clone()).await?; + if snapshot.object_exists() { + return Ok(snapshot.config.clone()); + } let cfg = new_server_config(); - save_server_config(api, &cfg).await?; + save_server_config_snapshot(api, &cfg, &snapshot).await?; Ok(cfg) } @@ -617,6 +646,10 @@ pub fn server_config_path() -> String { SERVER_CONFIG_OBJECT.to_string() } +fn server_config_transaction_lock_path() -> String { + format!("{}{CONFIG_TRANSACTION_LOCK_SUFFIX}", server_config_path()) +} + fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS { let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| { let mut section = HashMap::new(); @@ -819,6 +852,9 @@ fn apply_external_scalar_config_map( let Some(config_value) = root.get(descriptor.subsystem_key) else { return Ok(false); }; + if descriptor.subsystem_key == HEAL_SUB_SYS && config_value.is_null() { + return Ok(false); + } let overrides = decode_scalar_config_value(config_value, descriptor)?; if overrides.is_empty() { @@ -1463,21 +1499,142 @@ fn build_audit_object(cfg: &Config) -> Map<String, Value> { build_target_object(cfg, &audit_target_descriptors()) } +fn sync_rendered_target_instance(existing: Value, rendered: Option<&Value>, valid_keys: &[&str]) -> Option<Value> { + match existing { + Value::Object(mut instance) => { + for key in valid_keys { + instance.remove(*key); + } + if let Some(Value::Object(rendered)) = rendered { + instance.extend(rendered.clone()); + } + (!instance.is_empty()).then_some(Value::Object(instance)) + } + Value::Array(entries) => { + let mut pending = rendered + .and_then(Value::as_object) + .map(|rendered| { + rendered + .iter() + .filter_map(|(key, value)| parse_target_scalar_value(key, value).map(|value| (key.clone(), value))) + .collect::<HashMap<_, _>>() + }) + .unwrap_or_default(); + let mut updated = Vec::with_capacity(entries.len().saturating_add(pending.len())); + for entry in entries { + let Some(entry_obj) = entry.as_object() else { + updated.push(entry); + continue; + }; + let Some(key) = entry_obj.get("key").and_then(Value::as_str) else { + updated.push(entry); + continue; + }; + if !valid_keys.contains(&key) { + updated.push(entry); + continue; + } + let Some(value) = pending.remove(key) else { + continue; + }; + let mut entry_obj = entry_obj.clone(); + entry_obj.insert("value".to_string(), Value::String(value)); + updated.push(Value::Object(entry_obj)); + } + updated.extend(rendered_scalar_config_kvs_entries(&pending)); + (!updated.is_empty()).then_some(Value::Array(updated)) + } + value if rendered.is_none() => Some(value), + _ => rendered.cloned(), + } +} + fn sync_rendered_target_object( target_obj: &mut Map<String, Value>, rendered_target: &Map<String, Value>, descriptors: &[TargetConfigDescriptor], ) { for descriptor in descriptors { - match rendered_target.get(descriptor.external_key) { - Some(Value::Object(v)) => { - target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone())); - target_obj.remove(descriptor.subsystem_key); + let existing = target_obj.remove(descriptor.external_key); + let alias = target_obj.remove(descriptor.subsystem_key); + let mut section = existing + .or(alias) + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + let rendered = rendered_target.get(descriptor.external_key).and_then(Value::as_object); + + if is_target_instance_shorthand(§ion, descriptor.valid_keys) { + let has_named_instances = rendered.is_some_and(|instances| instances.keys().any(|name| name != "default")); + if !has_named_instances { + if let Some(section) = sync_rendered_target_instance( + Value::Object(section), + rendered.and_then(|instances| instances.get("default")), + descriptor.valid_keys, + ) { + target_obj.insert(descriptor.external_key.to_string(), section); + } + continue; } - _ => { - target_obj.remove(descriptor.external_key); - target_obj.remove(descriptor.subsystem_key); + + let mut nested = Map::new(); + if let Some(default) = sync_rendered_target_instance( + Value::Object(section), + rendered.and_then(|instances| instances.get("default")), + descriptor.valid_keys, + ) { + nested.insert("default".to_string(), default); } + if let Some(rendered) = rendered { + for (instance_name, instance) in rendered { + if instance_name != "default" { + nested.insert(instance_name.clone(), instance.clone()); + } + } + } + if !nested.is_empty() { + target_obj.insert(descriptor.external_key.to_string(), Value::Object(nested)); + } + continue; + } + + if let Some(default_alias) = section.remove(DEFAULT_DELIMITER) { + if let Some(default) = section.get_mut("default") { + if let Some(alias) = sync_rendered_target_instance(default_alias, None, descriptor.valid_keys) { + match (default, alias) { + (Value::Object(default), Value::Object(alias)) => { + for (key, value) in alias { + default.entry(key).or_insert(value); + } + } + (Value::Array(default), Value::Array(alias)) => default.extend(alias), + _ => {} + } + } + } else { + section.insert("default".to_string(), default_alias); + } + } + + let mut merged = Map::new(); + for (instance_name, instance) in section { + if let Some(instance) = sync_rendered_target_instance( + instance, + rendered.and_then(|instances| instances.get(&instance_name)), + descriptor.valid_keys, + ) { + merged.insert(instance_name, instance); + } + } + if let Some(rendered) = rendered { + for (instance_name, instance) in rendered { + if !merged.contains_key(instance_name) { + merged.insert(instance_name.clone(), instance.clone()); + } + } + } + + if !merged.is_empty() { + target_obj.insert(descriptor.external_key.to_string(), Value::Object(merged)); } } } @@ -1496,6 +1653,14 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8 Some(Value::Object(v)) => v, _ => Map::new(), }; + for key in [ + storageclass::CLASS_STANDARD, + storageclass::CLASS_RRS, + storageclass::OPTIMIZE, + storageclass::INLINE_BLOCK, + ] { + sc_obj.remove(key); + } for (k, v) in build_storageclass_object(cfg) { sc_obj.insert(k, v); } @@ -1503,7 +1668,10 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8 root.remove("storage_class"); for descriptor in [scanner_config_descriptor(), heal_config_descriptor()] { - let existing = root.remove(descriptor.subsystem_key); + let mut existing = root.remove(descriptor.subsystem_key); + if descriptor.subsystem_key == HEAL_SUB_SYS && existing.as_ref().is_some_and(Value::is_null) { + existing = None; + } let rendered = build_scalar_config_object(cfg, descriptor); if let Some(config_value) = sync_rendered_scalar_config_value(existing, &rendered, descriptor)? { root.insert(descriptor.subsystem_key.to_string(), config_value); @@ -1560,6 +1728,7 @@ fn is_standard_object_server_config(data: &[u8]) -> bool { matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty()) && matches!(root.get("storageclass"), Some(Value::Object(_))) && !root.contains_key("storage_class") + && !matches!(root.get(HEAL_SUB_SYS), Some(Value::Null)) } fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool { @@ -1593,7 +1762,7 @@ where FileInfo = FileInfo, ObjectToDelete = ObjectToDelete, DeletedObject = DeletedObject, - >, + > + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>, { if let Some(decrypt) = &decrypt_fn { register_server_config_decrypt_fn(decrypt.clone()); @@ -1601,14 +1770,7 @@ where let config_file = server_config_path(); match api - .get_object_info( - RUSTFS_META_BUCKET, - &config_file, - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) + .get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default()) .await { Ok(_) => { @@ -1624,7 +1786,6 @@ where let opts = ObjectOptions { max_parity: true, - no_lock: true, ..Default::default() }; @@ -1677,7 +1838,33 @@ where } }; - match save_config(api, &config_file, normalized).await { + let snapshot = match read_server_config_snapshot(api.clone()).await { + Ok(snapshot) => snapshot, + Err(err) => { + warn!("recheck target server config failed, skip migration: {:?}", err); + return; + } + }; + if snapshot.object_exists() { + debug!("server config was created while legacy migration was preparing, skip migration"); + return; + } + + match save_config_with_opts( + api, + &config_file, + normalized, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { Ok(()) => { info!("Migrated compatible server config from legacy metadata bucket"); } @@ -1769,8 +1956,13 @@ where { let config_file = server_config_path(); - // Try to read the configuration file - match read_config_no_lock(api.clone(), &config_file).await { + // Try to read the configuration file. + let data = if namespace_lock_held { + read_config_no_lock(api.clone(), &config_file).await + } else { + read_config(api.clone(), &config_file).await + }; + match data { Ok(data) => read_server_config(api, &data, namespace_lock_held).await, Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await, Err(err) => handle_config_read_error(err, &config_file), @@ -1787,7 +1979,12 @@ where warn!("Received empty configuration data, try to reread from '{}'", config_file); // Try to read the configuration again - match read_config_no_lock(api.clone(), &config_file).await { + let data = if namespace_lock_held { + read_config_no_lock(api.clone(), &config_file).await + } else { + read_config(api.clone(), &config_file).await + }; + match data { Ok(cfg_data) => { let cfg = decode_persisted_server_config(&cfg_data)?; return Ok(cfg.merge()); @@ -2036,11 +2233,16 @@ pub struct ServerConfigSnapshot { raw: Option<Vec<u8>>, seed: Option<Vec<u8>>, etag: Option<String>, + generation: Option<Uuid>, _local_guard: OwnedRwLockWriteGuard<()>, _guard: rustfs_lock::NamespaceLockGuard, } impl ServerConfigSnapshot { + pub fn object_exists(&self) -> bool { + self.raw.is_some() + } + pub fn ensure_lock_held(&self) -> Result<()> { if self._guard.is_lock_lost() { return Err(Error::other("server config transaction lock was lost")); @@ -2051,12 +2253,34 @@ impl ServerConfigSnapshot { pub fn is_lock_lost(&self) -> bool { self._guard.is_lock_lost() } + + pub fn generation(&self) -> Option<Uuid> { + self.generation + } } -/// Read a server config transaction snapshot while holding the same local and -/// distributed write locks used by every other server-config writer. Internal -/// reads and the later conditional write use no-lock object I/O; the guards -/// remain live until the snapshot is dropped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerConfigSaveResult { + persisted: bool, + generation: Option<Uuid>, +} + +impl ServerConfigSaveResult { + pub fn persisted(&self) -> bool { + self.persisted + } + + pub fn generation(&self) -> Option<Uuid> { + self.generation + } +} + +/// Read a server config transaction snapshot while holding a dedicated +/// transaction lock. The config object's normal namespace lock remains +/// available to fence reads and the conditional write at commit time. +/// The transaction guard remains live until the snapshot is dropped, +/// serializing persistence and history ordering across admin nodes. Runtime +/// state is reloaded from the durable object after this guard is released. pub async fn read_server_config_snapshot<S>(api: Arc<S>) -> Result<ServerConfigSnapshot> where S: ObjectIO< @@ -2071,12 +2295,10 @@ where { let config_file = server_config_path(); let local_guard = SERVER_CONFIG_LOCK.clone().write_owned().await; - let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &config_file).await?; + let transaction_lock = server_config_transaction_lock_path(); + let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?; - let read_options = ObjectOptions { - no_lock: true, - ..Default::default() - }; + let read_options = ObjectOptions::default(); match read_config_with_metadata_inner(api, &config_file, &read_options, true).await { Ok((raw, object_info)) => { let (config, seed) = decode_persisted_server_config_with_seed(&raw)?; @@ -2085,6 +2307,7 @@ where raw: Some(raw), seed: Some(seed), etag: object_info.etag, + generation: object_info.data_dir.filter(|generation| !generation.is_nil()), _local_guard: local_guard, _guard: guard, }) @@ -2094,6 +2317,7 @@ where raw: None, seed: None, etag: None, + generation: None, _local_guard: local_guard, _guard: guard, }), @@ -2108,6 +2332,27 @@ where /// lock, so a concurrent update or transaction lease loss cannot commit an /// unfenced overwrite. pub async fn save_server_config_snapshot<S>(api: Arc<S>, cfg: &Config, snapshot: &ServerConfigSnapshot) -> Result<bool> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>, +{ + save_server_config_snapshot_with_generation(api, cfg, snapshot) + .await + .map(|result| result.persisted()) +} + +pub async fn save_server_config_snapshot_with_generation<S>( + api: Arc<S>, + cfg: &Config, + snapshot: &ServerConfigSnapshot, +) -> Result<ServerConfigSaveResult> where S: ObjectIO< Error = Error, @@ -2126,13 +2371,19 @@ where && configs_semantically_equal(&snapshot.config, cfg) { debug!("server config unchanged and already in standard object shape, skip write"); - return Ok(false); + return Ok(ServerConfigSaveResult { + persisted: false, + generation: snapshot.generation(), + }); } let data = encode_server_config_blob(cfg, snapshot.seed.as_deref())?; if snapshot.raw.as_deref().is_some_and(|current| current == data.as_slice()) { debug!("server config bytes unchanged after encode, skip write"); - return Ok(false); + return Ok(ServerConfigSaveResult { + persisted: false, + generation: snapshot.generation(), + }); } let http_preconditions = if snapshot.raw.is_some() { @@ -2152,19 +2403,22 @@ where } }; - save_config_with_opts( + snapshot.ensure_lock_held()?; + let object_info = save_config_with_opts_and_metadata( api, &config_file, data, &ObjectOptions { max_parity: true, - no_lock: true, http_preconditions: Some(http_preconditions), ..Default::default() }, ) .await?; - Ok(true) + Ok(ServerConfigSaveResult { + persisted: true, + generation: object_info.data_dir.filter(|generation| !generation.is_nil()), + }) } /// Saves the server config while an upper layer holds the namespace write @@ -2301,8 +2555,9 @@ mod tests { use super::{ SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config, - lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate, - read_server_config_snapshot, save_server_config, save_server_config_snapshot, server_config_path, storage_class_kvs_mut, + lookup_configs, new_and_save_server_config, read_config, read_config_preserve_empty, read_config_with_metadata, + read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot, + save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut, }; use crate::config::{audit, heal, notify, oidc, scanner}; use crate::disk::endpoint::Endpoint; @@ -2311,7 +2566,9 @@ mod tests { use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::runtime::sources as runtime_sources; use crate::set_disk::SetDisks; - use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, range::HTTPRangeSpec}; + use crate::storage_api_contracts::{ + admin::StorageAdminApi, namespace::NamespaceLocking as _, object::HTTPPreconditions, range::HTTPRangeSpec, + }; use http::HeaderMap; use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; use rustfs_config::notify::{ @@ -3104,6 +3361,85 @@ mod tests { ); } + #[test] + fn root_heal_null_decodes_as_no_override_and_is_canonicalized_on_save() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "heal":null, + "future_root":{"mode":"keep"}, + "openid":{"default":{ + "config_url":"https://issuer.example/.well-known/openid-configuration", + "client_id":"console", + "client_secret":"oidc-secret", + "future_provider_control":"keep" + }}, + "notify":{"webhook":{"primary":{ + "enable":true, + "endpoint":"https://notify.example/hook", + "auth_token":"notify-secret", + "future_notify_control":"keep" + }}}, + "logger":{"webhook":{"primary":{ + "enable":true, + "endpoint":"https://audit.example/hook", + "auth_token":"audit-secret", + "future_audit_control":"keep" + }}} + }"#; + + let cfg = decode_server_config_blob(seed).expect("root heal null should mean no persisted override"); + assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none()); + assert!(!is_standard_object_server_config(seed)); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("legacy seed should canonicalize on an authorized save"); + let value: Value = serde_json::from_slice(&encoded).expect("canonical config should be valid JSON"); + assert!(value.get(HEAL_SUB_SYS).is_none()); + assert_eq!(value["future_root"]["mode"].as_str(), Some("keep")); + assert_eq!(value["openid"]["default"]["client_secret"].as_str(), Some("oidc-secret")); + assert_eq!(value["openid"]["default"]["future_provider_control"].as_str(), Some("keep")); + assert_eq!(value["notify"]["webhook"]["primary"]["auth_token"].as_str(), Some("notify-secret")); + assert_eq!(value["notify"]["webhook"]["primary"]["future_notify_control"].as_str(), Some("keep")); + assert_eq!(value["logger"]["webhook"]["primary"]["auth_token"].as_str(), Some("audit-secret")); + assert_eq!(value["logger"]["webhook"]["primary"]["future_audit_control"].as_str(), Some("keep")); + assert!(is_standard_object_server_config(&encoded)); + } + + #[test] + fn invalid_scalar_and_nested_null_config_shapes_remain_rejected() { + let invalid_sections = [ + r#""scanner":null"#, + r#""heal":"""#, + r#""heal":false"#, + r#""heal":0"#, + r#""heal":{"default":null}"#, + r#""heal":{"_":null}"#, + r#""heal":{"bitrot_cycle":null}"#, + r#""heal":[{"key":"bitrot_cycle","value":null}]"#, + ]; + + for section in invalid_sections { + let input = format!(r#"{{"version":"33","storageclass":{{"standard":"","rrs":""}},{section}}}"#); + let err = decode_server_config_blob(input.as_bytes()).expect_err("invalid scalar shape must remain rejected"); + assert!( + err.to_string().contains("expected"), + "invalid section {section} returned an unrelated error: {err}" + ); + } + } + + #[test] + fn valid_heal_object_and_kvs_array_shapes_remain_accepted() { + let empty_object = br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":{}}"#; + let cfg = decode_server_config_blob(empty_object).expect("empty heal object should decode as no override"); + assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none()); + + let kvs_array = + br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":[{"key":"bitrot_cycle","value":"off"}]}"#; + let cfg = decode_server_config_blob(kvs_array).expect("heal KVS array should decode"); + assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_some()); + } + #[test] fn scanner_update_preserves_unknown_root_and_oidc_provider_fields() { let seed = br#"{ @@ -3131,6 +3467,171 @@ mod tests { assert_eq!(value["openid"]["default"]["client_id"].as_str(), Some("console")); } + #[test] + fn storageclass_reset_removes_stale_inline_block_from_seed() { + let seed = br#"{ + "version":"33", + "storageclass":{ + "standard":"EC:2", + "rrs":"EC:1", + "optimize":"availability", + "inline_block":"64KiB", + "future_storage_control":"keep" + } + }"#; + + let encoded = encode_server_config_blob(&Config::new(), Some(seed)).expect("storageclass reset should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let storageclass = value["storageclass"].as_object().expect("storageclass object"); + + assert!(storageclass.get(crate::config::storageclass::INLINE_BLOCK).is_none()); + assert_eq!(storageclass["future_storage_control"].as_str(), Some("keep")); + } + + #[test] + fn target_update_preserves_unknown_fields_without_restoring_removed_instances() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{ + "primary":{ + "enable":true, + "endpoint":"https://notify.example/old", + "auth_token":"notify-secret", + "future_control":"keep" + }, + "removed":{"enable":true,"endpoint":"https://notify.example/removed"}, + "retained":{"enable":true,"endpoint":"https://notify.example/retained","future_control":"keep"}, + "enable":{"enable":true,"endpoint":"https://notify.example/named-enable","future_control":"keep"} + }} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("target seed should decode"); + let webhook = cfg + .0 + .get_mut(NOTIFY_WEBHOOK_SUB_SYS) + .expect("notify webhook subsystem should exist"); + webhook + .get_mut("primary") + .expect("primary target should exist") + .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string()); + webhook.remove("removed"); + webhook.remove("retained"); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let webhook = value["notify"]["webhook"].as_object().expect("webhook section"); + + assert_eq!(webhook["primary"]["endpoint"].as_str(), Some("https://notify.example/new")); + assert_eq!(webhook["primary"]["auth_token"].as_str(), Some("notify-secret")); + assert_eq!(webhook["primary"]["future_control"].as_str(), Some("keep")); + assert!(webhook.get("removed").is_none()); + assert_eq!(webhook["retained"]["future_control"].as_str(), Some("keep")); + assert!(webhook["retained"].get("enable").is_none()); + assert!(webhook["retained"].get("endpoint").is_none()); + assert_eq!(webhook["enable"]["endpoint"].as_str(), Some("https://notify.example/named-enable")); + assert_eq!(webhook["enable"]["future_control"].as_str(), Some("keep")); + } + + #[test] + fn shorthand_target_update_preserves_shape_and_unknown_nested_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{ + "enable":true, + "endpoint":"https://notify.example/old", + "future_control":{"endpoint":"leave-untouched","mode":"keep"} + }} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("shorthand target should decode"); + cfg.0 + .get_mut(NOTIFY_WEBHOOK_SUB_SYS) + .and_then(|targets| targets.get_mut(DEFAULT_DELIMITER)) + .expect("default webhook target should exist") + .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string()); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("shorthand target update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let webhook = value["notify"]["webhook"].as_object().expect("webhook shorthand object"); + + assert_eq!(webhook["endpoint"].as_str(), Some("https://notify.example/new")); + assert!(webhook.get("default").is_none()); + assert_eq!(webhook["future_control"]["endpoint"].as_str(), Some("leave-untouched")); + assert_eq!(webhook["future_control"]["mode"].as_str(), Some("keep")); + let decoded = decode_server_config_blob(&encoded).expect("updated shorthand target should remain decodable"); + assert_eq!( + decoded + .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER) + .expect("updated default webhook target") + .get(rustfs_config::WEBHOOK_ENDPOINT), + "https://notify.example/new" + ); + } + + #[test] + fn target_kvs_update_preserves_unknown_entries_and_attributes() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{"primary":[ + {"key":"enable","value":"on","hidden_if_empty":false}, + {"key":"endpoint","value":"https://notify.example/old","future_attribute":"keep-endpoint"}, + {"key":"future_control","value":"keep","future_attribute":"keep-control"} + ]}} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("target KVS seed should decode"); + cfg.0 + .get_mut(NOTIFY_WEBHOOK_SUB_SYS) + .and_then(|targets| targets.get_mut("primary")) + .expect("primary webhook target should exist") + .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string()); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target KVS update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let entries = value["notify"]["webhook"]["primary"] + .as_array() + .expect("target KVS shape should be preserved"); + let endpoint = entries + .iter() + .find(|entry| entry["key"].as_str() == Some(rustfs_config::WEBHOOK_ENDPOINT)) + .expect("endpoint entry should remain"); + let future = entries + .iter() + .find(|entry| entry["key"].as_str() == Some("future_control")) + .expect("unknown target entry should remain"); + + assert_eq!(endpoint["value"].as_str(), Some("https://notify.example/new")); + assert_eq!(endpoint["future_attribute"].as_str(), Some("keep-endpoint")); + assert_eq!(future["value"].as_str(), Some("keep")); + assert_eq!(future["future_attribute"].as_str(), Some("keep-control")); + } + + #[test] + fn target_default_alias_is_canonicalized_without_losing_unknown_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{ + "_":{"enable":false,"endpoint":"https://notify.example/alias","future_alias":"keep"}, + "default":{"enable":true,"endpoint":"https://notify.example/default","future_default":"keep"} + }} + }"#; + let cfg = decode_server_config_blob(seed).expect("dual default aliases should decode"); + let expected_endpoint = cfg + .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER) + .expect("default webhook target should exist") + .get(rustfs_config::WEBHOOK_ENDPOINT); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("default alias should canonicalize"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let webhook = value["notify"]["webhook"].as_object().expect("webhook section"); + + assert!(webhook.get(DEFAULT_DELIMITER).is_none()); + assert_eq!(webhook["default"]["endpoint"].as_str(), Some(expected_endpoint.as_str())); + assert_eq!(webhook["default"]["future_alias"].as_str(), Some("keep")); + assert_eq!(webhook["default"]["future_default"].as_str(), Some("keep")); + } + #[test] fn test_scanner_config_changes_are_semantically_significant() { let baseline = Config::new(); @@ -4124,6 +4625,7 @@ mod tests { /// What reads of the config object currently return. enum RecoveryReadState { + Missing, Blob(Vec<u8>), QuorumError, } @@ -4136,6 +4638,7 @@ mod tests { heal_calls: AtomicUsize, write_calls: AtomicUsize, last_put_no_lock: AtomicBool, + last_put_preconditions: Mutex<Option<HTTPPreconditions>>, revision: AtomicUsize, drive_counts: Vec<usize>, lock_manager: Arc<rustfs_lock::GlobalLockManager>, @@ -4150,6 +4653,7 @@ mod tests { heal_calls: AtomicUsize::new(0), write_calls: AtomicUsize::new(0), last_put_no_lock: AtomicBool::new(false), + last_put_preconditions: Mutex::new(None), revision: AtomicUsize::new(1), drive_counts: vec![2], lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()), @@ -4206,6 +4710,7 @@ mod tests { _opts: &ObjectOptions, ) -> Result<GetObjectReader> { let data = match &*self.state.lock().expect("state lock poisoned") { + RecoveryReadState::Missing => return Err(Error::ConfigNotFound), RecoveryReadState::Blob(data) => data.clone(), RecoveryReadState::QuorumError => return Err(Error::ErasureReadQuorum), }; @@ -4213,6 +4718,9 @@ mod tests { size: data.len() as i64, actual_size: data.len() as i64, etag: Some(format!("config-{}", self.revision.load(Ordering::SeqCst))), + data_dir: Some(uuid::Uuid::from_u128( + u128::try_from(self.revision.load(Ordering::SeqCst)).expect("test revision should fit in u128"), + )), ..Default::default() }; Ok(GetObjectReader { @@ -4231,15 +4739,19 @@ mod tests { opts: &ObjectOptions, ) -> Result<ObjectInfo> { let current_etag = format!("config-{}", self.revision.load(Ordering::SeqCst)); + let object_exists = matches!(&*self.state.lock().expect("state lock poisoned"), RecoveryReadState::Blob(_)); if let Some(preconditions) = &opts.http_preconditions - && (preconditions.if_match_value().is_some_and(|etag| etag != current_etag) - || preconditions.if_none_match_value() == Some("*")) + && (preconditions + .if_match_value() + .is_some_and(|etag| !object_exists || etag != current_etag) + || (object_exists && preconditions.if_none_match_value() == Some("*"))) { return Err(Error::PreconditionFailed); } let mut body = Vec::new(); data.stream.read_to_end(&mut body).await?; self.last_put_no_lock.store(opts.no_lock, Ordering::SeqCst); + *self.last_put_preconditions.lock().expect("preconditions lock poisoned") = opts.http_preconditions.clone(); self.write_calls.fetch_add(1, Ordering::SeqCst); *self.state.lock().expect("state lock poisoned") = RecoveryReadState::Blob(body.clone()); let revision = self.revision.fetch_add(1, Ordering::SeqCst) + 1; @@ -4247,6 +4759,7 @@ mod tests { size: i64::try_from(body.len()).expect("test config should fit in i64"), actual_size: i64::try_from(body.len()).expect("test config should fit in i64"), etag: Some(format!("config-{revision}")), + data_dir: Some(uuid::Uuid::from_u128(u128::try_from(revision).expect("test revision should fit in u128"))), ..Default::default() }) } @@ -4284,10 +4797,18 @@ mod tests { .expect("scanner-only config change should be persisted"); assert_eq!(store.write_calls.load(Ordering::SeqCst), 1); - assert!(store.last_put_no_lock.load(Ordering::SeqCst)); + assert!(!store.last_put_no_lock.load(Ordering::SeqCst)); + let preconditions = store + .last_put_preconditions + .lock() + .expect("preconditions lock poisoned") + .clone() + .expect("existing config update must be conditional"); + assert_eq!(preconditions.if_match_value(), Some("config-1")); + assert_eq!(preconditions.if_none_match_value(), None); assert_eq!( store.lock_resources.lock().expect("lock resources mutex poisoned").as_slice(), - &[server_config_path()] + &[server_config_transaction_lock_path()] ); let decoded = read_config_without_migrate(store) .await @@ -4301,6 +4822,73 @@ mod tests { ); } + #[tokio::test] + async fn server_config_snapshot_save_returns_committed_generation() { + let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + let snapshot = read_server_config_snapshot(store.clone()) + .await + .expect("server config snapshot"); + + let result = save_server_config_snapshot_with_generation(store, &config_with_scanner_cycle("61"), &snapshot) + .await + .expect("conditional config save"); + + assert!(result.persisted()); + assert_eq!(result.generation(), Some(uuid::Uuid::from_u128(2))); + } + + #[tokio::test] + async fn missing_server_config_is_created_with_if_none_match() { + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None)); + let cfg = config_with_scanner_cycle("61"); + + save_server_config(store.clone(), &cfg) + .await + .expect("missing config should be created conditionally"); + + assert_eq!(store.write_calls.load(Ordering::SeqCst), 1); + assert!(!store.last_put_no_lock.load(Ordering::SeqCst)); + let preconditions = store + .last_put_preconditions + .lock() + .expect("preconditions lock poisoned") + .clone() + .expect("missing config create must be conditional"); + assert_eq!(preconditions.if_match_value(), None); + assert_eq!(preconditions.if_none_match_value(), Some("*")); + let persisted = read_config_without_migrate(store) + .await + .expect("created config should reload"); + assert_eq!( + persisted + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("persisted scanner config") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[tokio::test] + async fn missing_config_initialization_recheck_preserves_concurrent_config() { + let existing = config_with_scanner_cycle("73"); + let baseline = encode_server_config_blob(&existing, None).expect("existing config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + + let observed = new_and_save_server_config(store.clone()) + .await + .expect("initialization recheck should return the config created by another writer"); + + assert_eq!(store.write_calls.load(Ordering::SeqCst), 0); + assert_eq!( + observed + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("concurrent scanner config") + .get(SCANNER_CYCLE), + "73" + ); + } + #[tokio::test] async fn stale_server_config_snapshot_cannot_overwrite_newer_update() { let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); @@ -4357,7 +4945,7 @@ mod tests { let lock = rustfs_lock::NamespaceLock::new("server-config-lease-loss".to_string(), client.clone()); let guard = lock .lock_guard( - rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_path()), + rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_transaction_lock_path()), "server-config-lease-loss", std::time::Duration::from_secs(1), std::time::Duration::from_millis(120), @@ -4372,6 +4960,7 @@ mod tests { raw: Some(baseline.clone()), seed: None, etag: Some("config-0".to_string()), + generation: Some(uuid::Uuid::from_u128(1)), _local_guard: local_guard, _guard: guard, }; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 6ff7488a2..2a6e3895a 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -4714,6 +4714,7 @@ mod tests { use crate::layout::endpoints::SetupType; use crate::object_api::BLOCK_SIZE_V2; use crate::object_api::ObjectInfo; + use crate::set_disk::core::io_primitives::rename_fanout_barrier; use crate::storage_api_contracts::{ heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart, namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _, @@ -9565,6 +9566,15 @@ mod tests { make_local_bucket_test_set_disks_with_drive_count(2).await } + fn assert_exclusive_object_lock_held(set_disks: &SetDisks, bucket: &str, object: &str) { + let lock = set_disks + .local_lock_manager_for_test() + .get_lock_info(&ObjectKey::new(bucket, object)) + .expect("object lock should be visible while rename is paused"); + assert!(matches!(lock.mode, rustfs_lock::LockMode::Exclusive)); + assert_eq!(lock.owner.as_ref(), set_disks.locker_owner.as_str()); + } + async fn make_local_bucket_test_set_disks_with_drive_count(drive_count: usize) -> Arc<SetDisks> { let format = FormatV3::new(1, drive_count); let mut endpoints = Vec::new(); @@ -9599,7 +9609,9 @@ mod tests { disks.push(Some(disk)); } - let set_disks = SetDisks::new( + let instance_ctx = Arc::new(InstanceContext::new()); + instance_ctx.update_erasure_type(SetupType::Erasure).await; + let set_disks = SetDisks::new_with_instance_ctx( "test-owner".to_string(), Arc::new(RwLock::new(disks)), drive_count, @@ -9609,6 +9621,7 @@ mod tests { endpoints, format, Vec::new(), + instance_ctx, ) .await; set_disks.set_test_storage_class_config( @@ -10262,6 +10275,216 @@ mod tests { )); } + #[tokio::test] + async fn conditional_replace_holds_object_lock_through_rename() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-conditional-replace-fence"; + let object = "config/conditional-replace.json"; + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let mut initial_reader = PutObjReader::from_vec(b"initial config".to_vec()); + let initial = set_disks + .put_object( + bucket, + object, + &mut initial_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("initial config should be written"); + let initial_etag = initial.etag.expect("initial config should have an ETag"); + + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer_store = set_disks.clone(); + let expected_etag = initial_etag.clone(); + let writer = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"replacement config".to_vec()); + writer_store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + preserve_etag: Some("replacement-etag".to_string()), + http_preconditions: Some(HTTPPreconditions { + if_match: Some(expected_etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + }); + + tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("conditional replace should reach the rename barrier"); + assert_exclusive_object_lock_held(&set_disks, bucket, object); + barrier.release(); + writer + .await + .expect("conditional writer task should finish") + .expect("matching conditional replace should commit"); + assert!( + set_disks + .local_lock_manager_for_test() + .get_lock_info(&ObjectKey::new(bucket, object)) + .is_none(), + "conditional replace should release the object lock after commit" + ); + let contender = set_disks + .new_ns_lock(bucket, object) + .await + .expect("contender namespace lock should be created"); + let contender_guard = contender + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("contender should acquire after conditional replace commits"); + drop(contender_guard); + + let mut stale_reader = PutObjReader::from_vec(b"stale config".to_vec()); + let err = set_disks + .put_object( + bucket, + object, + &mut stale_reader, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some(initial_etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect_err("the old ETag must fail after the fenced replacement commits"); + assert_eq!(err, StorageError::PreconditionFailed); + } + + #[tokio::test] + async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-write-generation"; + let object = "config/write-generation.json"; + let body = b"identical config body".to_vec(); + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let mut first_reader = PutObjReader::from_vec(body.clone()); + let first = set_disks + .put_object( + bucket, + object, + &mut first_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("first config body should be written"); + let mut second_reader = PutObjReader::from_vec(body); + let second = set_disks + .put_object( + bucket, + object, + &mut second_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("identical config body should be rewritten"); + + assert_eq!(first.etag, second.etag, "content ETag should expose the ABA collision"); + assert_ne!(first.data_dir, second.data_dir, "each committed body write needs a unique generation"); + assert!(first.data_dir.is_some() && second.data_dir.is_some()); + } + + #[tokio::test] + async fn conditional_create_holds_object_lock_through_rename() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-conditional-create-fence"; + let object = "config/conditional-create.json"; + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer_store = set_disks.clone(); + let writer = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"created config".to_vec()); + writer_store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + }); + + tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("conditional create should reach the rename barrier"); + assert_exclusive_object_lock_held(&set_disks, bucket, object); + barrier.release(); + writer + .await + .expect("conditional writer task should finish") + .expect("first conditional create should commit"); + assert!( + set_disks + .local_lock_manager_for_test() + .get_lock_info(&ObjectKey::new(bucket, object)) + .is_none(), + "conditional create should release the object lock after commit" + ); + let contender = set_disks + .new_ns_lock(bucket, object) + .await + .expect("contender namespace lock should be created"); + let contender_guard = contender + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("contender should acquire after conditional create commits"); + drop(contender_guard); + + let mut duplicate_reader = PutObjReader::from_vec(b"duplicate config".to_vec()); + let err = set_disks + .put_object( + bucket, + object, + &mut duplicate_reader, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect_err("a second create-only write must not replace the committed config"); + assert_eq!(err, StorageError::PreconditionFailed); + } + #[tokio::test] async fn set_level_if_none_match_fails_closed_without_read_quorum() { let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await; diff --git a/crates/notify/src/config_manager.rs b/crates/notify/src/config_manager.rs index 91df663d0..617a7b186 100644 --- a/crates/notify/src/config_manager.rs +++ b/crates/notify/src/config_manager.rs @@ -19,7 +19,7 @@ use crate::{ resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, - with_notify_server_config_read_lock, with_notify_server_config_write_lock, + with_notify_server_config_read_lock, }; use rustfs_config::notify::{ NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, @@ -41,12 +41,32 @@ enum NotifyConfigStoreError { StorageNotAvailable, Read(String), Save(String), + Converge { persisted: bool, error: String }, +} + +fn notification_convergence_error(persisted: bool, error: impl std::fmt::Display) -> NotificationError { + let durable_state = if persisted { "persisted" } else { "unchanged" }; + NotificationError::Configuration(format!( + "configuration was {durable_state} but notification runtime convergence failed: {error}" + )) +} + +async fn supervise_config_update<T>( + mutation: impl std::future::Future<Output = Result<T, NotifyConfigStoreError>> + Send + 'static, +) -> Result<T, NotifyConfigStoreError> +where + T: Send + 'static, +{ + tokio::spawn(mutation).await.map_err(|error| { + let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" }; + NotifyConfigStoreError::Lock(format!("notify config update task {outcome}")) + })? } async fn update_server_config<F>( - modifier: F, + mut modifier: F, lifecycle: NotifyLifecycleCoordinator, -) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError> +) -> Result<Option<(crate::lifecycle::NotificationLifecycleTransition, bool)>, NotifyConfigStoreError> where F: FnMut(&mut Config) -> bool + Send + 'static, { @@ -54,51 +74,31 @@ where return Err(NotifyConfigStoreError::StorageNotAvailable); }; - let store_for_read = store.clone(); - let store_for_save = store.clone(); - with_notify_server_config_write_lock(store, move || { - read_modify_write( - modifier, - move || async move { - crate::read_notify_server_config_without_migrate_no_lock(store_for_read) - .await - .map_err(NotifyConfigStoreError::Read) - }, - move |config| async move { - crate::save_notify_server_config_no_lock(store_for_save, &config) - .await - .map_err(NotifyConfigStoreError::Save) - }, - move |config| lifecycle.update_config(config), - ) + supervise_config_update(async move { + let snapshot = crate::read_notify_server_config_snapshot(store.clone()) + .await + .map_err(NotifyConfigStoreError::Read)?; + let mut config = snapshot.config.clone(); + if !modifier(&mut config) { + return Ok(None); + } + + let persisted = crate::save_notify_server_config_snapshot(store.clone(), &config, &snapshot) + .await + .map_err(NotifyConfigStoreError::Save)?; + drop(snapshot); + + let read_store = store.clone(); + with_notify_server_config_read_lock(store, move || async move { + let latest = crate::read_existing_notify_server_config_no_lock(read_store) + .await + .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?; + Ok::<_, NotifyConfigStoreError>(Some((lifecycle.update_config(latest), persisted))) + }) + .await + .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })? }) .await - .map_err(NotifyConfigStoreError::Lock)? -} - -async fn read_modify_write<F, R, RFut, S, SFut, P, T>( - mut modifier: F, - read: R, - save: S, - publish: P, -) -> Result<Option<T>, NotifyConfigStoreError> -where - F: FnMut(&mut Config) -> bool, - R: FnOnce() -> RFut, - RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>, - S: FnOnce(Config) -> SFut, - SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>, - P: FnOnce(Config) -> T, -{ - let mut new_config = read().await?; - - if !modifier(&mut new_config) { - return Ok(None); - } - - save(new_config.clone()).await?; - - Ok(Some(publish(new_config))) } pub(crate) fn notify_configuration_hint() -> String { @@ -345,16 +345,18 @@ impl NotifyConfigManager { if self.lifecycle.state() == NotificationRuntimeState::Terminated { return Err(NotificationError::Initialization("Notification runtime has terminated".to_string())); } - let Some(transition) = update_server_config(modifier, self.lifecycle.clone()) - .await - .map_err(|err| match err { - NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err), - NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable( - "Failed to save target configuration: server storage not initialized".to_string(), - ), - NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err), - NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err), - })? + let Some((transition, persisted)) = + update_server_config(modifier, self.lifecycle.clone()) + .await + .map_err(|err| match err { + NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err), + NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable( + "Failed to save target configuration: server storage not initialized".to_string(), + ), + NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err), + NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err), + NotifyConfigStoreError::Converge { persisted, error } => notification_convergence_error(persisted, error), + })? else { debug!( event = EVENT_NOTIFY_CONFIG_UPDATE, @@ -367,23 +369,53 @@ impl NotifyConfigManager { return Ok(()); }; + let result = if persisted { "updated" } else { "unchanged" }; info!( event = EVENT_NOTIFY_CONFIG_UPDATE, component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, action = "reload_if_changed", - result = "updated", + result, "notify config update" ); - transition.wait().await + transition + .wait() + .await + .map_err(|err| notification_convergence_error(persisted, err)) } } #[cfg(test)] mod tests { - use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem}; + use super::{ + NotifyConfigManager, NotifyConfigStoreError, notification_convergence_error, runtime_target_id_for_subsystem, + supervise_config_update, + }; use crate::rules::RulesMap; use crate::{NotificationError, NotificationRuntimeState}; + + #[tokio::test] + async fn supervised_config_update_survives_waiter_cancellation() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); + + let waiter = tokio::spawn(async move { + supervise_config_update(async move { + let _ = started_tx.send(()); + let _ = release_rx.await; + let _ = completed_tx.send(()); + Ok::<_, NotifyConfigStoreError>(()) + }) + .await + }); + + started_rx.await.expect("config update should start"); + waiter.abort(); + release_tx.send(()).expect("config update should be released"); + let completed = tokio::time::timeout(std::time::Duration::from_secs(30), completed_rx).await; + assert!(matches!(completed, Ok(Ok(()))), "detached config update should complete"); + } use crate::{ integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, @@ -392,14 +424,11 @@ mod tests { NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; - use rustfs_config::server_config::{Config, KVS}; + use rustfs_config::server_config::Config; use rustfs_s3_types::EventName; use rustfs_targets::ReplayWorkerManager; use rustfs_targets::arn::TargetID; - use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }; + use std::sync::Arc; use tokio::sync::{RwLock, Semaphore}; fn build_manager() -> NotifyConfigManager { @@ -463,38 +492,6 @@ mod tests { assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. })); } - #[tokio::test] - async fn read_modify_write_publishes_only_after_save() { - let saved = Arc::new(AtomicBool::new(false)); - let saved_by_writer = saved.clone(); - let observed_by_publisher = saved.clone(); - - read_modify_write( - |config| { - config - .0 - .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string()) - .or_default() - .insert("primary".to_string(), KVS::default()); - true - }, - || async { Ok::<_, NotifyConfigStoreError>(Config::default()) }, - move |_config| async move { - saved_by_writer.store(true, Ordering::Release); - Ok::<_, NotifyConfigStoreError>(()) - }, - move |_config| { - assert!( - observed_by_publisher.load(Ordering::Acquire), - "publication must observe the completed save" - ); - }, - ) - .await - .expect("read-modify-write should succeed") - .expect("changed config should publish"); - } - #[test] fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() { let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary"); @@ -550,4 +547,16 @@ mod tests { assert_eq!(target_id.id, "audittrail"); assert_eq!(target_id.name, "postgres"); } + + #[test] + fn convergence_error_reports_durable_write_state() { + for (persisted, expected) in [(true, "configuration was persisted"), (false, "configuration was unchanged")] { + let error = notification_convergence_error(persisted, "injected failure"); + let NotificationError::Configuration(message) = error else { + panic!("convergence error should be a configuration error"); + }; + assert!(message.contains(expected), "unexpected convergence error: {message}"); + assert!(message.contains("injected failure")); + } + } } diff --git a/crates/notify/src/lib.rs b/crates/notify/src/lib.rs index df93c5a48..dcc421d88 100644 --- a/crates/notify/src/lib.rs +++ b/crates/notify/src/lib.rs @@ -57,7 +57,6 @@ pub use services::NotifyServices; pub use status_view::NotifyStatusView; pub use storage_api::NotifyStore; pub(crate) use storage_api::crate_boundary::{ - read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock, - resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock, - with_notify_server_config_write_lock, + read_existing_notify_server_config_no_lock, read_notify_server_config_snapshot, resolve_notify_object_store_handle, + save_notify_server_config_snapshot, with_notify_server_config_read_lock, }; diff --git a/crates/notify/src/storage_api.rs b/crates/notify/src/storage_api.rs index 9f4a44368..ba1ac98ea 100644 --- a/crates/notify/src/storage_api.rs +++ b/crates/notify/src/storage_api.rs @@ -15,11 +15,10 @@ use std::sync::Arc; use rustfs_ecstore::api::config::com::{ - read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock, read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock, - save_server_config_no_lock as save_notify_server_config_to_backend_no_lock, + read_server_config_snapshot as read_notify_server_config_snapshot_from_backend, + save_server_config_snapshot as save_notify_server_config_snapshot_to_backend, with_server_config_read_lock as with_notify_server_config_read_lock_from_backend, - with_server_config_write_lock as with_notify_server_config_write_lock_from_backend, }; use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend; pub use rustfs_ecstore::api::storage::ECStore as NotifyStore; @@ -28,10 +27,10 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> { resolve_notify_object_store_handle_from_backend() } -pub(crate) async fn read_notify_server_config_without_migrate_no_lock( - store: Arc<NotifyStore>, -) -> Result<rustfs_config::server_config::Config, String> { - read_notify_config_without_migrate_from_backend_no_lock(store) +pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot; + +pub(crate) async fn read_notify_server_config_snapshot(store: Arc<NotifyStore>) -> Result<NotifyServerConfigSnapshot, String> { + read_notify_server_config_snapshot_from_backend(store) .await .map_err(|err| err.to_string()) } @@ -44,22 +43,12 @@ pub(crate) async fn read_existing_notify_server_config_no_lock( .map_err(|err| err.to_string()) } -pub(crate) async fn save_notify_server_config_no_lock( +pub(crate) async fn save_notify_server_config_snapshot( store: Arc<NotifyStore>, config: &rustfs_config::server_config::Config, -) -> Result<(), String> { - save_notify_server_config_to_backend_no_lock(store, config) - .await - .map_err(|err| err.to_string()) -} - -pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String> -where - F: FnOnce() -> Fut + Send + 'static, - Fut: std::future::Future<Output = T> + Send + 'static, - T: Send + 'static, -{ - with_notify_server_config_write_lock_from_backend(store, operation) + snapshot: &NotifyServerConfigSnapshot, +) -> Result<bool, String> { + save_notify_server_config_snapshot_to_backend(store, config, snapshot) .await .map_err(|err| err.to_string()) } @@ -77,8 +66,7 @@ where pub(crate) mod crate_boundary { pub(crate) use super::{ - read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock, - resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock, - with_notify_server_config_write_lock, + read_existing_notify_server_config_no_lock, read_notify_server_config_snapshot, resolve_notify_object_store_handle, + save_notify_server_config_snapshot, with_notify_server_config_read_lock, }; } diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index b4d7e5f9f..1055d3bf3 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -15,13 +15,15 @@ use crate::admin::handlers::supervise_admin_mutation; use crate::admin::handlers::target_descriptor::AdminTargetSpec; use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context}; +use crate::admin::service::config::with_runtime_config_reload_lock; use crate::admin::storage_api::config::{ - read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot, + read_admin_config_without_migrate, read_admin_server_config_snapshot, read_existing_admin_server_config_no_lock, + save_admin_server_config_snapshot, with_admin_server_config_read_lock, }; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::server_config::Config; -use s3s::{S3Result, s3_error}; +use s3s::{S3Error, S3Result, s3_error}; use tracing::warn; pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result<Config> { @@ -48,6 +50,14 @@ fn has_any_audit_targets(specs: &[AdminTargetSpec], config: &Config) -> bool { }) } +fn audit_config_convergence_error(persisted: bool, error: impl std::fmt::Display) -> S3Error { + if persisted { + s3_error!(InternalError, "audit config persisted but runtime convergence failed: {}", error) + } else { + s3_error!(InternalError, "audit config unchanged but runtime convergence failed: {}", error) + } +} + pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config: Config) -> S3Result<()> { let has_targets = has_any_audit_targets(specs, &config); @@ -107,15 +117,23 @@ where return Ok(()); } - save_admin_server_config_snapshot(store, &config, &snapshot) + let persisted = save_admin_server_config_snapshot(store.clone(), &config, &snapshot) .await - .map(|_| ()) - .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))? + .persisted(); + drop(snapshot); - // Keep persistence and runtime publication in one detached, serialized - // mutation. Otherwise a cancelled caller or two concurrent updates can - // leave the persisted config and active audit generation disagreeing. - apply_audit_runtime_config(&specs, config).await + let read_store = store.clone(); + with_runtime_config_reload_lock(async move { + let latest = with_admin_server_config_read_lock(store, move || read_existing_admin_server_config_no_lock(read_store)) + .await + .map_err(|e| s3_error!(InternalError, "failed to lock server config for audit reload: {}", e))? + .map_err(|e| s3_error!(InternalError, "failed to read latest server config for audit reload: {}", e))?; + + apply_audit_runtime_config(&specs, latest).await + }) + .await + .map_err(|e| audit_config_convergence_error(persisted, e)) }) .await } @@ -164,3 +182,154 @@ pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsys }) .await } + +#[cfg(test)] +mod tests { + use super::*; + use crate::admin::handlers::target_descriptor::admin_target_spec_from_builtin; + use crate::admin::runtime_sources::{IamInterface, KmsInterface}; + use crate::admin::storage_api::config::save_admin_server_config; + use rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS; + use rustfs_config::server_config::KVS; + use rustfs_config::{ENABLE_KEY, EnableState, SCANNER_CYCLE, SCANNER_SUB_SYS, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR}; + use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; + use rustfs_kms::KmsServiceManager; + use rustfs_targets::catalog::builtin::builtin_audit_target_admin_descriptors; + use std::sync::Arc; + use std::time::Duration; + use tempfile::TempDir; + + struct TestIam; + + impl IamInterface for TestIam { + fn handle(&self) -> Arc<IamSys<ObjectStore>> { + unreachable!("audit config tests do not use IAM") + } + + fn is_ready(&self) -> bool { + false + } + } + + struct TestKms; + + impl KmsInterface for TestKms { + fn handle(&self) -> Arc<KmsServiceManager> { + Arc::new(KmsServiceManager::new()) + } + } + + fn audit_specs() -> Vec<AdminTargetSpec> { + builtin_audit_target_admin_descriptors() + .into_iter() + .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) + .collect() + } + + async fn wait_for_persisted_target(store: Arc<crate::admin::storage_api::runtime::ECStore>, subsystem: &str, target: &str) { + tokio::time::timeout(Duration::from_secs(30), async { + let mut poll = tokio::time::interval(Duration::from_millis(10)); + loop { + poll.tick().await; + let config = read_admin_config_without_migrate(store.clone()) + .await + .expect("read persisted server config"); + if config.0.get(subsystem).is_some_and(|targets| targets.contains_key(target)) { + return; + } + } + }) + .await + .expect("config mutation should become durable"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial_test::serial] + async fn audit_reload_reads_latest_durable_config_after_releasing_write_snapshot() { + let temp_dir = TempDir::new().expect("audit config temp dir"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp_dir.path()) + .disk_count(1) + .init_bucket_metadata(false) + .build() + .await; + save_admin_server_config(env.ecstore.clone(), &Config::new()) + .await + .expect("persist baseline server config"); + let context = Arc::new(AppContext::new(env.ecstore.clone(), Arc::new(TestIam), Arc::new(TestKms))); + + let (locked_tx, locked_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let blocker = tokio::spawn(async move { + with_runtime_config_reload_lock(async move { + locked_tx.send(()).expect("signal runtime reload lock acquisition"); + release_rx.await.expect("release runtime reload lock"); + Ok(()) + }) + .await + .expect("runtime reload lock blocker"); + }); + locked_rx.await.expect("runtime reload lock should be held"); + + let older_context = context.clone(); + let older = tokio::spawn(async move { + let specs = audit_specs(); + update_audit_config_and_reload_for_context(Some(older_context.as_ref()), &specs, |config| { + config + .0 + .entry(SCANNER_SUB_SYS.to_string()) + .or_default() + .entry(DEFAULT_DELIMITER.to_string()) + .or_insert_with(KVS::new) + .insert(SCANNER_CYCLE.to_string(), "15s".to_string()); + true + }) + .await + }); + wait_for_persisted_target(env.ecstore.clone(), SCANNER_SUB_SYS, DEFAULT_DELIMITER).await; + assert!(!older.is_finished(), "audit runtime publication must wait for the shared reload lock"); + + let snapshot = read_admin_server_config_snapshot(env.ecstore.clone()) + .await + .expect("read newer server config snapshot"); + let mut latest = snapshot.config.clone(); + let mut latest_target = KVS::new(); + latest_target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + latest_target.insert(WEBHOOK_ENDPOINT.to_string(), "https://audit.invalid/hook".to_string()); + latest_target.insert( + WEBHOOK_QUEUE_DIR.to_string(), + temp_dir.path().join("audit-queue").to_string_lossy().into_owned(), + ); + latest + .0 + .entry(AUDIT_WEBHOOK_SUB_SYS.to_string()) + .or_default() + .insert("latest".to_string(), latest_target); + save_admin_server_config_snapshot(env.ecstore.clone(), &latest, &snapshot) + .await + .expect("persist newer audit config"); + drop(snapshot); + + release_tx.send(()).expect("release runtime reload blocker"); + blocker.await.expect("runtime reload blocker task"); + tokio::time::timeout(Duration::from_secs(30), older) + .await + .expect("older audit update should complete") + .expect("older audit update task should not panic") + .expect("older audit update should converge from the latest durable config"); + + let system = audit_system().expect("latest durable audit target should start the audit system"); + let targets = system.list_targets().await; + assert!(targets.iter().any(|target| target.contains("latest")), "active targets: {targets:?}"); + system.close().await.expect("audit system should stop after the test"); + } + + #[test] + fn audit_convergence_error_reports_durable_write_state() { + for (persisted, expected) in [(true, "audit config persisted"), (false, "audit config unchanged")] { + let error = audit_config_convergence_error(persisted, "injected failure"); + assert!(error.to_string().contains(expected), "unexpected convergence error: {error}"); + assert!(error.to_string().contains("injected failure")); + } + } +} diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index b9d409090..83641b017 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -17,18 +17,17 @@ use crate::admin::handlers::supervise_admin_mutation; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ current_action_credentials, current_app_context, current_object_store_handle_for_context, current_server_config_for_context, - publish_server_config, }; use crate::admin::service::config::{ - CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN, - LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload, - prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload, - signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked, + FULL_CONFIG_WORKER_SUBSYSTEMS, is_dynamic_config_subsystem, preflight_dynamic_config_reload, prepare_server_config, + publish_latest_runtime_config_snapshot, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, + signal_config_snapshot_reload, signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked, }; use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use crate::admin::storage_api::config::{ - AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, - read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot, + AdminServerConfigSaveResult, AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, + read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, + save_admin_server_config_snapshot, }; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; @@ -766,7 +765,10 @@ async fn load_server_config_snapshot_from_store() -> S3Result<AdminServerConfigS .map_err(Into::into) } -async fn save_server_config_to_store(config: &ServerConfig, snapshot: &AdminServerConfigSnapshot) -> S3Result<bool> { +async fn save_server_config_to_store( + config: &ServerConfig, + snapshot: &AdminServerConfigSnapshot, +) -> S3Result<AdminServerConfigSaveResult> { let store = object_store()?; save_admin_server_config_snapshot(store, config, snapshot) .await @@ -987,8 +989,8 @@ fn decode_config_history_snapshot(data: &[u8]) -> S3Result<ServerConfig> { Ok(config) } -fn validate_restore_rollback_generation(current: &ServerConfig, restored: &ServerConfig) -> S3Result<()> { - if current == restored { +fn validate_restore_rollback_generation(current: Option<Uuid>, committed: Option<Uuid>) -> S3Result<()> { + if current.is_some() && current == committed { Ok(()) } else { Err(s3_error!( @@ -1004,12 +1006,12 @@ async fn save_server_config_history_snapshot(config: &ServerConfig) -> S3Result< save_server_config_history(&sealed).await } -async fn cleanup_failed_config_history_snapshot(restore_id: &str) { +async fn cleanup_config_history_snapshot(restore_id: &str) { if let Err(err) = delete_server_config_history(restore_id).await { warn!( restore_id, error = %err, - "Failed to remove config history snapshot for an uncommitted mutation" + "Failed to remove config history snapshot" ); } } @@ -1843,23 +1845,6 @@ async fn reject_lost_config_transaction<T>(snapshot: AdminServerConfigSnapshot, )) } -fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> { - prepared.publish_storage_class()?; - publish_server_config(config); - Ok(()) -} - -/// Re-apply local mutable worker families after a full-config replacement. -/// Peers receive one full-snapshot signal after this returns; signaling each -/// family here as well would recreate audit/scanner targets twice per peer. -fn publish_notify_config_intent( - config: &ServerConfig, - sub_system: Option<&str>, -) -> Option<rustfs_notify::NotificationLifecycleTransition> { - (sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system))) - .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone())) -} - fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> { if let Some(sub_system) = sub_system { return is_dynamic_config_subsystem(sub_system) @@ -1884,78 +1869,26 @@ async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> { Ok(()) } -async fn wait_notify_config_intent(transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<bool> { - let Some(transition) = transition else { - return Ok(false); - }; - transition.wait().await.map_err(|err| { - warn!(error = %err, "Failed to apply local notification config"); - s3_error!(InternalError, "failed to apply notification config") - })?; - Ok(true) -} - -async fn reload_non_notify_dynamic_subsystems() -> Vec<String> { - let mut failures = Vec::new(); - for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { - if NOTIFY_SUB_SYSTEMS.contains(&sub_system) { - continue; - } - if reload_dynamic_config_runtime_state(sub_system).await.is_err() { - failures.push(format!("local {sub_system}")); - warn!( - event = EVENT_CONFIG_WORKER_RELOAD_FAILED, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_CONFIG, - config_subsystem = sub_system, - state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - reason = "apply_failed", - "Published server config but failed to reload a local worker subsystem" - ); - } - } - failures -} - -fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> { +fn finish_config_reconciliation(errors: Vec<String>, persisted: bool) -> S3Result<()> { if errors.is_empty() { Ok(()) - } else { + } else if persisted { Err(s3_error!( InternalError, "server config persisted but runtime convergence failed: {}", errors.join("; ") )) + } else { + Err(s3_error!( + InternalError, + "server config was unchanged but runtime convergence failed: {}", + errors.join("; ") + )) } } -async fn reconcile_targeted_config( - sub_system: Option<String>, - storage_class_applied: bool, - notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>, -) -> S3Result<bool> { - let mut errors = Vec::new(); - let notify_applied = notify_transition.is_some(); - if let Err(err) = wait_notify_config_intent(notify_transition).await { - warn!(error = %err, "Local notification config failed to converge"); - errors.push("local notify".to_string()); - } - - let config_applied = if notify_applied { - if let Some(sub_system) = sub_system.as_deref() - && let Err(err) = signal_dynamic_config_reload_checked(sub_system).await - { - warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed"); - errors.push(format!("peer {sub_system}")); - } - true - } else if storage_class_applied { - if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await { - warn!(error = %err, "Peer storage-class reload failed"); - errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); - } - true - } else if let Some(sub_system) = sub_system.as_deref() +async fn reconcile_targeted_config(sub_system: Option<String>, persisted: bool, mut errors: Vec<String>) -> S3Result<bool> { + let config_applied = if let Some(sub_system) = sub_system.as_deref() && is_dynamic_config_subsystem(sub_system) { let config_applied = match reload_dynamic_config_runtime_state(sub_system).await { @@ -1979,21 +1912,19 @@ async fn reconcile_targeted_config( false }; - finish_config_reconciliation(errors)?; + finish_config_reconciliation(errors, persisted)?; Ok(config_applied) } -async fn reconcile_full_config(notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<()> { - let mut errors = Vec::new(); - if let Err(err) = wait_notify_config_intent(notify_transition).await { - warn!(error = %err, "Local notification config failed to converge"); +async fn reconcile_full_config(persisted: bool, mut errors: Vec<String>) -> S3Result<()> { + if let Err(err) = reload_dynamic_config_runtime_state(NOTIFY_WEBHOOK_SUB_SYS).await { + warn!(error = %err, "Local notification config failed to converge from durable config"); errors.push("local notify".to_string()); } if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await { warn!(error = %err, "Peer storage-class reload failed"); errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); } - errors.extend(reload_non_notify_dynamic_subsystems().await); if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await { warn!(error = %err, "Peer notification config reload failed"); errors.push("peer notify".to_string()); @@ -2002,91 +1933,82 @@ async fn reconcile_full_config(notify_transition: Option<rustfs_notify::Notifica warn!(error = %err, "Peer config snapshot reload failed"); errors.push("peer config snapshot".to_string()); } - finish_config_reconciliation(errors) + finish_config_reconciliation(errors, persisted) } struct PersistedConfigTransaction { previous_config: ServerConfig, history_restore_id: Option<String>, - storage_class_applied: bool, - notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>, + persisted: bool, + committed_generation: Option<Uuid>, } async fn persist_server_config_transaction( config: ServerConfig, - prepared: PreparedRuntimeConfig, snapshot: AdminServerConfigSnapshot, - sub_system: Option<&str>, ) -> S3Result<PersistedConfigTransaction> { snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?; let previous_config = snapshot.config.clone(); let history_restore_id = save_server_config_history_snapshot(&previous_config).await?; if snapshot.is_lock_lost() { - cleanup_failed_config_history_snapshot(&history_restore_id).await; + cleanup_config_history_snapshot(&history_restore_id).await; return reject_lost_config_transaction(snapshot, "before persistence").await; } - let persisted = match save_server_config_to_store(&config, &snapshot).await { - Ok(persisted) => persisted, + let save_result = match save_server_config_to_store(&config, &snapshot).await { + Ok(result) => result, Err(err) => { - cleanup_failed_config_history_snapshot(&history_restore_id).await; + cleanup_config_history_snapshot(&history_restore_id).await; return Err(err); } }; + let persisted = save_result.persisted(); + let committed_generation = save_result.generation(); let history_restore_id = if persisted { Some(history_restore_id) } else { - cleanup_failed_config_history_snapshot(&history_restore_id).await; + cleanup_config_history_snapshot(&history_restore_id).await; None }; - if snapshot.is_lock_lost() { - return reject_lost_config_transaction(snapshot, "after persistence").await; - } - - let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS); - if storage_class_applied { - if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) { - return Err(match history_restore_id.as_deref() { - Some(restore_id) => s3_error!( - InternalError, - "config persisted but runtime publish failed; recovery snapshot restoreId={}: {}", - restore_id, - err - ), - None => err, - }); - } - } else { - publish_server_config(config.clone()); - } - let notify_transition = publish_notify_config_intent(&config, sub_system); - if snapshot.is_lock_lost() { - return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await; - } - drop(snapshot); Ok(PersistedConfigTransaction { previous_config, history_restore_id, - storage_class_applied, - notify_transition, + persisted, + committed_generation, }) } +async fn reconcile_committed_config(sub_system: Option<String>, persisted: bool) -> S3Result<bool> { + let mut errors = Vec::new(); + let publication_result = match sub_system.as_deref() { + Some(sub_system) => publish_latest_runtime_config_snapshot(sub_system).await, + None => reload_runtime_config_snapshot().await, + }; + if let Err(err) = publication_result { + warn!(error = %err, "Failed to publish the latest durable server config"); + errors.push("local config snapshot".to_string()); + } + + if sub_system.is_none() { + reconcile_full_config(persisted, errors).await?; + return Ok(false); + } + + reconcile_targeted_config(sub_system, persisted, errors).await +} + async fn commit_server_config_transaction( config: ServerConfig, - prepared: PreparedRuntimeConfig, snapshot: AdminServerConfigSnapshot, sub_system: Option<String>, ) -> S3Result<bool> { - let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?; - - if sub_system.is_none() { - reconcile_full_config(transaction.notify_transition).await?; - return Ok(false); + let transaction = persist_server_config_transaction(config, snapshot).await?; + let result = reconcile_committed_config(sub_system, transaction.persisted).await; + match (result, transaction.history_restore_id.as_deref()) { + (Err(err), Some(restore_id)) => Err(s3_error!(InternalError, "{}; recovery snapshot restoreId={}", err, restore_id)), + (result, _) => result, } - - reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await } pub struct GetConfigKVHandler {} @@ -2127,8 +2049,8 @@ impl Operation for SetConfigKVHandler { let snapshot = load_server_config_snapshot_from_store().await?; let mut config = snapshot.config.clone(); apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, sub_system.as_deref()).await?; - commit_server_config_transaction(config, prepared, snapshot, sub_system).await + prepare_server_config(&config, sub_system.as_deref()).await?; + commit_server_config_transaction(config, snapshot, sub_system).await }) .await?; @@ -2155,8 +2077,8 @@ impl Operation for DelConfigKVHandler { let snapshot = load_server_config_snapshot_from_store().await?; let mut config = snapshot.config.clone(); apply_delete_directives(&mut config, &directives); - let prepared = prepare_server_config(&config, sub_system.as_deref()).await?; - commit_server_config_transaction(config, prepared, snapshot, sub_system).await + prepare_server_config(&config, sub_system.as_deref()).await?; + commit_server_config_transaction(config, snapshot, sub_system).await }) .await?; @@ -2239,15 +2161,19 @@ impl Operation for RestoreConfigHistoryKVHandler { supervise_admin_mutation("config mutation", async move { preflight_config_intent(None).await?; let snapshot = load_server_config_snapshot_from_store().await?; - let prepared = prepare_server_config(&config, None).await?; - let restored_config = config.clone(); - let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?; - let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else { + prepare_server_config(&config, None).await?; + let transaction = persist_server_config_transaction(config, snapshot).await?; + let Err(restore_error) = reconcile_committed_config(None, transaction.persisted).await else { return Ok(()); }; + if !transaction.persisted { + return Err(restore_error); + } + let previous_config = transaction.previous_config; let recovery_restore_id = transaction.history_restore_id; + let committed_generation = transaction.committed_generation; let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created"); let rollback_snapshot = match load_server_config_snapshot_from_store().await { Ok(snapshot) => snapshot, @@ -2261,7 +2187,9 @@ impl Operation for RestoreConfigHistoryKVHandler { )); } }; - if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) { + if let Err(rollback_error) = + validate_restore_rollback_generation(rollback_snapshot.generation(), committed_generation) + { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", @@ -2271,12 +2199,20 @@ impl Operation for RestoreConfigHistoryKVHandler { )); } - let rollback_prepared = prepare_server_config(&previous_config, None).await?; - rollback_snapshot + if let Err(rollback_error) = prepare_server_config(&previous_config, None).await { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + if let Err(rollback_error) = rollback_snapshot .ensure_lock_held() .map_err(ApiError::from) - .map_err(S3Error::from)?; - if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await { + .map_err(S3Error::from) + { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", @@ -2285,10 +2221,9 @@ impl Operation for RestoreConfigHistoryKVHandler { recovery_reference )); } - if rollback_snapshot.is_lock_lost() { - if let Err(rollback_error) = - reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await - { + let rollback_persisted = match save_server_config_to_store(&previous_config, &rollback_snapshot).await { + Ok(result) => result.persisted(), + Err(rollback_error) => { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", @@ -2297,46 +2232,20 @@ impl Operation for RestoreConfigHistoryKVHandler { recovery_reference )); } - return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly")); - } - - if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) { - return Err(s3_error!( - InternalError, - "config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}", - restore_error, - rollback_error, - recovery_reference - )); - } - let rollback_transition = publish_notify_config_intent(&previous_config, None); - if rollback_snapshot.is_lock_lost() { - if let Err(rollback_error) = - reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await - { - return Err(s3_error!( - InternalError, - "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", - restore_error, - rollback_error, - recovery_reference - )); - } - return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly")); - } + }; drop(rollback_snapshot); - if let Err(rollback_error) = reconcile_full_config(rollback_transition).await { + if let Err(rollback_error) = reconcile_committed_config(None, rollback_persisted).await { return Err(s3_error!( InternalError, - "config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}", + "config restore failed: {}; automatic rollback convergence failed: {}; recovery snapshot restoreId={}", restore_error, rollback_error, recovery_reference )); } - if let Some(recovery_restore_id) = recovery_restore_id.as_deref() { - cleanup_failed_config_history_snapshot(recovery_restore_id).await; + if rollback_persisted && let Some(recovery_restore_id) = recovery_restore_id.as_deref() { + cleanup_config_history_snapshot(recovery_restore_id).await; } Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error)) }) @@ -2377,8 +2286,8 @@ impl Operation for SetConfigHandler { let snapshot = load_server_config_snapshot_from_store().await?; let mut config = ServerConfig::new(); apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, None).await?; - commit_server_config_transaction(config, prepared, snapshot, None).await?; + prepare_server_config(&config, None).await?; + commit_server_config_transaction(config, snapshot, None).await?; Ok(()) }) .await?; @@ -2408,6 +2317,23 @@ mod tests { ); } + #[test] + fn reconciliation_error_reports_whether_config_was_persisted() { + let persisted = finish_config_reconciliation(vec!["local config snapshot".to_string()], true) + .expect_err("committed config convergence failure must be reported"); + let unchanged = finish_config_reconciliation(vec!["local config snapshot".to_string()], false) + .expect_err("unchanged config convergence failure must be reported"); + + assert_eq!( + persisted.message(), + Some("server config persisted but runtime convergence failed: local config snapshot") + ); + assert_eq!( + unchanged.message(), + Some("server config was unchanged but runtime convergence failed: local config snapshot") + ); + } + #[test] fn tokenize_config_line_handles_quotes_and_escapes() { let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#) @@ -3191,23 +3117,32 @@ notify_webhook:secondary endpoint="https://secondary.example" auth_token="second } #[test] - fn restore_rollback_generation_rejects_concurrent_config_change() { - crate::admin::storage_api::config::init_admin_config_defaults(); - let restored = ServerConfig::new(); - let mut concurrent = restored.clone(); - apply_set_directives( - &mut concurrent, - &parse_config_directives(r#"identity_openid client_id="concurrent-client""#, false).expect("parse concurrent"), - ) - .expect("apply concurrent"); + fn restore_rollback_generation_accepts_committed_write_identity() { + let committed = Uuid::from_u128(1); + validate_restore_rollback_generation(Some(committed), Some(committed)).expect("matching committed generation"); + } - validate_restore_rollback_generation(&restored, &restored).expect("unchanged restore generation"); - let error = validate_restore_rollback_generation(&concurrent, &restored) - .expect_err("concurrent change must fence automatic rollback"); + #[test] + fn restore_rollback_generation_rejects_aba_with_matching_config_content() { + let error = validate_restore_rollback_generation(Some(Uuid::from_u128(2)), Some(Uuid::from_u128(1))) + .expect_err("a later generation must fence automatic rollback even when config content matches"); assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); assert!(error.to_string().contains("concurrent configuration change")); } + #[test] + fn restore_rollback_generation_rejects_missing_write_identity() { + for (current, committed) in [ + (None, Some(Uuid::from_u128(1))), + (Some(Uuid::from_u128(1)), None), + (None, None), + ] { + let error = validate_restore_rollback_generation(current, committed) + .expect_err("missing generation metadata must fence automatic rollback"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + } + } + #[test] fn legacy_directive_history_is_rejected_without_being_applied_as_a_snapshot() { let error = decode_config_history_snapshot(b"identity_openid client_id=\"legacy\"") diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index 43e500d7c..c406dfafa 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -16,7 +16,9 @@ use crate::admin::runtime_sources::{ AppContext, current_app_context, current_notification_system_for_context, current_object_store_handle_for_context, publish_server_config, publish_storage_class_config, }; -use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass}; +use crate::admin::storage_api::config::{ + STORAGE_CLASS_SUB_SYS, read_existing_admin_server_config_no_lock, storageclass, with_admin_server_config_read_lock, +}; use crate::admin::storage_api::contract::admin::StorageAdminApi; use crate::admin::storage_api::runtime::ECStore; use crate::server::{ @@ -43,6 +45,25 @@ use url::Url; static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(()); +// Runtime publication lock order: reload mutex -> server-config local lock -> +// transaction lock -> config object lock. +pub(crate) async fn with_runtime_config_reload_lock<Fut, T>(operation: Fut) -> S3Result<T> +where + Fut: Future<Output = S3Result<T>> + Send + 'static, + T: Send + 'static, +{ + let reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; + tokio::spawn(async move { + let _reload_guard = reload_guard; + operation.await + }) + .await + .map_err(|err| { + let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" }; + internal_error(format!("runtime config reload task {outcome}")) + })? +} + pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool { NOTIFY_SUB_SYSTEMS.contains(&sub_system) || matches!( @@ -121,11 +142,6 @@ impl PreparedRuntimeConfig { fn publish_storage_class_for_context(self, context: Option<&AppContext>) -> S3Result<()> { self.publish_storage_class_for_context_with(context, publish_storage_class_config) } - - pub(crate) fn publish_storage_class(self) -> S3Result<()> { - let context = current_app_context(); - self.publish_storage_class_for_context(context.as_deref()) - } } fn publish_server_config_for_context(context: Option<&AppContext>, config: ServerConfig) { @@ -396,7 +412,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste } pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> { - let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; + let context = context.cloned(); + let sub_system = sub_system.to_owned(); + with_runtime_config_reload_lock(async move { + reload_dynamic_config_runtime_state_under_reload_lock_for_context(context.as_ref(), &sub_system).await + }) + .await +} + +async fn reload_dynamic_config_runtime_state_under_reload_lock_for_context( + context: Option<&AppContext>, + sub_system: &str, +) -> S3Result<()> { if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM { let store = resolve_runtime_config_store_for_context(context)?; let notify_result = reconcile_event_notifier_from_store(store).await; @@ -414,18 +441,54 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap } let store = resolve_runtime_config_store_for_context(context)?; - let config = read_admin_config_without_migrate(store).await.map_err(|err| { - warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}"); - internal_error(format!("failed to load server config: {err}")) - })?; + let read_store = store.clone(); + let publication_context = context.cloned(); + let publication_sub_system = sub_system.to_owned(); + let notify_config = with_admin_server_config_read_lock(store, move || async move { + let config = read_existing_admin_server_config_no_lock(read_store).await.map_err(|err| { + warn!("peer reload_dynamic_config: failed to load server config for {publication_sub_system}: {err}"); + internal_error(format!("failed to load server config: {err}")) + })?; + let prepared = prepare_server_config_for_context(publication_context.as_ref(), &config, Some(&publication_sub_system)) + .await + .map_err(|err| { + if publication_sub_system == STORAGE_CLASS_SUB_SYS { + internal_error(format!("failed to apply storage class config: {err}")) + } else { + err + } + })?; - if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) { - validate_server_config_for_context(context, &config, Some(sub_system)).await?; - // Scanner cycles refresh from the process-wide server config before - // each pass. Publish the same validated snapshot first so that refresh - // cannot overwrite this peer's dynamic scanner update with stale data. - publish_server_config_for_context(context, config.clone()); - } + if publication_sub_system == STORAGE_CLASS_SUB_SYS { + prepared.publish_storage_class_for_context(publication_context.as_ref())?; + return Ok::<Option<ServerConfig>, S3Error>(None); + } + if NOTIFY_SUB_SYSTEMS.contains(&publication_sub_system.as_str()) { + return Ok(Some(config)); + } + if matches!(publication_sub_system.as_str(), SCANNER_SUB_SYS | HEAL_SUB_SYS) { + publish_server_config_for_context(publication_context.as_ref(), config.clone()); + } + apply_dynamic_config_for_subsystem_for_context(publication_context.as_ref(), &config, &publication_sub_system) + .await + .inspect_err(|_| { + warn!( + config_subsystem = publication_sub_system, + reason = "apply_failed", + "Peer dynamic config apply failed" + ); + })?; + Ok(None) + }) + .await + .map_err(|err| { + warn!("peer reload_dynamic_config: failed to acquire server config publication fence for {sub_system}: {err}"); + internal_error(format!("failed to lock server config: {err}")) + })??; + + let Some(config) = notify_config else { + return Ok(()); + }; apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) .await .inspect_err(|_| { @@ -439,23 +502,16 @@ pub async fn reload_dynamic_config_runtime_state(sub_system: &str) -> S3Result<( reload_dynamic_config_runtime_state_for_context(context.as_deref(), sub_system).await } -async fn reload_runtime_config_snapshot_with<ReadFuture, Prepare, PrepareFuture, Publish, ApplyWorkers, ApplyWorkersFuture>( - read: ReadFuture, - prepare: Prepare, - publish: Publish, +async fn reload_runtime_config_snapshot_with<PublishFuture, ApplyWorkers, ApplyWorkersFuture>( + publish_snapshot: PublishFuture, apply_workers: ApplyWorkers, ) -> S3Result<()> where - ReadFuture: Future<Output = S3Result<ServerConfig>>, - Prepare: FnOnce(ServerConfig) -> PrepareFuture, - PrepareFuture: Future<Output = S3Result<(ServerConfig, PreparedRuntimeConfig)>>, - Publish: FnOnce(&ServerConfig, PreparedRuntimeConfig) -> S3Result<()>, + PublishFuture: Future<Output = S3Result<ServerConfig>>, ApplyWorkers: FnOnce(ServerConfig) -> ApplyWorkersFuture, ApplyWorkersFuture: Future<Output = S3Result<()>>, { - let config = read.await?; - let (config, prepared) = prepare(config).await?; - publish(&config, prepared)?; + let config = publish_snapshot.await?; // Worker reloads mutate live state and have no rollback contract, so the // validated snapshots stay published. The RPC still reports convergence @@ -474,55 +530,105 @@ where Ok(()) } -pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> { - let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; +async fn publish_latest_runtime_config_snapshot_under_reload_lock_for_context<ApplyWorkers, ApplyWorkersFuture>( + context: Option<&AppContext>, + sub_system: Option<&str>, + apply_workers: ApplyWorkers, +) -> S3Result<()> +where + ApplyWorkers: FnOnce(ServerConfig) -> ApplyWorkersFuture + Send + 'static, + ApplyWorkersFuture: Future<Output = S3Result<()>> + Send + 'static, +{ let store = resolve_runtime_config_store_for_context(context)?; + let read_store = store.clone(); + let publication_context = context.cloned(); + let publication_sub_system = sub_system.map(str::to_owned); - reload_runtime_config_snapshot_with( - async move { - read_admin_config_without_migrate(store).await.map_err(|err| { - warn!("peer reload_runtime_config_snapshot: failed to load server config: {err}"); - internal_error(format!("failed to load server config: {err}")) - }) - }, - |config| async move { - let prepared = prepare_server_config_for_context(context, &config, None).await.map_err(|_| { - warn!("peer reload_runtime_config_snapshot: failed to prepare server config"); - internal_error("failed to prepare server config") - })?; - Ok((config, prepared)) - }, - |config, prepared| { - prepared.publish_storage_class_for_context(context)?; - publish_server_config_for_context(context, config.clone()); - Ok(()) - }, - |config| async move { - let mut failures = Vec::new(); - for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { - if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) - .await - .is_err() + with_admin_server_config_read_lock(store, move || async move { + let config = read_existing_admin_server_config_no_lock(read_store).await.map_err(|err| { + warn!("runtime config publication failed to load server config: {err}"); + internal_error(format!("failed to load server config: {err}")) + })?; + let prepared = + prepare_server_config_for_context(publication_context.as_ref(), &config, publication_sub_system.as_deref()) + .await + .map_err(|err| match publication_sub_system.as_deref() { + None => { + warn!("peer reload_runtime_config_snapshot: failed to prepare server config"); + internal_error("failed to prepare server config") + } + Some(STORAGE_CLASS_SUB_SYS) => internal_error(format!("failed to apply storage class config: {err}")), + Some(_) => err, + })?; + reload_runtime_config_snapshot_with( + async move { + if publication_sub_system + .as_deref() + .is_none_or(|sub_system| sub_system == STORAGE_CLASS_SUB_SYS) { - failures.push(sub_system); - warn!( - event = EVENT_CONFIG_WORKER_RELOAD_FAILED, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_CONFIG, - config_subsystem = sub_system, - state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - reason = "apply_failed", - "Peer runtime config snapshot was published but a subsystem worker reload failed" - ); + prepared.publish_storage_class_for_context(publication_context.as_ref())?; } + publish_server_config_for_context(publication_context.as_ref(), config.clone()); + Ok(config) + }, + apply_workers, + ) + .await + }) + .await + .map_err(|err| { + warn!("runtime config publication failed to acquire server config publication fence: {err}"); + internal_error(format!("failed to lock server config: {err}")) + })? +} + +pub(crate) async fn publish_latest_runtime_config_snapshot(sub_system: &str) -> S3Result<()> { + let context = current_app_context(); + let sub_system = sub_system.to_owned(); + with_runtime_config_reload_lock(async move { + publish_latest_runtime_config_snapshot_under_reload_lock_for_context(context.as_deref(), Some(&sub_system), |_| async { + Ok(()) + }) + .await + }) + .await +} + +pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> { + let context = context.cloned(); + with_runtime_config_reload_lock(async move { + reload_runtime_config_snapshot_under_reload_lock_for_context(context.as_ref()).await + }) + .await +} + +async fn reload_runtime_config_snapshot_under_reload_lock_for_context(context: Option<&AppContext>) -> S3Result<()> { + let worker_context = context.cloned(); + publish_latest_runtime_config_snapshot_under_reload_lock_for_context(context, None, move |config| async move { + let mut failures = Vec::new(); + for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { + if apply_dynamic_config_for_subsystem_for_context(worker_context.as_ref(), &config, sub_system) + .await + .is_err() + { + failures.push(sub_system); + warn!( + event = EVENT_CONFIG_WORKER_RELOAD_FAILED, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_CONFIG, + config_subsystem = sub_system, + state = CONFIG_WORKER_RELOAD_FAILURE_STATE, + reason = "apply_failed", + "Peer runtime config snapshot was published but a subsystem worker reload failed" + ); } - if failures.is_empty() { - Ok(()) - } else { - Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; ")))) - } - }, - ) + } + if failures.is_empty() { + Ok(()) + } else { + Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; ")))) + } + }) .await } @@ -699,6 +805,77 @@ mod tests { } } + #[tokio::test] + async fn runtime_reload_waiter_cancellation_does_not_leave_queued_work() { + let _blocker = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; + let (operation_started_tx, mut operation_started_rx) = tokio::sync::oneshot::channel(); + let mut reload = Box::pin(with_runtime_config_reload_lock(async move { + let _ = operation_started_tx.send(()); + Ok(()) + })); + + poll_fn(|cx| match reload.as_mut().poll(cx) { + Poll::Pending => Poll::Ready(()), + Poll::Ready(_) => panic!("reload must wait while the mutex is held"), + }) + .await; + drop(reload); + + assert!( + matches!(operation_started_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Closed)), + "cancelling a queued reload must drop its work instead of detaching it" + ); + } + + #[tokio::test] + async fn runtime_reload_lock_survives_waiter_cancellation() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); + + let waiter = tokio::spawn(async move { + with_runtime_config_reload_lock(async move { + started_tx.send(()).expect("signal runtime reload start"); + release_rx.await.expect("release supervised runtime reload"); + completed_tx.send(()).expect("signal runtime reload completion"); + Ok(()) + }) + .await + }); + + started_rx.await.expect("supervised runtime reload started"); + waiter.abort(); + assert!(waiter.await.expect_err("reload waiter should be cancelled").is_cancelled()); + + let (contender_polled_tx, contender_polled_rx) = tokio::sync::oneshot::channel(); + let (contender_entered_tx, mut contender_entered_rx) = tokio::sync::oneshot::channel(); + let contender = tokio::spawn(async move { + let mut lock = Box::pin(RUNTIME_CONFIG_RELOAD_MUTEX.lock()); + let mut contender_polled_tx = Some(contender_polled_tx); + let _guard = poll_fn(|cx| { + if let Some(tx) = contender_polled_tx.take() { + let _ = tx.send(()); + } + lock.as_mut().poll(cx) + }) + .await; + let _ = contender_entered_tx.send(()); + }); + + contender_polled_rx.await.expect("contending reload should be polled"); + assert!( + matches!(contender_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)), + "a cancelled waiter must not release the reload mutex while its detached reload is still running" + ); + + release_tx.send(()).expect("release detached runtime reload"); + completed_rx.await.expect("detached runtime reload should complete"); + contender_entered_rx + .await + .expect("contending reload should enter after completion"); + contender.await.expect("contending reload task should not panic"); + } + #[tokio::test] async fn checked_scanner_reload_reports_unreachable_peer() { let temp_dir = TempDir::new().expect("scanner reload temp dir"); @@ -1221,6 +1398,123 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial_test::serial(storage_class_env)] + async fn full_reload_keeps_worker_apply_inside_durable_read_fence() { + temp_env::async_with_vars( + [ + (storageclass::STANDARD_ENV, None::<&str>), + (storageclass::RRS_ENV, None::<&str>), + (storageclass::OPTIMIZE_ENV, None::<&str>), + (storageclass::INLINE_BLOCK_ENV, None::<&str>), + ], + async { + let fixture = runtime_config_reload_fixture().await; + let older = scanner_server_config("41"); + save_admin_server_config(fixture.context.object_store(), &older) + .await + .expect("persist older scanner config"); + + let (worker_entered_tx, worker_entered_rx) = tokio::sync::oneshot::channel(); + let (release_worker_tx, release_worker_rx) = tokio::sync::oneshot::channel(); + let reload_context = fixture.context.clone(); + let reload = tokio::spawn(async move { + publish_latest_runtime_config_snapshot_under_reload_lock_for_context( + Some(&reload_context), + None, + move |config| async move { + worker_entered_tx.send(config).expect("signal runtime config worker entry"); + release_worker_rx.await.expect("release runtime config worker"); + Ok(()) + }, + ) + .await + }); + + let worker_config = tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, worker_entered_rx) + .await + .expect("worker apply should enter") + .expect("worker apply entry signal should be delivered"); + assert_eq!( + worker_config + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("older scanner config should be loaded") + .get(SCANNER_CYCLE), + "41" + ); + + let latest = scanner_server_config("71"); + let writer_store = fixture.context.object_store(); + let (writer_polled_tx, writer_polled_rx) = tokio::sync::oneshot::channel(); + let (writer_entered_tx, mut writer_entered_rx) = tokio::sync::oneshot::channel(); + let writer = tokio::spawn(async move { + let transaction_store = writer_store.clone(); + let transaction = with_admin_server_config_write_lock(writer_store, move || async move { + writer_entered_tx.send(()).expect("signal writer entry"); + save_admin_server_config_no_lock(transaction_store, &latest).await + }); + tokio::pin!(transaction); + let mut writer_polled_tx = Some(writer_polled_tx); + poll_fn(|cx| match transaction.as_mut().poll(cx) { + Poll::Pending => { + if let Some(tx) = writer_polled_tx.take() { + let _ = tx.send(()); + } + Poll::Ready(()) + } + Poll::Ready(_) => panic!("writer entered before the worker apply released its read fence"), + }) + .await; + transaction + .await + .expect("writer should acquire the server-config locks") + .expect("writer should persist the latest config"); + }); + + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, writer_polled_rx) + .await + .expect("writer should be polled") + .expect("writer poll signal should be delivered"); + assert!( + matches!(writer_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)), + "a server-config writer must wait until the old worker apply completes" + ); + + release_worker_tx.send(()).expect("release worker apply"); + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async { + reload + .await + .expect("full reload task should not panic") + .expect("full reload should finish"); + writer.await.expect("writer task should not panic"); + }) + .await + .expect("reload and writer should finish"); + + reload_runtime_config_snapshot_for_context(Some(&fixture.context)) + .await + .expect("a later full reload should publish the latest durable config"); + let snapshot = fixture + .server_snapshot + .lock() + .expect("server config result lock") + .clone() + .expect("latest server config should be published"); + assert_eq!( + snapshot + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("latest scanner config should be present") + .get(SCANNER_CYCLE), + "71" + ); + assert_eq!(rustfs_scanner::scanner_runtime_config_status().cycle_interval_seconds.value, 71); + + rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults"); + }, + ) + .await; + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn peer_full_reload_rejects_later_pool_without_publishing() { @@ -1251,11 +1545,9 @@ mod tests { let worker_events = events.clone(); let err = reload_runtime_config_snapshot_with( - async { Ok(ServerConfig::new()) }, - |config| async { Ok((config, PreparedRuntimeConfig::default())) }, - move |_config, _prepared| { + async move { publish_events.lock().expect("reload event lock").push("publish"); - Ok(()) + Ok(ServerConfig::new()) }, move |_config| async move { let mut events = worker_events.lock().expect("reload event lock"); diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index c678b3e88..cd437819e 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -644,12 +644,17 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu ecstore_config::com::read_config_without_migrate(api).await } +pub(crate) async fn read_existing_admin_server_config_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> { + ecstore_config::com::read_existing_server_config_no_lock(api).await +} + #[cfg(test)] pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> { ecstore_config::com::read_config_without_migrate_no_lock(api).await } pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot; +pub(crate) type AdminServerConfigSaveResult = ecstore_config::com::ServerConfigSaveResult; pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> { ecstore_config::com::save_config(api, file, data).await @@ -690,8 +695,17 @@ pub(crate) async fn save_admin_server_config_snapshot( api: Arc<ECStore>, cfg: &rustfs_config::server_config::Config, snapshot: &AdminServerConfigSnapshot, -) -> Result<bool> { - ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await +) -> Result<AdminServerConfigSaveResult> { + ecstore_config::com::save_server_config_snapshot_with_generation(api, cfg, snapshot).await +} + +pub(crate) async fn with_admin_server_config_read_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T> +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future<Output = T> + Send + 'static, + T: Send + 'static, +{ + ecstore_config::com::with_server_config_read_lock(api, operation).await } pub(crate) fn init_admin_config_defaults() { @@ -793,9 +807,10 @@ pub(crate) mod cluster { pub(crate) mod config { pub(crate) use super::storageclass; pub(crate) use super::{ - AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, - read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, - save_admin_server_config_snapshot, + AdminServerConfigSaveResult, AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, + init_admin_config_defaults, read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, + read_existing_admin_server_config_no_lock, save_admin_config, save_admin_server_config_snapshot, + with_admin_server_config_read_lock, }; #[cfg(test)] pub(crate) use super::{ diff --git a/scripts/e2e-run.sh b/scripts/e2e-run.sh index ed66c1aed..645184736 100755 --- a/scripts/e2e-run.sh +++ b/scripts/e2e-run.sh @@ -40,7 +40,19 @@ export RUST_BACKTRACE=full "$BIN" --address "127.0.0.1:${RUSTFS_TEST_PORT}" "$VOLUME" > "${RUSTFS_TEST_LOG}" 2>&1 & RUSTFS_PID=$! -sleep 10 +for _ in {1..60}; do + if curl --noproxy '*' --silent --fail "http://127.0.0.1:${RUSTFS_TEST_PORT}/health/ready" >/dev/null; then + break + fi + + if ! kill -0 "${RUSTFS_PID}" 2>/dev/null; then + wait "${RUSTFS_PID}" + fi + + sleep 1 +done + +curl --noproxy '*' --silent --show-error --fail "http://127.0.0.1:${RUSTFS_TEST_PORT}/health/ready" >/dev/null export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY:-rustfsadmin}" export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" From ffc9de72cbe79dfc68c4c8c0ef0e6e6a37b9f04a Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 09:31:03 +0800 Subject: [PATCH 45/52] docs: make validation proportional to change risk (#5527) --- AGENTS.md | 83 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1ae7c5fc0..5a13a84b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,33 +105,78 @@ CI) fails the build if anything is committed under `docs/superpowers/`, even via ## Verification Before PR -Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion. -Non-exempt changes must also pass Adversarial Validation (next section) before the checks below count as completion. +Convert changes into independently verifiable outcomes. This section controls +agent-run local validation; preparing a commit or PR does not by itself require +the broadest gate. Inspect only the final task-owned diff, classify it by +behavioral impact rather than line count or path alone, and run the smallest +set of checks that provides meaningful coverage. Do not let unrelated +worktree changes or a generic contributor checklist expand the scope. +Non-exempt changes must also pass Adversarial Validation (next section) before +the checks below count as completion. -For code changes, run and pass the following before opening a PR: +### Validation floor -```bash -make pre-pr -``` +- Every change that is not documentation-only must finish with + `cargo fmt --all --check` passing. An umbrella gate that runs this exact + check satisfies the requirement; do not run it twice. Use `cargo fmt --all` + only when formatting needs to be fixed. Run the configured formatter or + validator for other changed languages when one exists. +- Documentation-only or instruction-only means all task-owned changes are + prose or documentation assets and cannot affect runtime, builds, CI, + dependencies, generated code, or tests. Run `git diff --check` and any + relevant documentation guard, but skip Cargo formatting, compilation, + Clippy, tests, `make pre-commit`, and `make pre-pr`. +- Behavior changes require relevant existing or new tests. Prefer the most + focused test or affected package. A passing targeted test can also provide + sufficient compilation coverage when it builds every changed target and + feature involved; do not add a redundant `cargo check` in that case. +- `cargo check` supplements compilation coverage; it never substitutes for a + behavioral test. If a relevant test cannot reasonably be added or run, use + the narrowest compilation check and report the reason and remaining risk. -Before committing code changes, prefer focused verification for the touched -surface and use the faster local gate when a broad smoke check is needed: +### Validation tiers -```bash -make pre-commit -``` +1. **Documentation/instruction-only:** Apply the exemption above. Run a guard + such as `make doc-paths-check` only when it is relevant to the edited text. +2. **Non-behavioral source change:** For comments, formatting, or another + demonstrably non-executable change, run the formatting floor. Compilation, + Clippy, and tests may be skipped only when the edit cannot affect + compilation or runtime behavior; run targeted doctests if executable + documentation examples changed. +3. **Localized or bounded behavior change:** Run the formatting floor and the + narrowest relevant tests. Add package-scoped `cargo check` or Clippy only + for changed targets, features, APIs, error handling, async behavior, or + control flow not already covered. When several crates are affected but the + dependency set is identifiable, validate those packages and known + dependents instead of the whole workspace. Use `make pre-commit` only when + a repository-wide fast gate adds useful confidence beyond those checks. +4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage + cannot bound the impact, including: + - dependency, feature, build-script, procedural-macro, code-generation, + toolchain, or CI changes that alter compilation or the test matrix; + - cross-crate public APIs, shared foundational code, or broad refactors with + an unbounded dependent set; + - locking, storage durability or formats, erasure coding, replication, + RPC/protocol compatibility, IAM/KMS/auth, cryptography, or other + security-sensitive behavior; + - a targeted check that reveals wider impact, an explicit user request, or + a release policy that requires the full gate. -For migration batches, do not run the full `make pre-pr` gate before every -intermediate commit. Use focused tests and `make pre-commit` during -development, then reserve `make pre-pr` for the final PR-ready branch. +Documentation-only and non-behavioral classifications take precedence over +path-based triggers. A small diff can still be high-risk, while a CI comment, +manifest comment, or release-note edit does not require full validation. -Before pushing code changes, make sure formatting is clean: +`make pre-pr` includes `make pre-commit` coverage. Never run both for the same +unchanged diff, and do not repeat equivalent checks during PR preparation or +because a local hook already ran them. Rerun only checks whose scope is affected +by later edits. Full workspace checks do not replace a relevant integration or +E2E test for changed behavior; run that focused test when required and +available, or report why it was not run and the remaining risk. -- Run `cargo fmt --all`. -- Run `cargo fmt --all --check` and ensure no files are modified unexpectedly. +If `make` is unavailable, run the equivalent checks defined under +`.config/make/`. At handoff, list the checks actually run, checks intentionally +skipped, and the reason for the selected tier. -If `make` is unavailable, run the equivalent checks defined under `.config/make/`. -Documentation-only or instruction-only changes are exempt from the verification commands above (including the `.config/make/` equivalents), though any locally installed git pre-commit hooks may still run on commit unless explicitly skipped. After build-based verification completes, clean generated build artifacts before wrapping up to avoid unnecessary disk usage. Do not open a PR with code changes when the required checks fail. Make a failing check pass by fixing the cause, never by weakening the gate: From 98b20b4231dac7d297f839f58eb9f776b51270e5 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 09:31:21 +0800 Subject: [PATCH 46/52] test(s3): cover delete marker list visibility (#5503) --- .../delete_marker_migration_semantics_test.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs index aac225d7a..5a51c053a 100644 --- a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs +++ b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs @@ -56,6 +56,21 @@ mod tests { ); } + async fn assert_current_list_hides_delete_marker(client: &Client, bucket: &str, key: &str) { + let listed = client + .list_objects_v2() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("list current objects after delete marker"); + + assert!( + listed.contents().iter().all(|object| object.key() != Some(key)), + "ListObjectsV2 must hide an object whose latest version is a delete marker" + ); + } + #[tokio::test] #[serial] async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() { @@ -94,6 +109,7 @@ mod tests { assert_eq!(markers[0].version_id(), Some(delete_marker_version_id)); assert_eq!(markers[0].is_latest(), Some(true)); assert_current_get_is_delete_marker_not_found(&client, bucket, key).await; + assert_current_list_hides_delete_marker(&client, bucket, key).await; } #[tokio::test] @@ -118,6 +134,17 @@ mod tests { .await .expect("put historical version"); let data_version_id = put.version_id().expect("put should return data version id"); + let listed_before_delete = client + .list_objects_v2() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("list current object before creating delete marker"); + assert!( + listed_before_delete.contents().iter().any(|object| object.key() == Some(key)), + "ListObjectsV2 must include the current object before it is deleted" + ); let delete_marker = client .delete_object() @@ -145,6 +172,7 @@ mod tests { assert_eq!(markers[0].version_id(), Some(delete_marker_version_id)); assert_eq!(markers[0].is_latest(), Some(true)); assert_current_get_is_delete_marker_not_found(&client, bucket, key).await; + assert_current_list_hides_delete_marker(&client, bucket, key).await; let historical = client .get_object() From fdac60b0e27ea529c6f15035e98b83f6c674fc35 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 09:36:12 +0800 Subject: [PATCH 47/52] fix(kms): make Vault KV2 lifecycle writes check-and-set (#5518) * fix(kms): drop stale KmsClient trait import in local_export tests The KmsClient trait was folded into KmsBackend (#5501), but the backup export tests merged afterwards (#5499) still imported it, breaking the crate's test build; create_key is an inherent LocalKmsClient method, so the import is simply unused. * fix(kms): make Vault KV2 lifecycle writes check-and-set Every KV2 lifecycle write used to be a blind whole-record overwrite, so two nodes racing on the same key could lose updates: a disable racing a rotation wrote the pre-rotation record back (rolling back the version and material of a committed rotation), concurrent same-name creates let the later material win (orphaning DEKs wrapped under the earlier one), and a cancellation racing the deletion sweep could be overwritten by the tombstone (or resurrect an already tombstoned key). All lifecycle mutations now go through a bounded check-and-set read-modify-write loop: each attempt re-reads the record pinned to its KV2 secret version, re-runs the state gate against the fresh snapshot, and writes back check-and-set against exactly that version; after LIFECYCLE_CAS_ATTEMPTS lost races the typed conflict error surfaces. The loop composes with the operation policy's single-attempt rule for non-idempotent writes: each write is still sent at most once, only the whole read-gate-write cycle repeats. create_key becomes a create-only write (cas=0) so exactly one of two concurrent creates commits and the loser reports KeyAlreadyExists. The blind store_key_data primitive is now test-only. Reads and rotation additionally fail closed when the version history is inconsistent: resolving material through a version record above the current pointer is refused (that state only arises when a lost update rolled back a committed rotation), and rotation refuses to extend a history whose records reach more than one step past the current pointer (one step ahead is the footprint of an interrupted rotation and still recovers through the adopt path). Refs rustfs/backlog#1581 --- crates/kms/src/backends/scripted_vault.rs | 42 +- crates/kms/src/backends/vault.rs | 1061 ++++++++++++++++++--- 2 files changed, 938 insertions(+), 165 deletions(-) diff --git a/crates/kms/src/backends/scripted_vault.rs b/crates/kms/src/backends/scripted_vault.rs index 19453ea46..320b78c1a 100644 --- a/crates/kms/src/backends/scripted_vault.rs +++ b/crates/kms/src/backends/scripted_vault.rs @@ -61,7 +61,7 @@ impl ScriptedResponse { pub(crate) struct ScriptedVault { /// Base address (`http://127.0.0.1:port`) to point a Vault client at. pub(crate) address: String, - requests: Arc<Mutex<Vec<String>>>, + requests: Arc<Mutex<Vec<(String, String)>>>, } impl ScriptedVault { @@ -81,13 +81,10 @@ impl ScriptedVault { let Ok((mut stream, _)) = listener.accept().await else { return; }; - let Some(request_line) = read_request(&mut stream).await else { + let Some(request) = read_request(&mut stream).await else { continue; }; - recorded - .lock() - .expect("scripted vault request log poisoned") - .push(request_line); + recorded.lock().expect("scripted vault request log poisoned").push(request); let response = responses .next() .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted")); @@ -107,14 +104,32 @@ impl ScriptedVault { /// The `METHOD /path` lines of every request served so far, in order. pub(crate) fn requests(&self) -> Vec<String> { - self.requests.lock().expect("scripted vault request log poisoned").clone() + self.requests + .lock() + .expect("scripted vault request log poisoned") + .iter() + .map(|(line, _)| line.clone()) + .collect() + } + + /// The request bodies, in the same order as [`Self::requests`]; empty for + /// bodyless requests. Lets tests assert what a write actually persisted + /// (record contents, check-and-set options), not just that a write happened. + pub(crate) fn request_bodies(&self) -> Vec<String> { + self.requests + .lock() + .expect("scripted vault request log poisoned") + .iter() + .map(|(_, body)| body.clone()) + .collect() } } /// Read one HTTP/1.1 request (head plus content-length body) and return its -/// `METHOD /path` line. Draining the body before responding keeps the client -/// from seeing a connection reset while it is still writing. -async fn read_request(stream: &mut TcpStream) -> Option<String> { +/// `METHOD /path` line together with the body. Draining the body before +/// responding keeps the client from seeing a connection reset while it is +/// still writing. +async fn read_request(stream: &mut TcpStream) -> Option<(String, String)> { let mut buffer = Vec::new(); let mut chunk = [0u8; 4096]; let head_end = loop { @@ -146,14 +161,17 @@ async fn read_request(stream: &mut TcpStream) -> Option<String> { }) .next() .unwrap_or(0); - let mut remaining = content_length.saturating_sub(buffer.len() - head_end); + let mut body = buffer[head_end..].to_vec(); + let mut remaining = content_length.saturating_sub(body.len()); while remaining > 0 { let read = stream.read(&mut chunk).await.ok()?; if read == 0 { break; } + body.extend_from_slice(&chunk[..read]); remaining = remaining.saturating_sub(read); } + body.truncate(content_length); - Some(format!("{method} {path}")) + Some((format!("{method} {path}"), String::from_utf8_lossy(&body).into_owned())) } diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 24d9c9c39..287ad4eb0 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -113,6 +113,29 @@ struct VaultKeyVersionRecord { /// Sub-path (under each key path) reserved for immutable version records. const KEY_VERSIONS_SUBPATH: &str = "versions"; +/// Upper bound on read-modify-write attempts for a check-and-set lifecycle +/// mutation. A lost race means live contention on the key; each retry re-reads +/// and re-validates, and a small bound keeps a pathologically contended key +/// from spinning while still absorbing ordinary interleavings. +const LIFECYCLE_CAS_ATTEMPTS: u32 = 3; + +/// Typed error for a lifecycle write that lost its check-and-set race (and, in +/// the retry loop, kept losing it up to the attempt bound). +fn concurrent_modification(key_id: &str) -> KmsError { + KmsError::invalid_operation(format!("Concurrent modification of key {key_id} detected, retry the operation")) +} + +/// Decision returned by a [`VaultKmsClient::update_key_data_with_cas`] +/// mutation closure. +enum CasMutation<T> { + /// Persist the mutated record with a check-and-set write, then yield the + /// value. + Write(T), + /// The freshly observed state already settles the operation; yield the + /// value without writing. + Skip(T), +} + /// Drop KV2 directory entries from a key listing. /// /// Once a key has version records, listing the key prefix returns both the key @@ -293,7 +316,23 @@ impl VaultKmsClient { let encrypted_material = if version == key_data.version { key_data.encrypted_key_material.clone() } else { - self.get_key_version_record(key_id, version).await?.encrypted_key_material + let record = self.get_key_version_record(key_id, version).await?; + if version > key_data.version { + // The requested version has an immutable record, yet the + // current pointer sits below it. Material for a version is + // only requested once an envelope references it, and + // envelopes are only stamped after the pointer switch + // committed — so the pointer must have regressed (a lost + // update rolled back a committed rotation). Fail closed: + // serving in this state would keep new encryptions on the + // rolled-back material. A version with no record at all still + // fails as KeyVersionNotFound above. + return Err(KmsError::internal_error(format!( + "current version {} of key {key_id} is behind existing version record {version}; refusing to use an inconsistent key record", + key_data.version + ))); + } + record.encrypted_key_material }; decode_stored_key_material(key_id, &encrypted_material).inspect_err(|error| { @@ -345,39 +384,84 @@ impl VaultKmsClient { Ok((cas, key_data)) } - /// Check-and-set write of the key record. + /// Check-and-set write of the key record, reporting a lost race as + /// `Ok(None)`. /// /// `cas` must match the KV2 secret version currently holding the record. - /// Returns the secret version created by this write so a caller can chain - /// further check-and-set writes. - async fn cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result<u32> { + /// On success returns the secret version created by this write so a caller + /// can chain further check-and-set writes. + async fn try_cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result<Option<u32>> { let path = self.key_path(key_id); let path = path.as_str(); // Single attempt: replaying a lost-response write would double-apply - // the mutation, and a CAS conflict is a normal concurrency signal that - // must reach the caller untouched. + // the mutation, and a CAS conflict is a normal concurrency signal the + // caller resolves by re-reading, never by resending the same write. let written = self .run("vault_kv2_cas_write_key", OpClass::MutatingNonIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; - kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas }) - .await - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - if is_cas_conflict(&e) { - KmsError::invalid_operation(format!( - "Concurrent modification of key {key_id} detected, retry the rotation" - )) - } else { - KmsError::backend_error(format!("Failed to store key in Vault: {e}")) - } - }) - }) + match kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas }).await + { + Ok(written) => Ok(Some(written)), + Err(e) if is_cas_conflict(&e) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to store key in Vault: {e}")) + })), + } }) .await?; - u32::try_from(written.version) - .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32"))) + written + .map(|written| { + u32::try_from(written.version) + .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32"))) + }) + .transpose() + } + + /// Check-and-set write of the key record, surfacing a lost race as the + /// typed concurrent-modification error. + async fn cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result<u32> { + self.try_cas_store_key_data(key_id, key_data, cas) + .await? + .ok_or_else(|| concurrent_modification(key_id)) + } + + /// Apply a lifecycle mutation to the key record as a check-and-set + /// read-modify-write loop. + /// + /// Every attempt re-reads the record pinned to its current KV2 secret + /// version, re-derives the mutation from that fresh snapshot — `mutate` + /// must re-run its state gate, so a transition that lost a race against + /// e.g. a rotation or a cancellation is re-validated against the committed + /// state instead of being replayed — and writes back check-and-set against + /// exactly the version it read. A conflict means another writer committed + /// in between; after [`LIFECYCLE_CAS_ATTEMPTS`] lost races the typed + /// conflict error is surfaced to the caller. + /// + /// This loop does not bypass the operation policy's single-attempt rule + /// for `MutatingNonIdempotent` writes: each individual write is still sent + /// at most once and never replayed on a lost response. Only the whole + /// read-gate-mutate-write cycle repeats, and every repeat is derived from + /// newly observed state, so the two layers compose instead of conflicting. + async fn update_key_data_with_cas<T, F>(&self, key_id: &str, mut mutate: F) -> Result<(VaultKeyData, T)> + where + F: FnMut(&mut VaultKeyData) -> Result<CasMutation<T>>, + { + for attempt in 1..=LIFECYCLE_CAS_ATTEMPTS { + let (cas, mut key_data) = self.get_key_data_versioned(key_id).await?; + match mutate(&mut key_data)? { + CasMutation::Skip(value) => return Ok((key_data, value)), + CasMutation::Write(value) => { + if self.try_cas_store_key_data(key_id, &key_data, cas).await?.is_some() { + return Ok((key_data, value)); + } + debug!(key_id, attempt, "Vault KV2 lifecycle write lost a check-and-set race; re-reading"); + } + } + } + + Err(concurrent_modification(key_id)) } /// Create-only write of an immutable version record (KV2 check-and-set of 0). @@ -405,14 +489,42 @@ impl VaultKmsClient { .await } - /// Store key data in Vault + /// Create-only write of the top-level key record (KV2 check-and-set of 0). + /// + /// Returns `Ok(true)` when this call created the record and `Ok(false)` + /// when a record already exists — i.e. a concurrent create committed + /// first. An existing record is never overwritten. + async fn try_create_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<bool> { + let path = self.key_path(key_id); + let path = path.as_str(); + + // Single attempt: the create-only CAS makes a duplicate replay fail + // with a conflict, which create_key reports as the key already + // existing, so retrying here would only mask that signal. + self.run("vault_kv2_create_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas: 0 }).await { + Ok(_) => Ok(true), + Err(e) if is_cas_conflict(&e) => Ok(false), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to store key in Vault: {e}")) + })), + } + }) + .await + } + + /// Blind, last-writer-wins overwrite of the key record. + /// + /// Test-only: production writes go through the create-only or + /// check-and-set paths so concurrent writers cannot silently clobber each + /// other. Kept for tests that need to inject corrupted or downgraded + /// records. + #[cfg(test)] async fn store_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> { let path = self.key_path(key_id); let path = path.as_str(); - // Single attempt: this is a whole-record overwrite without a CAS - // precondition, so a replay after a lost response could clobber a - // concurrent writer. self.run("vault_kv2_write_key", OpClass::MutatingNonIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; kv2::set(&vault.client, &self.kv_mount, path, key_data) @@ -431,40 +543,28 @@ impl VaultKmsClient { async fn store_key_metadata(&self, key_id: &str, request: &CreateKeyRequest) -> Result<()> { debug!("Storing key metadata for {}, input tags: {:?}", key_id, request.tags); - // Get existing key data to preserve encrypted_key_material and other fields - // This is called after create_key, so the key should already exist - let existing_key_data = self.get_key_data(key_id).await?; + // Read-modify-write under check-and-set: only the request-driven + // fields change, everything else — most importantly the key material, + // version and status — is carried over from the freshly read record, + // so a rotation or state transition landing in between is preserved + // instead of clobbered. + self.update_key_data_with_cas(key_id, |key_data| { + // A key that was just created must already carry material; an empty value means + // the create flow failed to persist it. Fail closed instead of minting replacement + // material: silently generating a new key here would mask the broken create and + // orphan any DEK already wrapped by a different copy of this key. + if key_data.encrypted_key_material.is_empty() { + warn!(key_id, "Vault KMS key metadata missing encrypted key material"); + return Err(KmsError::material_missing(key_id)); + } - // A key that was just created must already carry material; an empty value means - // the create flow failed to persist it. Fail closed instead of minting replacement - // material: silently generating a new key here would mask the broken create and - // orphan any DEK already wrapped by a different copy of this key. - if existing_key_data.encrypted_key_material.is_empty() { - warn!(key_id, "Vault KMS key metadata missing encrypted key material"); - return Err(KmsError::material_missing(key_id)); - } - - // Update only the metadata fields, preserving the encrypted_key_material - let key_data = VaultKeyData { - algorithm: existing_key_data.algorithm.clone(), - usage: request.key_usage.clone(), - created_at: existing_key_data.created_at, - status: existing_key_data.status, - version: existing_key_data.version, - description: request.description.clone(), - metadata: existing_key_data.metadata.clone(), - tags: request.tags.clone(), - deletion_date: existing_key_data.deletion_date.clone(), - encrypted_key_material: existing_key_data.encrypted_key_material.clone(), // Preserve the key material - baseline_version: existing_key_data.baseline_version, - }; - - debug!( - "VaultKeyData tags before storage: {:?}, encrypted_key_material length: {}", - key_data.tags, - key_data.encrypted_key_material.len() - ); - self.store_key_data(key_id, &key_data).await + key_data.usage = request.key_usage.clone(); + key_data.description = request.description.clone(); + key_data.tags = request.tags.clone(); + Ok(CasMutation::Write(())) + }) + .await?; + Ok(()) } /// Retrieve key data from Vault @@ -520,6 +620,53 @@ impl VaultKmsClient { } } + /// List the names of a key's immutable version records. + /// + /// `None` means the versions directory does not exist — the key was never + /// rotated and has no version records. + async fn list_key_version_records(&self, key_id: &str) -> Result<Option<Vec<String>>> { + let versions_dir = self.key_versions_dir(key_id); + let versions_dir = versions_dir.as_str(); + self.run("vault_kv2_list_key_versions", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::list(&vault.client, &self.kv_mount, versions_dir).await { + Ok(versions) => Ok(Some(versions)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to list key version records in Vault: {e}")) + })), + } + }) + .await + } + + /// Fail closed when the version history extends more than one step past + /// the current version. + /// + /// A record exactly one above current is the footprint of an interrupted + /// rotation (material persisted, pointer switch never committed) and is + /// recovered by the next rotation's adopt path. Anything further ahead + /// cannot come from the rotation protocol: it means the top-level record + /// regressed (for example a historical lost update rolled back committed + /// rotations), and extending the history from the rolled-back state would + /// re-mint version numbers that already have immutable records. + async fn ensure_current_version_not_behind(&self, key_id: &str, current_version: u32) -> Result<()> { + let max_recorded = self + .list_key_version_records(key_id) + .await? + .unwrap_or_default() + .iter() + .filter_map(|entry| entry.trim_end_matches('/').parse::<u32>().ok()) + .max(); + + match max_recorded { + Some(max_recorded) if max_recorded > current_version.saturating_add(1) => Err(KmsError::internal_error(format!( + "current version {current_version} of key {key_id} is behind existing version record {max_recorded}; refusing to extend an inconsistent version history" + ))), + _ => Ok(()), + } + } + /// Physically delete a key from Vault storage async fn delete_key(&self, key_id: &str) -> Result<()> { let path = self.key_path(key_id); @@ -531,18 +678,7 @@ impl VaultKmsClient { let versions_dir = self.key_versions_dir(key_id); let versions_dir = versions_dir.as_str(); // `None` means no version records exist (the key was never rotated). - let versions = self - .run("vault_kv2_list_key_versions", OpClass::ReadIdempotent, move || async move { - let vault = self.vault().map_err(AttemptError::fatal)?; - match kv2::list(&vault.client, &self.kv_mount, versions_dir).await { - Ok(versions) => Ok(Some(versions)), - Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), - Err(e) => Err(AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to list key version records in Vault: {e}")) - })), - } - }) - .await?; + let versions = self.list_key_version_records(key_id).await?; for version in versions.unwrap_or_default() { let version_path = format!("{versions_dir}/{version}"); let version_path = version_path.as_str(); @@ -768,8 +904,14 @@ impl VaultKmsClient { baseline_version: None, }; - // Store in Vault - self.store_key_data(key_id, &key_data).await?; + // Create-only write: the not-found pre-check above is only advisory — + // another node can create the same key in between — so the write + // itself must refuse to overwrite. Exactly one of two concurrent + // creates commits; the loser reports the key as already existing + // instead of adopting material it did not persist. + if !self.try_create_key_data(key_id, &key_data).await? { + return Err(KmsError::key_already_exists(key_id)); + } let master_key = MasterKeyInfo { key_id: key_id.to_string(), @@ -853,10 +995,12 @@ impl VaultKmsClient { pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); - let mut key_data = self.get_key_data(key_id).await?; - ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Enable)?; - key_data.status = KeyStatus::Active; - self.store_key_data(key_id, &key_data).await?; + self.update_key_data_with_cas(key_id, |key_data| { + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Enable)?; + key_data.status = KeyStatus::Active; + Ok(CasMutation::Write(())) + }) + .await?; debug!(key_id, "Vault KMS key enabled"); Ok(()) @@ -865,10 +1009,12 @@ impl VaultKmsClient { pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); - let mut key_data = self.get_key_data(key_id).await?; - ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Disable)?; - key_data.status = KeyStatus::Disabled; - self.store_key_data(key_id, &key_data).await?; + self.update_key_data_with_cas(key_id, |key_data| { + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Disable)?; + key_data.status = KeyStatus::Disabled; + Ok(CasMutation::Write(())) + }) + .await?; debug!(key_id, "Vault KMS key disabled"); Ok(()) @@ -901,6 +1047,11 @@ impl VaultKmsClient { decode_stored_key_material(key_id, &key_data.encrypted_key_material) .inspect_err(|error| warn!(key_id, %error, "Vault KMS key material failed validation"))?; + // A version history that already extends past what this rotation would + // commit means the current pointer regressed; fail closed instead of + // re-minting version numbers that have immutable records. + self.ensure_current_version_not_behind(key_id, key_data.version).await?; + // Step 1: freeze the baseline on first rotation. if key_data.baseline_version.is_none() { let baseline = VaultKeyVersionRecord { @@ -1028,31 +1179,27 @@ impl VaultKmsBackend { self.client.credentials.spawn_renewal_task() } - /// Update key metadata in Vault storage - async fn update_key_metadata_in_storage(&self, key_id: &str, metadata: &KeyMetadata) -> Result<()> { - // Get the current key data from Vault - let mut key_data = self.client.get_key_data(key_id).await?; - - // This is a read-modify-write of the whole VaultKeyData document. Refuse to write - // back a record whose key material is missing: persisting it would cement the - // empty-material state under a fresh document version. A damaged key must go - // through an explicit repair operation, not a metadata update. - if key_data.encrypted_key_material.is_empty() { - return Err(KmsError::material_missing(key_id)); - } - - // Update the status based on the new metadata - key_data.status = match metadata.key_state { - KeyState::Enabled => KeyStatus::Active, - KeyState::Disabled => KeyStatus::Disabled, - KeyState::PendingDeletion => KeyStatus::PendingDeletion, - KeyState::Unavailable => KeyStatus::Deleted, - KeyState::PendingImport => KeyStatus::Disabled, // Treat as disabled until import completes - }; - key_data.deletion_date = metadata.deletion_date.clone(); - - // Update the key data in Vault storage - self.client.store_key_data(key_id, &key_data).await?; + /// Mark a key `PendingDeletion` with the given deadline, under + /// check-and-set with per-attempt re-validation. + /// + /// The state gate re-runs on every attempt, so a transition that lost a + /// race (for example against a concurrent schedule or rotation) is + /// re-validated against the committed state. Refuses to write back a + /// record whose key material is missing: persisting it would cement the + /// empty-material state under a fresh document version — a damaged key + /// must go through an explicit repair operation, not a lifecycle update. + async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &Zoned) -> Result<()> { + self.client + .update_key_data_with_cas(key_id, |key_data| { + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?; + if key_data.encrypted_key_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + key_data.status = KeyStatus::PendingDeletion; + key_data.deletion_date = Some(deletion_date.clone()); + Ok(CasMutation::Write(())) + }) + .await?; Ok(()) } } @@ -1183,12 +1330,22 @@ impl KmsBackend for VaultKmsBackend { if key_metadata.key_state == KeyState::PendingDeletion || key_metadata.key_state == KeyState::Unavailable { // Tombstone first: mark the record Deleted before removing it, // so a crash between the two steps leaves a key that is already - // unusable and whose removal can simply be re-run. - if key_metadata.key_state == KeyState::PendingDeletion { - let mut key_data = self.client.get_key_data(key_id).await?; - key_data.status = KeyStatus::Deleted; - self.client.store_key_data(key_id, &key_data).await?; - } + // unusable and whose removal can simply be re-run. Written + // check-and-set with re-validation: a concurrent cancellation + // that commits first wins and fails this removal instead of + // being overwritten. + self.client + .update_key_data_with_cas(key_id, |key_data| match key_data.status { + KeyStatus::Deleted => Ok(CasMutation::Skip(())), + KeyStatus::PendingDeletion => { + key_data.status = KeyStatus::Deleted; + Ok(CasMutation::Write(())) + } + KeyStatus::Active | KeyStatus::Disabled => { + Err(KmsError::invalid_key_state(format!("Key {key_id} is no longer pending deletion"))) + } + }) + .await?; // Force immediate deletion: physically delete the key from Vault storage self.client.delete_key(key_id).await?; @@ -1196,11 +1353,10 @@ impl KmsBackend for VaultKmsBackend { None } else { // For non-pending keys, mark as PendingDeletion + let marked_at = Zoned::now(); + self.mark_key_pending_deletion(key_id, &marked_at).await?; key_metadata.key_state = KeyState::PendingDeletion; - key_metadata.deletion_date = Some(Zoned::now()); - - // Update the key metadata in Vault storage to reflect the new state - self.update_key_metadata_in_storage(key_id, &key_metadata).await?; + key_metadata.deletion_date = Some(marked_at); None } @@ -1216,12 +1372,10 @@ impl KmsBackend for VaultKmsBackend { } let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400); + self.mark_key_pending_deletion(key_id, &deletion_date).await?; key_metadata.key_state = KeyState::PendingDeletion; key_metadata.deletion_date = Some(deletion_date.clone()); - // Update the key metadata in Vault storage to reflect the new state - self.update_key_metadata_in_storage(key_id, &key_metadata).await?; - Some(deletion_date.to_string()) }; @@ -1248,15 +1402,32 @@ impl KmsBackend for VaultKmsBackend { return Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); } + // Persist the reset state back to Vault. Without this the key stays PendingDeletion in + // storage and would still be reaped, so we must fail the request if the write fails + // rather than report a false success. Check-and-set with per-attempt + // re-validation: once the deletion sweep has tombstoned the record, the + // cancellation must fail instead of resurrecting a key whose material + // is about to be (or already is) destroyed. + self.client + .update_key_data_with_cas(key_id, |key_data| match key_data.status { + KeyStatus::PendingDeletion => { + if key_data.encrypted_key_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + key_data.status = KeyStatus::Active; + key_data.deletion_date = None; + Ok(CasMutation::Write(())) + } + KeyStatus::Active | KeyStatus::Disabled | KeyStatus::Deleted => { + Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))) + } + }) + .await?; + // Cancel the deletion by resetting the state key_metadata.key_state = KeyState::Enabled; key_metadata.deletion_date = None; - // Persist the reset state back to Vault. Without this the key stays PendingDeletion in - // storage and would still be reaped, so we must fail the request if the write fails - // rather than report a false success. - self.update_key_metadata_in_storage(key_id, &key_metadata).await?; - Ok(CancelKeyDeletionResponse { key_id: key_id.clone(), key_metadata, @@ -1293,31 +1464,36 @@ impl KmsBackend for VaultKmsBackend { } async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> { - // Vault KV2 offers no compare-and-swap here, so a cancellation racing - // the read below can still lose; the window is a single read-write - // gap and the sweep re-reads on every pass. - let mut key_data = match self.client.get_key_data(key_id).await { - Ok(key_data) => key_data, - Err(KmsError::KeyNotFound { .. }) => return Ok(ExpiredKeyRemoval::Removed), - Err(error) => return Err(error), - }; - match key_data.status { - // Tombstone left by a crashed removal: complete it. - KeyStatus::Deleted => {} - KeyStatus::PendingDeletion => { - match &key_data.deletion_date { - Some(deadline) if deadline <= now => {} + // Tombstone under check-and-set with per-attempt re-validation: a + // cancellation landing between the read and the write makes the write + // conflict, and the re-read then observes the cancelled state and + // reports StateChanged instead of overwriting it. + let settled = self + .client + .update_key_data_with_cas(key_id, |key_data| match key_data.status { + // Tombstone left by a crashed removal: complete it. + KeyStatus::Deleted => Ok(CasMutation::Skip(None)), + KeyStatus::PendingDeletion => match &key_data.deletion_date { + Some(deadline) if deadline <= now => { + // Tombstone first: mark the record Deleted before + // removing it, so a crash between the two steps leaves + // a key that is already unusable and whose removal can + // simply be re-run. + key_data.status = KeyStatus::Deleted; + Ok(CasMutation::Write(None)) + } // Not yet due, or a legacy record without a persisted // deadline — never auto-remove those. - _ => return Ok(ExpiredKeyRemoval::NotExpired), - } - // Tombstone first: mark the record Deleted before removing it, - // so a crash between the two steps leaves a key that is - // already unusable and whose removal can simply be re-run. - key_data.status = KeyStatus::Deleted; - self.client.store_key_data(key_id, &key_data).await?; - } - KeyStatus::Active | KeyStatus::Disabled => return Ok(ExpiredKeyRemoval::StateChanged), + _ => Ok(CasMutation::Skip(Some(ExpiredKeyRemoval::NotExpired))), + }, + KeyStatus::Active | KeyStatus::Disabled => Ok(CasMutation::Skip(Some(ExpiredKeyRemoval::StateChanged))), + }) + .await; + match settled { + Ok((_, Some(outcome))) => return Ok(outcome), + Ok((_, None)) => {} + Err(KmsError::KeyNotFound { .. }) => return Ok(ExpiredKeyRemoval::Removed), + Err(error) => return Err(error), } match self.client.delete_key(key_id).await { @@ -1438,7 +1614,7 @@ mod tests { let (vault, client) = scripted_client(vec![ScriptedResponse::error(503, "sealed")]).await; let error = client - .store_key_data("wired-key", &healthy_key_data()) + .cas_store_key_data("wired-key", &healthy_key_data(), 1) .await .expect_err("the scripted 503 must fail the write"); assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); @@ -2167,10 +2343,12 @@ mod tests { let mut disabled = healthy_key_data(); disabled.status = KeyStatus::Disabled; let vault = ScriptedVault::serve(vec![ - // disable: read the Active record, persist it Disabled. + // disable: versioned read of the Active record, persist it Disabled. + ScriptedResponse::ok(kv2_metadata_read_data(1)), ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), ScriptedResponse::ok(kv2_write_ack()), - // enable: read the Disabled record, persist it Active. + // enable: versioned read of the Disabled record, persist it Active. + ScriptedResponse::ok(kv2_metadata_read_data(2)), ScriptedResponse::ok(kv2_read_data(&disabled)), ScriptedResponse::ok(kv2_write_ack()), ]) @@ -2192,8 +2370,585 @@ mod tests { .expect("KmsBackend::enable_key must persist through the client"); let requests = vault.requests(); - assert_eq!(requests.len(), 4, "each transition is one read plus one write: {requests:?}"); - assert!(requests[0].starts_with("GET ") && requests[2].starts_with("GET "), "{requests:?}"); - assert!(requests[1].starts_with("POST ") && requests[3].starts_with("POST "), "{requests:?}"); + assert_eq!( + requests.len(), + 6, + "each transition is one versioned read (metadata + data) plus one write: {requests:?}" + ); + assert!( + requests[0].starts_with("GET /v1/secret/metadata/") && requests[3].starts_with("GET /v1/secret/metadata/"), + "{requests:?}" + ); + assert!( + requests[1].starts_with("GET /v1/secret/data/") && requests[4].starts_with("GET /v1/secret/data/"), + "{requests:?}" + ); + assert!(requests[2].starts_with("POST ") && requests[5].starts_with("POST "), "{requests:?}"); + + // Both lifecycle writes must carry a check-and-set precondition pinned + // to the KV2 secret version they read. + let bodies = vault.request_bodies(); + for (index, cas) in [(2usize, 1u64), (5, 2)] { + let body: serde_json::Value = serde_json::from_str(&bodies[index]).expect("lifecycle write body must be JSON"); + assert_eq!( + body["options"]["cas"], + serde_json::json!(cas), + "write {index} must be check-and-set: {body}" + ); + } + } + + /// Parse a captured KV2 write body (`{"data": ..., "options": {"cas": N}}`). + fn parse_write_body(body: &str) -> serde_json::Value { + serde_json::from_str(body).expect("KV2 write body must be JSON") + } + + /// KV2 read payload for an immutable version record. + fn kv2_read_version_record_data(record: &VaultKeyVersionRecord) -> serde_json::Value { + serde_json::json!({ + "data": serde_json::to_value(record).expect("serialize version record"), + "metadata": { + "created_time": "2026-01-01T00:00:00Z", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1, + }, + }) + } + + const CAS_CONFLICT_MESSAGE: &str = "check-and-set parameter did not match the current version"; + + /// Base64 material distinct from `healthy_key_data`'s, standing in for the + /// material a concurrent rotation committed. + fn rotated_material() -> String { + general_purpose::STANDARD.encode([0x43u8; 32]) + } + + /// The issue's lost-update scenario: node A disables a key while node B's + /// rotation commits in between. The blind write this replaces would have + /// written A's stale snapshot back — rolling the key from version 2 to + /// version 1 and resurrecting the pre-rotation material, which is exactly + /// what the final-write assertions below reject. Under check-and-set the + /// stale write conflicts, A re-reads, re-passes the state gate against the + /// rotated record, and persists only the status change on top of it. + #[tokio::test] + async fn wired_disable_interleaved_with_rotate_preserves_committed_rotation() { + let pre_rotate = healthy_key_data(); + let mut rotated = healthy_key_data(); + rotated.version = 2; + rotated.baseline_version = Some(1); + rotated.encrypted_key_material = rotated_material(); + + let (vault, client) = scripted_client(vec![ + // Attempt 1: versioned read observes the pre-rotation record... + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&pre_rotate)), + // ...but the concurrent rotation committed KV2 versions 2 and 3 in + // between, so the check-and-set write loses. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // Attempt 2: the re-read observes the rotated record and the write + // pinned to it succeeds. + ScriptedResponse::ok(kv2_metadata_read_data(3)), + ScriptedResponse::ok(kv2_read_data(&rotated)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + client + .disable_key("wired-key", None) + .await + .expect("the disable must retry past the lost race and commit"); + + let requests = vault.requests(); + assert_eq!( + requests, + vec![ + "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(), + "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=1".to_string(), + "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(), + "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=3".to_string(), + "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + ], + "a conflict must trigger exactly one full re-read before the retry write" + ); + + let bodies = vault.request_bodies(); + let first_write = parse_write_body(&bodies[2]); + assert_eq!(first_write["options"]["cas"], serde_json::json!(1), "{first_write}"); + + // The committed write must be the *rotated* snapshot with only the + // status changed. A blind write would have persisted version 1 and the + // pre-rotation material here. + let committed = parse_write_body(&bodies[5]); + assert_eq!(committed["options"]["cas"], serde_json::json!(3), "{committed}"); + assert_eq!(committed["data"]["status"], serde_json::json!("Disabled"), "{committed}"); + assert_eq!( + committed["data"]["version"], + serde_json::json!(2), + "the rotation's version bump must survive: {committed}" + ); + assert_eq!(committed["data"]["baseline_version"], serde_json::json!(1), "{committed}"); + assert_eq!( + committed["data"]["encrypted_key_material"], + serde_json::json!(rotated_material()), + "the rotation's material must survive the disable: {committed}" + ); + } + + #[tokio::test] + async fn wired_create_key_write_is_create_only() { + let (vault, client) = scripted_client(vec![ + // Existence pre-check: not found. + ScriptedResponse::error(404, "not found"), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + let created = client + .create_key("wired-key", "AES_256", None) + .await + .expect("create against an absent key must succeed"); + assert_eq!(created.version, 1); + + let requests = vault.requests(); + assert_eq!( + requests, + vec![ + "GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + ] + ); + + // The write must be create-only (check-and-set of 0). A blind + // overwrite — the pre-CAS behavior — carries no options at all. + let body = parse_write_body(&vault.request_bodies()[1]); + assert_eq!(body["options"]["cas"], serde_json::json!(0), "create must be create-only: {body}"); + } + + /// Concurrent same-name create: both nodes pass the not-found pre-check, + /// exactly one create-only write commits, and the loser reports + /// KeyAlreadyExists instead of overwriting the winner's material (which + /// would permanently orphan every DEK the winner already wrapped). + #[tokio::test] + async fn wired_concurrent_create_loser_reports_key_already_exists() { + let (vault, client) = scripted_client(vec![ + // Existence pre-check: not found (the racing create has not + // committed yet). + ScriptedResponse::error(404, "not found"), + // The create-only write loses: the racing create committed first. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ]) + .await; + + let error = client + .create_key("wired-key", "AES_256", None) + .await + .expect_err("the losing create must fail"); + assert!(matches!(error, KmsError::KeyAlreadyExists { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "the loser must not retry or fall back to a blind write: {requests:?}"); + } + + /// Deletion sweep racing a cancellation: the sweep's tombstone write loses + /// its check-and-set race, the re-read observes the cancelled (Active) + /// record, and the sweep reports StateChanged without deleting anything. + /// The blind tombstone this replaces would have overwritten the committed + /// cancellation and destroyed the key. + #[tokio::test] + async fn wired_expired_key_sweep_yields_to_concurrent_cancellation() { + let now = Zoned::now() + Duration::from_secs(3600); + let mut pending = healthy_key_data(); + pending.status = KeyStatus::PendingDeletion; + pending.deletion_date = Some(Zoned::now()); + + let vault = ScriptedVault::serve(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&pending)), + // The cancellation commits between the read and the tombstone. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ScriptedResponse::ok(kv2_metadata_read_data(2)), + // The re-read observes the cancelled (Active again) record. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let outcome = backend + .remove_expired_key("wired-key", &now) + .await + .expect("the sweep must settle by observing the cancelled state"); + assert_eq!(outcome, ExpiredKeyRemoval::StateChanged); + + let requests = vault.requests(); + assert_eq!(requests.len(), 5, "{requests:?}"); + assert!( + !requests.iter().any(|line| line.starts_with("DELETE ")), + "a sweep that lost to a cancellation must not delete anything: {requests:?}" + ); + // The one write attempt was the check-and-set tombstone. + let body = parse_write_body(&vault.request_bodies()[2]); + assert_eq!(body["options"]["cas"], serde_json::json!(1), "{body}"); + assert_eq!(body["data"]["status"], serde_json::json!("Deleted"), "{body}"); + } + + /// The other half of the cancel × sweep interleaving: once the sweep has + /// tombstoned the record, a cancellation re-validates against the fresh + /// state and fails instead of resurrecting a key whose material is about + /// to be destroyed. + #[tokio::test] + async fn wired_cancel_deletion_after_sweep_tombstone_fails_closed() { + let mut pending = healthy_key_data(); + pending.status = KeyStatus::PendingDeletion; + pending.deletion_date = Some(Zoned::now()); + let mut tombstoned = healthy_key_data(); + tombstoned.status = KeyStatus::Deleted; + + let vault = ScriptedVault::serve(vec![ + // describe_key still observes the pre-sweep PendingDeletion state + // (one read for the key info, one for the stored metadata). + ScriptedResponse::ok(kv2_read_data(&pending)), + ScriptedResponse::ok(kv2_read_data(&pending)), + // The check-and-set update re-reads and observes the tombstone. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(kv2_read_data(&tombstoned)), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let error = backend + .cancel_key_deletion(CancelKeyDeletionRequest { + key_id: "wired-key".to_string(), + }) + .await + .expect_err("cancelling after the sweep tombstoned the key must fail"); + assert!( + matches!(&error, KmsError::InvalidOperation { message } if message.contains("not pending deletion")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert!( + !requests.iter().any(|line| line.starts_with("POST ")), + "a cancellation that lost to the sweep must not write anything: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_schedule_deletion_retries_after_cas_conflict() { + let vault = ScriptedVault::serve(vec![ + // describe_key: key info plus stored metadata. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // Attempt 1 loses its check-and-set race. + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // Attempt 2: the re-read re-passes the state gate and commits. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let response = backend + .delete_key(DeleteKeyRequest { + key_id: "wired-key".to_string(), + pending_window_in_days: Some(7), + force_immediate: Some(false), + }) + .await + .expect("the schedule must retry past the lost race and commit"); + assert!(response.deletion_date.is_some()); + + let requests = vault.requests(); + assert_eq!(requests.len(), 8, "{requests:?}"); + let committed = parse_write_body(&vault.request_bodies()[7]); + assert_eq!(committed["options"]["cas"], serde_json::json!(2), "{committed}"); + assert_eq!(committed["data"]["status"], serde_json::json!("PendingDeletion"), "{committed}"); + assert!( + !committed["data"]["deletion_date"].is_null(), + "the deadline must be persisted: {committed}" + ); + } + + /// Conflict semantics are re-read *and* re-gate: when the re-read after a + /// lost race shows the key was concurrently scheduled for deletion, the + /// state gate rejects the retry instead of blindly re-applying it. + #[tokio::test] + async fn wired_schedule_deletion_regates_after_conflict() { + let mut already_pending = healthy_key_data(); + already_pending.status = KeyStatus::PendingDeletion; + already_pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400)); + + let vault = ScriptedVault::serve(vec![ + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // A concurrent schedule committed first. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(kv2_read_data(&already_pending)), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let error = backend + .delete_key(DeleteKeyRequest { + key_id: "wired-key".to_string(), + pending_window_in_days: Some(7), + force_immediate: Some(false), + }) + .await + .expect_err("the retry must re-run the state gate against the fresh record"); + assert!( + matches!(&error, KmsError::InvalidOperation { message } if message.contains("pending deletion")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 7, "{requests:?}"); + assert_eq!( + requests.iter().filter(|line| line.starts_with("POST ")).count(), + 1, + "the rejected retry must not write again: {requests:?}" + ); + } + + /// The read-modify-write loop is bounded: persistent contention surfaces + /// the typed conflict error after `LIFECYCLE_CAS_ATTEMPTS` full + /// read-gate-write cycles instead of spinning or falling back to a blind + /// write. + #[tokio::test] + async fn wired_lifecycle_cas_retries_are_bounded() { + let mut responses = Vec::new(); + for secret_version in 1..=LIFECYCLE_CAS_ATTEMPTS as u64 { + responses.push(ScriptedResponse::ok(kv2_metadata_read_data(secret_version))); + responses.push(ScriptedResponse::ok(kv2_read_data(&healthy_key_data()))); + responses.push(ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE)); + } + let (vault, client) = scripted_client(responses).await; + + let error = client + .disable_key("wired-key", None) + .await + .expect_err("persistent contention must surface the typed conflict error"); + assert!( + matches!(&error, KmsError::InvalidOperation { message } if message.contains("Concurrent modification")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3 * LIFECYCLE_CAS_ATTEMPTS as usize, "{requests:?}"); + assert_eq!( + requests.iter().filter(|line| line.starts_with("POST ")).count(), + LIFECYCLE_CAS_ATTEMPTS as usize, + "every attempt must be a fresh read-gate-write cycle: {requests:?}" + ); + } + + /// The tags write-back after a create is a check-and-set read-modify-write + /// that carries the key material over from the freshly read record. + #[tokio::test] + async fn wired_create_key_tags_writeback_is_check_and_set() { + let vault = ScriptedVault::serve(vec![ + // create_key: existence pre-check misses, create-only write lands. + ScriptedResponse::error(404, "not found"), + ScriptedResponse::ok(kv2_write_ack()), + // store_key_metadata: versioned read plus check-and-set write. + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let response = backend + .create_key(CreateKeyRequest { + key_name: Some("wired-key".to_string()), + key_usage: KeyUsage::EncryptDecrypt, + tags: HashMap::from([("team".to_string(), "storage".to_string())]), + ..Default::default() + }) + .await + .expect("create with tags must succeed"); + assert_eq!(response.key_id, "wired-key"); + + let bodies = vault.request_bodies(); + let create = parse_write_body(&bodies[1]); + assert_eq!(create["options"]["cas"], serde_json::json!(0), "{create}"); + + let writeback = parse_write_body(&bodies[4]); + assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}"); + assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}"); + assert_eq!( + writeback["data"]["encrypted_key_material"], + serde_json::json!(healthy_key_data().encrypted_key_material), + "the write-back must preserve the material of the freshly read record: {writeback}" + ); + } + + /// A version record above the current pointer means the top-level record + /// regressed (a lost update rolled back a committed rotation). Resolving + /// material through such a record must fail closed instead of quietly + /// serving it while new encryptions keep using the rolled-back material. + #[tokio::test] + async fn wired_decrypt_fails_closed_when_current_version_regressed() { + let material_v2 = [0x43u8; 32]; + let record_v2 = VaultKeyVersionRecord { + version: 2, + encrypted_key_material: general_purpose::STANDARD.encode(material_v2), + created_at: Zoned::now(), + }; + // A well-formed envelope wrapped under version 2 — under a reverted + // guard this decrypt would *succeed*, which is exactly the masked + // rollback this test pins down. + let (encrypted_key, nonce) = AesDekCrypto::new() + .encrypt(&material_v2, b"dek-plaintext") + .await + .expect("wrap test DEK"); + let envelope = DataKeyEnvelope { + key_id: "dek".to_string(), + master_key_id: "wired-key".to_string(), + key_spec: "AES_256".to_string(), + encrypted_key, + nonce, + encryption_context: HashMap::new(), + created_at: Zoned::now(), + master_key_version: Some(2), + }; + let ciphertext = serde_json::to_vec(&envelope).expect("serialize envelope"); + + let (vault, client) = scripted_client(vec![ + // Top-level record: current version rolled back to 1. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // ...yet the immutable record for version 2 exists. + ScriptedResponse::ok(kv2_read_version_record_data(&record_v2)), + ]) + .await; + + let error = client + .decrypt( + &DecryptRequest { + ciphertext, + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("a version record above the current pointer must fail the decrypt"); + assert!( + matches!(&error, KmsError::InternalError { message } if message.contains("behind existing version record")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "the inconsistency must be decided from the two reads: {requests:?}"); + } + + /// Rotation refuses to extend a version history whose records already + /// reach more than one step past the current pointer: that state cannot + /// come from the rotation protocol and re-minting those version numbers + /// would collide with immutable records. + #[tokio::test] + async fn wired_rotate_fails_closed_when_version_history_regressed() { + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(4)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // Version records reach 3 while the current pointer says 1. + ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2", "3"] })), + ]) + .await; + + let error = client + .rotate_key("wired-key", None) + .await + .expect_err("a regressed version history must fail the rotation"); + assert!( + matches!(&error, KmsError::InternalError { message } if message.contains("refusing to extend")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "{requests:?}"); + assert!( + !requests.iter().any(|line| line.starts_with("POST ")), + "nothing may be written on a regressed history: {requests:?}" + ); + } + + /// A record exactly one past the current pointer is the footprint of an + /// interrupted rotation; the next rotation must adopt its persisted + /// material (the monotonicity guard must not misread it as a regression). + #[tokio::test] + async fn wired_rotate_adopts_interrupted_rotation_record() { + let mut key_data = healthy_key_data(); + key_data.baseline_version = Some(1); + let adopted_material = rotated_material(); + let record_v2 = VaultKeyVersionRecord { + version: 2, + encrypted_key_material: adopted_material.clone(), + created_at: Zoned::now(), + }; + + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(5)), + ScriptedResponse::ok(kv2_read_data(&key_data)), + // The interrupted rotation left a record for version 2. + ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })), + // The create-only write for version 2 conflicts... + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // ...so the rotation reads the persisted record back and adopts it. + ScriptedResponse::ok(kv2_read_version_record_data(&record_v2)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + let rotated = client + .rotate_key("wired-key", None) + .await + .expect("an interrupted rotation must be recoverable"); + assert_eq!(rotated.version, 2); + + // The pointer switch must commit the adopted (persisted) material, not + // freshly generated material that no record holds. + let committed = parse_write_body(&vault.request_bodies()[5]); + assert_eq!(committed["options"]["cas"], serde_json::json!(5), "{committed}"); + assert_eq!(committed["data"]["version"], serde_json::json!(2), "{committed}"); + assert_eq!( + committed["data"]["encrypted_key_material"], + serde_json::json!(adopted_material), + "{committed}" + ); } } From 8763cd0c67dc641d25c8b67e420c8eeec573a318 Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 10:05:33 +0800 Subject: [PATCH 48/52] fix(kms): CAS Transit metadata writes and bound the metadata cache (#5520) * fix(kms): drop the KmsClient trait import removed by #5501 The local backup export tests (#5499) merged after #5501 folded the KmsClient trait into KmsBackend, leaving a dead trait import that breaks 'cargo test -p rustfs-kms' compilation on main. create_key is an inherent method on LocalKmsClient since #5501, so the import is unnecessary. * fix(kms): CAS Transit metadata writes and bound the metadata cache Transit KV metadata writes were whole-record overwrites with no precondition, so two nodes mutating the same key could silently clobber each other's lifecycle state, and the process-local metadata cache had neither a TTL nor a capacity bound, so a key disabled or scheduled for deletion on one node stayed usable on every other node until restart. - Replace write_metadata_to_kv with a versioned read (read_metadata_from_kv_versioned) plus a check-and-set write (cas_write_metadata_to_kv); mutate_key_metadata re-reads the authoritative record and re-runs the state gate on every attempt, and a lost CAS race retries with a fresh snapshot (bounded budget) instead of replaying the stale one. - Migrate every read-modify-write caller: enable, disable, schedule and cancel deletion, rotate version bump, the expired-key tombstone, and both create paths (create-only CAS that read-confirms the winner on a lost race). - Bound the metadata cache with moka (300s TTL, 1024 entries) and drop a key's entry when a transit data call reports it gone server-side. - Fail closed when the synthesized-metadata fallback cannot be read or persisted: the fabricated Enabled record is only served after a durable create-only CAS write, closing the gate weakening documented as a KNOWN RISK; the persistence fallback for pre-metadata keys is kept. Refs rustfs/backlog#1581 (part of rustfs/backlog#1562) --- crates/kms/src/backends/vault_transit.rs | 1107 +++++++++++++++++----- 1 file changed, 878 insertions(+), 229 deletions(-) diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 2b01e8478..2c8db47d7 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -27,25 +27,53 @@ use crate::types::*; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use jiff::Zoned; +use moka::future::Cache; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::sync::Arc; use std::time::Duration; -use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; use tracing::info; use vaultrs::{ + api::kv2::requests::SetSecretRequestOptions, api::transit::{ KeyType, requests::{ CreateKeyRequestBuilder, DecryptDataRequestBuilder, EncryptDataRequestBuilder, UpdateKeyConfigurationRequestBuilder, }, }, + error::ClientError, kv2, transit::{data, key}, }; +/// Attempt budget for metadata read-modify-write cycles: every check-and-set +/// conflict triggers a fresh read plus state-gate re-validation, never a blind +/// replay of the stale snapshot. +const METADATA_CAS_ATTEMPTS: usize = 3; + +/// TTL bound on cached metadata records. This caps how long one node can keep +/// acting on lifecycle state another node has since changed (disable, +/// schedule-deletion): the divergence window is one TTL instead of "until +/// process restart". Matches the manager-level `KmsCache` TTL. +const METADATA_CACHE_TTL: Duration = Duration::from_secs(300); + +/// Capacity bound on the metadata cache so an unbounded key namespace cannot +/// grow process memory without limit. +const METADATA_CACHE_CAPACITY: u64 = 1024; + +/// Whether a KV2 write failed its check-and-set precondition. +/// +/// Mirrors the helper of the same name in `vault.rs`; the two backends keep +/// separate copies because they share no private module. +fn is_cas_conflict(error: &ClientError) -> bool { + matches!( + error, + ClientError::APIError { code: 400, errors } if errors.iter().any(|message| message.contains("check-and-set")) + ) +} + #[derive(Debug, Clone)] struct TransitKeyMetadata { key_usage: KeyUsage, @@ -88,12 +116,16 @@ impl TransitKeyMetadata { } } - // KNOWN RISK (rustfs/backlog#1571, residual of rustfs/backlog#808): this - // fallback defaults to Enabled, so a key whose KV metadata read fails is - // treated as usable — a disabled or pending-deletion key can transiently - // "revive" on that path. State gates therefore only hold as strongly as - // metadata reads do. Changing the fallback is out of scope here; the - // synthesized_metadata_defaults_to_enabled test pins the current behavior. + // Fallback record for transit keys created before metadata persistence + // existed (rustfs#4256 / rustfs#4262): those keys have no KV record at + // all, and failing closed on the missing record would brick every one of + // them, so the record defaults to Enabled to match their pre-persistence + // behavior. The historical fail-open around it (rustfs/backlog#808, + // rustfs/backlog#1571: any metadata read failure yielded a usable Enabled + // key) is resolved for rustfs/backlog#1581: `get_key_metadata` only serves + // this record after durably persisting it with a create-only + // check-and-set, and any read or persist failure on that path fails + // closed. fn synthesized() -> Self { Self { key_usage: KeyUsage::EncryptDecrypt, @@ -148,7 +180,10 @@ pub struct VaultTransitKmsClient { metadata_kv_mount: String, /// Path prefix under metadata_kv_mount for storing transit key metadata records metadata_key_prefix: String, - metadata_cache: RwLock<HashMap<String, TransitKeyMetadata>>, + /// Process-local metadata cache, TTL- and capacity-bounded (see + /// [`METADATA_CACHE_TTL`]): a lifecycle change made by another node + /// becomes visible here within one TTL window at the latest. + metadata_cache: Cache<String, TransitKeyMetadata>, /// Budgets wrapping every outbound Vault call (see `crate::policy`). retry: RetryPolicy, /// Cancellation point for the operation executor: aborts in-flight @@ -179,7 +214,10 @@ impl VaultTransitKmsClient { metadata_kv_mount: config.metadata_kv_mount.clone(), metadata_key_prefix: config.metadata_key_prefix.clone(), config, - metadata_cache: RwLock::new(HashMap::new()), + metadata_cache: Cache::builder() + .max_capacity(METADATA_CACHE_CAPACITY) + .time_to_live(METADATA_CACHE_TTL) + .build(), retry: RetryPolicy::from_config(kms_config), cancel: CancellationToken::new(), }) @@ -276,11 +314,7 @@ impl VaultTransitKmsClient { } data::encrypt(&vault.client, &self.config.mount_path, key_id, plaintext_b64, Some(&mut builder)) .await - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to encrypt data with Vault Transit key {key_id}: {e}")) - }) - }) + .map_err(|e| AttemptError::from_vaultrs(e, |e| Self::map_vault_error(key_id, e, "encrypt"))) }) .await?; @@ -305,11 +339,7 @@ impl VaultTransitKmsClient { } data::decrypt(&vault.client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) .await - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to decrypt data with Vault Transit key {key_id}: {e}")) - }) - }) + .map_err(|e| AttemptError::from_vaultrs(e, |e| Self::map_vault_error(key_id, e, "decrypt"))) }) .await?; @@ -339,28 +369,87 @@ impl VaultTransitKmsClient { .await } - async fn write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { + /// Read the persisted metadata record together with the KV2 secret version + /// holding it, so a later write can be check-and-set against exactly this + /// snapshot. `None` means no record exists (a pre-persistence key). + async fn read_metadata_from_kv_versioned(&self, key_id: &str) -> Result<Option<(u32, TransitKeyMetadata)>> { + let path = self.metadata_key_path(key_id); + let path = path.as_str(); + + let kv_metadata = self + .run("vault_transit_read_metadata_version", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::read_metadata(&vault.client, &self.metadata_kv_mount, path).await { + Ok(metadata) => Ok(Some(metadata)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to read transit key metadata version from Vault KV: {e}")) + })), + } + }) + .await?; + let Some(kv_metadata) = kv_metadata else { + return Ok(None); + }; + let cas = u32::try_from(kv_metadata.current_version) + .map_err(|_| KmsError::backend_error(format!("KV2 secret version for transit key {key_id} metadata exceeds u32")))?; + + // Read the exact secret version named by the metadata so the + // (cas, record) pair stays consistent even if another writer lands in + // between the two reads. + let secret_version = kv_metadata.current_version; + let record: Option<TransitKeyMetadataPersisted> = self + .run("vault_transit_read_metadata_at_version", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::read_version(&vault.client, &self.metadata_kv_mount, path, secret_version).await { + Ok(persisted) => Ok(Some(persisted)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to read transit key metadata from Vault KV: {e}")) + })), + } + }) + .await?; + + Ok(record.map(|persisted| (cas, persisted.into()))) + } + + /// Check-and-set write of the metadata record. + /// + /// `cas` must match the KV2 secret version currently holding the record + /// (0 = create-only). Returns `Ok(false)` when the precondition failed — a + /// concurrent writer landed first — so the caller re-reads instead of + /// clobbering. Single attempt: replaying a lost-response write would + /// double-apply the mutation, and a CAS conflict is a normal concurrency + /// signal, not a backend failure. + async fn cas_write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata, cas: u32) -> Result<bool> { let path = self.metadata_key_path(key_id); let path = path.as_str(); let persisted: TransitKeyMetadataPersisted = metadata.clone().into(); let persisted = &persisted; - // Single attempt: this is a whole-record overwrite without a CAS - // precondition, so a replay after a lost response could clobber a - // concurrent writer. - self.run("vault_transit_write_metadata", OpClass::MutatingNonIdempotent, move || async move { + self.run("vault_transit_cas_write_metadata", OpClass::MutatingNonIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; - kv2::set(&vault.client, &self.metadata_kv_mount, path, persisted) + match kv2::set_with_options(&vault.client, &self.metadata_kv_mount, path, persisted, SetSecretRequestOptions { cas }) .await - .map(|_| ()) - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}")) - }) - }) + { + Ok(_) => Ok(true), + Err(e) if is_cas_conflict(&e) => Ok(false), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}")) + })), + } }) .await } + /// The error surfaced when a metadata read-modify-write exhausts its + /// [`METADATA_CAS_ATTEMPTS`] budget without winning a check-and-set write. + fn metadata_cas_conflict(key_id: &str) -> KmsError { + KmsError::invalid_operation(format!( + "Concurrent modification of transit key {key_id} metadata detected; retry the operation" + )) + } + async fn delete_metadata_from_kv(&self, key_id: &str) -> Result<()> { let path = self.metadata_key_path(key_id); let path = path.as_str(); @@ -414,45 +503,103 @@ impl VaultTransitKmsClient { } async fn get_key_metadata(&self, key_id: &str) -> Result<TransitKeyMetadata> { - // Check in-memory cache first. - if let Some(metadata) = self.metadata_cache.read().await.get(key_id).cloned() { + // Check in-memory cache first (TTL-bounded, so a stale entry can only + // survive one TTL window). + if let Some(metadata) = self.metadata_cache.get(key_id).await { return Ok(metadata); } - // On cache miss, try reading from the persistent KV store. - if let Some(persisted) = self.read_metadata_from_kv(key_id).await? { - self.metadata_cache - .write() - .await - .insert(key_id.to_string(), persisted.clone()); - return Ok(persisted); - } + for _ in 0..METADATA_CAS_ATTEMPTS { + // On cache miss, try reading from the persistent KV store. + if let Some(persisted) = self.read_metadata_from_kv(key_id).await? { + self.metadata_cache.insert(key_id.to_string(), persisted.clone()).await; + return Ok(persisted); + } - // Deliberate exemption from the "read paths never write" rule (rustfs#4256 / - // rustfs#4262): transit keys created before metadata persistence existed have no - // KV record at all, so failing closed here would brick every pre-existing transit - // key. The synthesised record only describes metadata — key material lives solely - // inside Vault's transit engine and is never generated or written by this path. - // - // Verify the transit key actually exists in Vault before synthesising. - self.read_transit_key(key_id).await?; - let metadata = TransitKeyMetadata::synthesized(); - // Persist the synthesised metadata so future cache misses pick it up (best - // effort: the KV write failing must not fail the read). - let _ = self.write_metadata_to_kv(key_id, &metadata).await; - self.metadata_cache.write().await.insert(key_id.to_string(), metadata.clone()); - Ok(metadata) + // Deliberate exemption from the "read paths never write" rule (rustfs#4256 / + // rustfs#4262): transit keys created before metadata persistence existed have no + // KV record at all, so failing closed here would brick every pre-existing transit + // key. The synthesised record only describes metadata — key material lives solely + // inside Vault's transit engine and is never generated or written by this path. + // + // Verify the transit key actually exists in Vault before synthesising. + self.read_transit_key(key_id).await?; + let metadata = TransitKeyMetadata::synthesized(); + // Fail closed on the persist (rustfs/backlog#1581): the synthesised record is + // only served once it is durable, so every node gates on the same stored + // state; a failed KV write must fail the read instead of minting a usable + // Enabled record out of thin air. The create-only check-and-set keeps two + // nodes from fabricating divergent records — losing that race loops back to + // re-read the winner's record. + if self.cas_write_metadata_to_kv(key_id, &metadata, 0).await? { + self.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + return Ok(metadata); + } + } + Err(Self::metadata_cas_conflict(key_id)) } - async fn store_key_metadata(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { - self.write_metadata_to_kv(key_id, metadata).await?; - self.metadata_cache.write().await.insert(key_id.to_string(), metadata.clone()); - Ok(()) + /// Create-only write of the metadata record (check-and-set of 0). + /// + /// Returns `Ok(false)` when a record already exists — a concurrent creator + /// won the race — and never overwrites it; the caller reconciles by + /// reading the stored record back. + async fn create_key_metadata(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<bool> { + if self.cas_write_metadata_to_kv(key_id, metadata, 0).await? { + self.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + return Ok(true); + } + Ok(false) + } + + /// Read-modify-write of the persisted metadata record under KV2 + /// check-and-set. + /// + /// Every attempt re-reads the authoritative record, re-runs `apply` — + /// which owns state-gate validation — against that fresh snapshot, and + /// writes back with the snapshot's KV2 secret version as the check-and-set + /// precondition, so a concurrent writer is never clobbered blind. Losing + /// the race drops the (now stale) cache entry and retries with a fresh + /// read; exhausting the budget surfaces the conflict to the caller. + async fn mutate_key_metadata<F>(&self, key_id: &str, mut apply: F) -> Result<TransitKeyMetadata> + where + F: FnMut(&mut TransitKeyMetadata) -> Result<()>, + { + for _ in 0..METADATA_CAS_ATTEMPTS { + let (cas, mut metadata) = match self.read_metadata_from_kv_versioned(key_id).await? { + Some(snapshot) => snapshot, + None => { + // Pre-persistence key without a KV record (see + // get_key_metadata): mutate the synthesised record and + // create it with a create-only check-and-set so two nodes + // cannot fabricate divergent records. + self.read_transit_key(key_id).await?; + (0, TransitKeyMetadata::synthesized()) + } + }; + apply(&mut metadata)?; + if self.cas_write_metadata_to_kv(key_id, &metadata, cas).await? { + self.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + return Ok(metadata); + } + self.metadata_cache.invalidate(key_id).await; + } + Err(Self::metadata_cas_conflict(key_id)) + } + + /// Drop the cached metadata record when a transit data-path call failed in + /// a way that signals the cached lifecycle state diverged from Vault (the + /// key is gone server-side), so the next state gate re-reads the + /// authoritative record instead of trusting the stale entry until its TTL. + async fn invalidate_metadata_on_state_error(&self, key_id: &str, error: &KmsError) { + if matches!(error, KmsError::KeyNotFound { .. }) { + self.metadata_cache.invalidate(key_id).await; + } } async fn delete_key_metadata(&self, key_id: &str) -> Result<()> { self.delete_metadata_from_kv(key_id).await?; - self.metadata_cache.write().await.remove(key_id); + self.metadata_cache.invalidate(key_id).await; Ok(()) } @@ -514,9 +661,16 @@ impl VaultTransitKmsClient { .await?; let plaintext_key = generate_key_material(&request.key_spec)?; - let encrypted_key = self + let encrypted_key = match self .transit_encrypt(&request.master_key_id, &plaintext_key, &request.encryption_context) - .await?; + .await + { + Ok(encrypted_key) => encrypted_key, + Err(error) => { + self.invalidate_metadata_on_state_error(&request.master_key_id, &error).await; + return Err(error); + } + }; let envelope = DataKeyEnvelope { key_id: uuid::Uuid::new_v4().to_string(), @@ -545,9 +699,16 @@ impl VaultTransitKmsClient { let metadata = self .ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt) .await?; - let ciphertext = self + let ciphertext = match self .transit_encrypt(&request.key_id, &request.plaintext, &request.encryption_context) - .await?; + .await + { + Ok(ciphertext) => ciphertext, + Err(error) => { + self.invalidate_metadata_on_state_error(&request.key_id, &error).await; + return Err(error); + } + }; Ok(EncryptResponse { ciphertext: ciphertext.into_bytes(), @@ -575,8 +736,16 @@ impl VaultTransitKmsClient { let encrypted_key = std::str::from_utf8(&envelope.encrypted_key) .map_err(|e| KmsError::cryptographic_error("utf8", format!("Invalid Transit ciphertext: {e}")))?; - self.transit_decrypt(&envelope.master_key_id, encrypted_key, &envelope.encryption_context) + match self + .transit_decrypt(&envelope.master_key_id, encrypted_key, &envelope.encryption_context) .await + { + Ok(plaintext) => Ok(plaintext), + Err(error) => { + self.invalidate_metadata_on_state_error(&envelope.master_key_id, &error).await; + Err(error) + } + } } /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. @@ -598,59 +767,68 @@ impl VaultTransitKmsClient { // this create would have produced; report it as the create result. // Anything else keeps failing. A failed pre-check read must fail the // create rather than fall through to re-creating over an unknown key. - match self.read_transit_key(key_id).await { - Ok(_) => { - let existing = self.get_key_metadata(key_id).await?; - return if existing.key_state == KeyState::Enabled && existing.key_usage == KeyUsage::EncryptDecrypt { - info!( - key_id, - "Vault Transit create found an identical enabled key; treating it as a recovered create" - ); - Ok(MasterKeyInfo { - key_id: key_id.to_string(), - version: existing.current_version, - algorithm: algorithm.to_string(), - usage: existing.key_usage, - status: KeyStatus::Active, - description: existing.description, - metadata: existing.tags.clone(), - created_at: existing.created_at, - rotated_at: None, - created_by: existing.created_by, - deletion_date: None, - }) - } else { - Err(KmsError::key_already_exists(key_id)) - }; + // + // Two passes: losing the create-only metadata check-and-set race loops + // back here so the pre-check read-confirms the winning record. + for _ in 0..2 { + match self.read_transit_key(key_id).await { + Ok(_) => { + let existing = self.get_key_metadata(key_id).await?; + return if existing.key_state == KeyState::Enabled && existing.key_usage == KeyUsage::EncryptDecrypt { + info!( + key_id, + "Vault Transit create found an identical enabled key; treating it as a recovered create" + ); + Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: existing.current_version, + algorithm: algorithm.to_string(), + usage: existing.key_usage, + status: KeyStatus::Active, + description: existing.description, + metadata: existing.tags.clone(), + created_at: existing.created_at, + rotated_at: None, + created_by: existing.created_by, + deletion_date: None, + }) + } else { + Err(KmsError::key_already_exists(key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } - Err(KmsError::KeyNotFound { .. }) => {} - Err(error) => return Err(error), + + self.create_transit_key(key_id).await?; + + let metadata = TransitKeyMetadata { + created_by: Some("vault-transit".to_string()), + ..TransitKeyMetadata::from_create_request(&CreateKeyRequest { + key_name: Some(key_id.to_string()), + ..Default::default() + }) + }; + if self.create_key_metadata(key_id, &metadata).await? { + return Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: metadata.current_version, + algorithm: algorithm.to_string(), + usage: metadata.key_usage, + status: KeyStatus::Active, + description: metadata.description, + metadata: metadata.tags, + created_at: metadata.created_at, + rotated_at: None, + created_by: metadata.created_by, + deletion_date: None, + }); + } + // A concurrent creator persisted metadata first; make sure the + // pre-check reads their record, not a stale cache entry. + self.metadata_cache.invalidate(key_id).await; } - - self.create_transit_key(key_id).await?; - - let metadata = TransitKeyMetadata { - created_by: Some("vault-transit".to_string()), - ..TransitKeyMetadata::from_create_request(&CreateKeyRequest { - key_name: Some(key_id.to_string()), - ..Default::default() - }) - }; - self.store_key_metadata(key_id, &metadata).await?; - - Ok(MasterKeyInfo { - key_id: key_id.to_string(), - version: metadata.current_version, - algorithm: algorithm.to_string(), - usage: metadata.key_usage, - status: KeyStatus::Active, - description: metadata.description, - metadata: metadata.tags, - created_at: metadata.created_at, - rotated_at: None, - created_by: metadata.created_by, - deletion_date: None, - }) + Err(KmsError::key_already_exists(key_id)) } /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. @@ -708,17 +886,26 @@ impl VaultTransitKmsClient { pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { // A pending deletion must be reverted through cancel_key_deletion, not - // silently by enabling, so the gate rejects PendingDeletion here. - let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Enable).await?; - metadata.key_state = KeyState::Enabled; - metadata.deletion_date = None; - self.store_key_metadata(key_id, &metadata).await + // silently by enabling, so the gate rejects PendingDeletion here. The + // gate runs inside the check-and-set loop against every fresh snapshot. + self.mutate_key_metadata(key_id, |metadata| { + ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::Enable)?; + metadata.key_state = KeyState::Enabled; + metadata.deletion_date = None; + Ok(()) + }) + .await + .map(|_| ()) } pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Disable).await?; - metadata.key_state = KeyState::Disabled; - self.store_key_metadata(key_id, &metadata).await + self.mutate_key_metadata(key_id, |metadata| { + ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::Disable)?; + metadata.key_state = KeyState::Disabled; + Ok(()) + }) + .await + .map(|_| ()) } /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. @@ -729,12 +916,15 @@ impl VaultTransitKmsClient { pending_window_days: u32, _context: Option<&OperationContext>, ) -> Result<()> { - let mut metadata = self - .ensure_key_state_allows(key_id, StateGatedOperation::ScheduleDeletion) - .await?; - metadata.key_state = KeyState::PendingDeletion; - metadata.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); - self.store_key_metadata(key_id, &metadata).await + let deletion_date = Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400); + self.mutate_key_metadata(key_id, |metadata| { + ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?; + metadata.key_state = KeyState::PendingDeletion; + metadata.deletion_date = Some(deletion_date.clone()); + Ok(()) + }) + .await + .map(|_| ()) } pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> { @@ -755,9 +945,15 @@ impl VaultTransitKmsClient { }) .await?; - let mut metadata = self.get_key_metadata(key_id).await?; - metadata.current_version += 1; - self.store_key_metadata(key_id, &metadata).await?; + let metadata = self + .mutate_key_metadata(key_id, |metadata| { + // The transit rotation above has already happened; recording + // the version bump must not be blocked by a concurrent + // lifecycle transition, so no state gate here. + metadata.current_version += 1; + Ok(()) + }) + .await?; Ok(MasterKeyInfo { key_id: key_id.to_string(), @@ -788,6 +984,15 @@ impl VaultTransitKmsClient { } } +#[cfg(test)] +impl VaultTransitKmsClient { + /// Rebuild the metadata cache with test-controlled bounds so TTL and + /// capacity behavior can be exercised without real sleeps. + fn rebuild_metadata_cache_for_tests(&mut self, capacity: u64, ttl: Duration) { + self.metadata_cache = Cache::builder().max_capacity(capacity).time_to_live(ttl).build(); + } +} + pub struct VaultTransitKmsBackend { client: VaultTransitKmsClient, } @@ -835,59 +1040,68 @@ impl KmsBackend for VaultTransitKmsBackend { // what this request would have written, report it as the create // result; any divergence keeps failing so a create can never adopt or // reshape a key it would not have produced. - match self.client.read_transit_key(&key_id).await { - Ok(_) => { - let existing = self.client.get_key_metadata(&key_id).await?; - let requested = TransitKeyMetadata::from_create_request(&request); - return if existing.key_state == KeyState::Enabled - && existing.key_usage == requested.key_usage - && existing.description == requested.description - && existing.tags == requested.tags - { - info!( - key_id, - "Vault Transit create found an identical enabled key; treating it as a recovered create" - ); - Ok(CreateKeyResponse { - key_id: key_id.clone(), - key_metadata: KeyMetadata { + // + // Two passes: losing the create-only metadata check-and-set race loops + // back here so the pre-check read-confirms the winning record. + for _ in 0..2 { + match self.client.read_transit_key(&key_id).await { + Ok(_) => { + let existing = self.client.get_key_metadata(&key_id).await?; + let requested = TransitKeyMetadata::from_create_request(&request); + return if existing.key_state == KeyState::Enabled + && existing.key_usage == requested.key_usage + && existing.description == requested.description + && existing.tags == requested.tags + { + info!( key_id, - key_state: existing.key_state, - key_usage: existing.key_usage, - description: existing.description, - creation_date: existing.created_at, - deletion_date: existing.deletion_date, - origin: existing.origin, - key_manager: "VAULT_TRANSIT".to_string(), - tags: existing.tags, - }, - }) - } else { - Err(KmsError::key_already_exists(&key_id)) - }; + "Vault Transit create found an identical enabled key; treating it as a recovered create" + ); + Ok(CreateKeyResponse { + key_id: key_id.clone(), + key_metadata: KeyMetadata { + key_id, + key_state: existing.key_state, + key_usage: existing.key_usage, + description: existing.description, + creation_date: existing.created_at, + deletion_date: existing.deletion_date, + origin: existing.origin, + key_manager: "VAULT_TRANSIT".to_string(), + tags: existing.tags, + }, + }) + } else { + Err(KmsError::key_already_exists(&key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } - Err(KmsError::KeyNotFound { .. }) => {} - Err(error) => return Err(error), + + self.client.create_transit_key(&key_id).await?; + let metadata = TransitKeyMetadata::from_create_request(&request); + if self.client.create_key_metadata(&key_id, &metadata).await? { + return Ok(CreateKeyResponse { + key_id: key_id.clone(), + key_metadata: KeyMetadata { + key_id, + key_state: metadata.key_state, + key_usage: metadata.key_usage, + description: metadata.description, + creation_date: metadata.created_at, + deletion_date: metadata.deletion_date, + origin: metadata.origin, + key_manager: "VAULT_TRANSIT".to_string(), + tags: metadata.tags, + }, + }); + } + // A concurrent creator persisted metadata first; make sure the + // pre-check reads their record, not a stale cache entry. + self.client.metadata_cache.invalidate(&key_id).await; } - - self.client.create_transit_key(&key_id).await?; - let metadata = TransitKeyMetadata::from_create_request(&request); - self.client.store_key_metadata(&key_id, &metadata).await?; - - Ok(CreateKeyResponse { - key_id: key_id.clone(), - key_metadata: KeyMetadata { - key_id, - key_state: metadata.key_state, - key_usage: metadata.key_usage, - description: metadata.description, - creation_date: metadata.created_at, - deletion_date: metadata.deletion_date, - origin: metadata.origin, - key_manager: "VAULT_TRANSIT".to_string(), - tags: metadata.tags, - }, - }) + Err(KmsError::key_already_exists(&key_id)) } async fn encrypt(&self, request: EncryptRequest) -> Result<EncryptResponse> { @@ -946,10 +1160,14 @@ impl KmsBackend for VaultTransitKmsBackend { self.client.delete_key_metadata(&key_id).await?; None } else { - let mut metadata = self.client.get_key_metadata(&key_id).await?; - metadata.key_state = KeyState::PendingDeletion; - metadata.deletion_date = Some(Zoned::now()); - self.client.store_key_metadata(&key_id, &metadata).await?; + let now = Zoned::now(); + self.client + .mutate_key_metadata(&key_id, |metadata| { + metadata.key_state = KeyState::PendingDeletion; + metadata.deletion_date = Some(now.clone()); + Ok(()) + }) + .await?; key_metadata = self.client.key_metadata_response(&key_id).await?; None } @@ -961,11 +1179,17 @@ impl KmsBackend for VaultTransitKmsBackend { return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30")); } - let mut metadata = self.client.get_key_metadata(&key_id).await?; let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400); - metadata.key_state = KeyState::PendingDeletion; - metadata.deletion_date = Some(scheduled.clone()); - self.client.store_key_metadata(&key_id, &metadata).await?; + self.client + .mutate_key_metadata(&key_id, |metadata| { + // Re-run the gate against every fresh snapshot: the check + // above used a possibly cached record. + ensure_key_state_permits(&key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?; + metadata.key_state = KeyState::PendingDeletion; + metadata.deletion_date = Some(scheduled.clone()); + Ok(()) + }) + .await?; key_metadata = self.client.key_metadata_response(&key_id).await?; Some(scheduled.to_string()) }; @@ -978,14 +1202,20 @@ impl KmsBackend for VaultTransitKmsBackend { } async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> { - let mut metadata = self.client.get_key_metadata(&request.key_id).await?; - if metadata.key_state != KeyState::PendingDeletion { - return Err(KmsError::invalid_key_state(format!("Key {} is not pending deletion", request.key_id))); - } - - metadata.key_state = KeyState::Enabled; - metadata.deletion_date = None; - self.client.store_key_metadata(&request.key_id, &metadata).await?; + let key_id = request.key_id.as_str(); + self.client + .mutate_key_metadata(key_id, |metadata| { + // Re-checked against every fresh snapshot: a concurrent sweep + // that tombstoned the key must fail this cancel, not be + // overwritten blind. + if metadata.key_state != KeyState::PendingDeletion { + return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); + } + metadata.key_state = KeyState::Enabled; + metadata.deletion_date = None; + Ok(()) + }) + .await?; Ok(CancelKeyDeletionResponse { key_id: request.key_id.clone(), @@ -1033,27 +1263,51 @@ impl KmsBackend for VaultTransitKmsBackend { Err(error) => return Err(error), } - // A metadata read failure synthesizes an Enabled record (see - // TransitKeyMetadata::synthesized), which lands in StateChanged below: - // the worker never destroys material based on synthesized state. - let mut metadata = self.client.get_key_metadata(key_id).await?; - match metadata.key_state { - // Tombstone left by a crashed removal: complete it. - KeyState::Unavailable => {} - KeyState::PendingDeletion => { - match &metadata.deletion_date { - Some(deadline) if deadline <= now => {} - // Not yet due, or no persisted deadline — never auto-remove. - _ => return Ok(ExpiredKeyRemoval::NotExpired), - } - // Tombstone first: an Unavailable record is rejected by every - // state gate, and a crashed removal can simply be re-run. - metadata.key_state = KeyState::Unavailable; - self.client.store_key_metadata(key_id, &metadata).await?; - } - KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => { + // Tombstone under check-and-set: every attempt re-reads the record and + // re-validates state and due-ness, so a cancel_key_deletion racing the + // sweep either lands before the tombstone (the re-read sees Enabled + // and the sweep backs off) or after it (the cancel's own + // check-and-set write fails). + let mut tombstoned = false; + for _ in 0..METADATA_CAS_ATTEMPTS { + let Some((cas, mut metadata)) = self.client.read_metadata_from_kv_versioned(key_id).await? else { + // No persisted lifecycle record (pre-persistence key): the + // worker never destroys material whose scheduling state was + // never recorded. return Ok(ExpiredKeyRemoval::StateChanged); + }; + match metadata.key_state { + // Tombstone left by a crashed removal: complete it. + KeyState::Unavailable => { + tombstoned = true; + } + KeyState::PendingDeletion => { + match &metadata.deletion_date { + Some(deadline) if deadline <= now => {} + // Not yet due, or no persisted deadline — never auto-remove. + _ => return Ok(ExpiredKeyRemoval::NotExpired), + } + // Tombstone first: an Unavailable record is rejected by every + // state gate, and a crashed removal can simply be re-run. + metadata.key_state = KeyState::Unavailable; + if self.client.cas_write_metadata_to_kv(key_id, &metadata, cas).await? { + self.client.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + tombstoned = true; + } else { + // Lost the check-and-set race — most likely a + // concurrent cancel; re-read and re-decide. + self.client.metadata_cache.invalidate(key_id).await; + continue; + } + } + KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => { + return Ok(ExpiredKeyRemoval::StateChanged); + } } + break; + } + if !tombstoned { + return Err(VaultTransitKmsClient::metadata_cas_conflict(key_id)); } if !self.client.read_transit_key(key_id).await?.deletion_allowed { @@ -1432,10 +1686,14 @@ mod tests { let _ = client.schedule_key_deletion(&key_id, 7, None).await; } - /// Pins the known-risk fallback documented on `TransitKeyMetadata::synthesized`: - /// when KV metadata cannot be read, the synthesized record defaults to Enabled, - /// which weakens every state gate on that path. If this test turns red the - /// fallback semantics changed on purpose — update the comment there as well. + /// The persistence fallback for pre-metadata keys deliberately fabricates + /// an Enabled record (rustfs#4256 / rustfs#4262): those keys were usable + /// before metadata persistence existed and must stay usable once the + /// record is durably persisted. The old fail-open this test used to pin — + /// a failed metadata read or persist still yielded a usable Enabled key — + /// was flipped to fail closed for rustfs/backlog#1581; that side is + /// covered by `wired_encrypt_fails_closed_when_the_metadata_read_fails` + /// and `wired_synthesized_metadata_is_not_served_when_the_persist_fails`. #[test] fn synthesized_metadata_defaults_to_enabled() { let metadata = TransitKeyMetadata::synthesized(); @@ -1457,15 +1715,21 @@ mod tests { #[tokio::test] async fn wired_backend_lifecycle_overrides_reach_the_client() { let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; let vault = ScriptedVault::serve(vec![ - // disable: metadata cache miss reads KV, then persists Disabled. + // disable: versioned read (secret metadata + pinned version), then + // the check-and-set write persisting Disabled. + ScriptedResponse::ok(kv2_metadata_read_data(1)), ScriptedResponse::ok(metadata_read_data(&metadata)), ScriptedResponse::ok(kv2_write_ack()), - // enable: the state gate hits the metadata cache, so only the - // persisting write goes out. + // enable: another versioned read against the Disabled record, then + // the check-and-set write persisting Enabled. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&disabled)), ScriptedResponse::ok(kv2_write_ack()), - // rotate: the gate hits the cache again; the single rotate - // attempt fails and must not be retried. + // rotate: the state gate hits the metadata cache; the single + // rotate attempt fails and must not be retried. ScriptedResponse::error(503, "standby"), ]) .await; @@ -1493,7 +1757,392 @@ mod tests { assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); let requests = vault.requests(); - assert_eq!(requests.len(), 4, "gated reads, two writes and one rotate attempt: {requests:?}"); - assert_eq!(requests[3], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); + assert_eq!(requests.len(), 7, "two versioned read+write cycles plus one rotate attempt: {requests:?}"); + assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); + } + + /// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the + /// current secret version used as the check-and-set base. + fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value { + serde_json::json!({ + "cas_required": false, + "created_time": "2026-01-01T00:00:00Z", + "current_version": current_version, + "delete_version_after": "0s", + "max_versions": 0, + "oldest_version": 0, + "updated_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "versions": {}, + }) + } + + const CAS_CONFLICT_MESSAGE: &str = "check-and-set parameter did not match the current version"; + + const METADATA_PATH: &str = "/v1/secret/data/rustfs/kms/transit-metadata/wired-key"; + const METADATA_VERSION_PATH: &str = "/v1/secret/metadata/rustfs/kms/transit-metadata/wired-key"; + + #[tokio::test] + async fn wired_disable_retries_past_a_cas_conflict_with_a_fresh_read() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // The conflict must trigger a fresh versioned read, then the + // write is check-and-set against the new snapshot. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + client + .disable_key("wired-key", None) + .await + .expect("a single check-and-set conflict must be absorbed by a re-read"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 6, "two read+read+write cycles: {requests:?}"); + assert_eq!(requests[0], format!("GET {METADATA_VERSION_PATH}")); + assert_eq!(requests[1], format!("GET {METADATA_PATH}?version=1")); + assert_eq!(requests[2], format!("POST {METADATA_PATH}")); + assert_eq!(requests[3], format!("GET {METADATA_VERSION_PATH}"), "conflict must re-read: {requests:?}"); + assert_eq!(requests[4], format!("GET {METADATA_PATH}?version=2")); + assert_eq!(requests[5], format!("POST {METADATA_PATH}")); + } + + #[tokio::test] + async fn wired_disable_cas_conflict_budget_is_bounded() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut responses = Vec::new(); + for cycle in 0..3u64 { + responses.push(ScriptedResponse::ok(kv2_metadata_read_data(cycle + 1))); + responses.push(ScriptedResponse::ok(metadata_read_data(&enabled))); + responses.push(ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE)); + } + let (vault, client) = scripted_client(responses).await; + + let error = client + .disable_key("wired-key", None) + .await + .expect_err("exhausting the check-and-set budget must surface the conflict"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!( + error.to_string().contains("Concurrent modification"), + "the error must name the conflict: {error}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 9, "exactly three read+read+write cycles, no blind replays: {requests:?}"); + } + + #[tokio::test] + async fn wired_cas_conflict_reread_revalidates_the_state_gate() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + pending.key_state = KeyState::PendingDeletion; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // The concurrent writer scheduled the key for deletion; the + // re-read must re-run the state gate and reject the disable. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&pending)), + ]) + .await; + + let error = client + .disable_key("wired-key", None) + .await + .expect_err("the re-read state gate must reject a pending-deletion key"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("pending deletion"), "got {error}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 5, "the gate rejection must not issue another write: {requests:?}"); + assert!(requests[4].starts_with("GET "), "{requests:?}"); + } + + #[tokio::test] + async fn wired_encrypt_key_not_found_invalidates_the_cached_metadata() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; + let (vault, client) = scripted_client(vec![ + // First encrypt: gate reads Enabled and caches it, then the + // transit call reports the key gone server-side. + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::error(404, "encryption key not found"), + // Second encrypt: the state error must have dropped the cache + // entry, so the gate re-reads and sees the Disabled record. + ScriptedResponse::ok(metadata_read_data(&disabled)), + ]) + .await; + + let request = EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }; + let error = client + .encrypt(&request, None) + .await + .expect_err("the scripted 404 must fail the encrypt"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}"); + + let error = client + .encrypt(&request, None) + .await + .expect_err("the re-read Disabled record must reject the encrypt"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!( + requests.len(), + 3, + "the second gate must re-read instead of trusting the stale Enabled entry, \ + and must not reach the encrypt endpoint: {requests:?}" + ); + assert_eq!(requests[2], format!("GET {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn wired_metadata_cache_ttl_expiry_forces_a_fresh_read() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; + let (vault, mut client) = scripted_client(vec![ + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })), + // Post-expiry gate read observes the disable another node + // persisted in the meantime. + ScriptedResponse::ok(metadata_read_data(&disabled)), + ]) + .await; + // A 1ns TTL expires between any two awaits, standing in for the real + // 300s bound without a wall-clock sleep. + client.rebuild_metadata_cache_for_tests(METADATA_CACHE_CAPACITY, Duration::from_nanos(1)); + + let request = EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }; + client + .encrypt(&request, None) + .await + .expect("the first encrypt must pass the Enabled gate"); + + let error = client + .encrypt(&request, None) + .await + .expect_err("after TTL expiry the gate must see the remote disable"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "the expired entry must force a fresh KV read: {requests:?}"); + assert_eq!(requests[2], format!("GET {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn metadata_cache_capacity_is_bounded() { + let records: Vec<_> = (0..3) + .map(|_| { + ScriptedResponse::ok(metadata_read_data(&TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()))) + }) + .collect(); + let (_vault, mut client) = scripted_client(records).await; + client.rebuild_metadata_cache_for_tests(2, METADATA_CACHE_TTL); + + for key_id in ["key-a", "key-b", "key-c"] { + client + .get_key_metadata(key_id) + .await + .expect("each scripted metadata read must succeed"); + } + + client.metadata_cache.run_pending_tasks().await; + assert!( + client.metadata_cache.entry_count() <= 2, + "the cache must not hold more entries than its capacity, got {}", + client.metadata_cache.entry_count() + ); + } + + #[tokio::test] + async fn wired_encrypt_fails_closed_when_the_metadata_read_fails() { + let (vault, client) = scripted_client(vec![ScriptedResponse::error(403, "permission denied")]).await; + + let error = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("a failed metadata read must fail the encrypt, not synthesize Enabled"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 1, "the gate failure must never reach the encrypt endpoint: {requests:?}"); + } + + #[tokio::test] + async fn wired_synthesized_metadata_is_not_served_when_the_persist_fails() { + // Regression for the rustfs/backlog#1581 fail-open flip: a missing + // metadata record used to synthesize a usable Enabled record even when + // persisting it failed, letting encrypt proceed on state no other node + // could observe. The persist failure must now fail the read. + let (vault, client) = scripted_client(vec![ + ScriptedResponse::error(404, "no value found"), + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::error(500, "kv write failed"), + ]) + .await; + + let error = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("an unpersisted synthesized record must never gate an encrypt open"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "read, existence check, failed persist — and no encrypt: {requests:?}"); + assert_eq!(requests[2], format!("POST {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn wired_synthesized_metadata_create_race_adopts_the_winning_record() { + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::error(404, "no value found"), + ScriptedResponse::ok(transit_key_read_data("wired-key")), + // Another node persisted a record first; the create-only + // check-and-set loses and the re-read adopts the winner. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ScriptedResponse::ok(metadata_read_data(&disabled)), + ]) + .await; + + let error = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("the winner's Disabled record must gate the encrypt, not the loser's Enabled one"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 4, "the lost create race must re-read, never overwrite: {requests:?}"); + assert_eq!(requests[3], format!("GET {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn wired_backend_create_loses_the_metadata_create_race_and_read_confirms() { + let winner = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // Pre-check: the transit key does not exist yet. + ScriptedResponse::error(404, "not found"), + // Transit create succeeds, but a concurrent creator persists the + // metadata record first. + ScriptedResponse::ok(serde_json::json!({})), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // Second pass: the pre-check now read-confirms the winner. + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::ok(metadata_read_data(&winner)), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + let response = backend + .create_key(CreateKeyRequest { + key_name: Some("wired-key".to_string()), + ..Default::default() + }) + .await + .expect("losing the metadata create race to an identical record must recover the create"); + assert_eq!(response.key_metadata.key_state, KeyState::Enabled); + + let requests = vault.requests(); + assert_eq!(requests.len(), 5, "one lost create pass plus one read-confirm pass: {requests:?}"); + assert!( + requests[3].starts_with("GET ") && requests[4].starts_with("GET "), + "the recovery pass must be reads only: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() { + let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + pending.key_state = KeyState::PendingDeletion; + pending.deletion_date = Some(Zoned::now()); + let cancelled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // The transit key still exists. + ScriptedResponse::ok(transit_key_read_data("wired-key")), + // Versioned read finds a due pending-deletion record, but the + // tombstone write loses the check-and-set race to a cancel. + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(metadata_read_data(&pending)), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // The re-read sees the cancelled (Enabled) record: back off. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&cancelled)), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + let now = Zoned::now() + Duration::from_secs(3600); + let outcome = backend + .remove_expired_key("wired-key", &now) + .await + .expect("losing the tombstone race to a cancel must back off cleanly"); + assert_eq!(outcome, ExpiredKeyRemoval::StateChanged); + + let requests = vault.requests(); + assert_eq!(requests.len(), 6, "no delete may follow a lost tombstone race: {requests:?}"); + assert!( + !requests + .iter() + .any(|line| line.contains("/transit/keys/wired-key/config") || line.starts_with("DELETE ")), + "the sweep must not touch the transit key after backing off: {requests:?}" + ); } } From b6d4689c750334fe569ccfbdd4fafa67bd23b23f Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 10:05:40 +0800 Subject: [PATCH 49/52] feat(policy): parse and match KMS key resources with a kms:Decrypt action (#5515) Bring the dormant Resource::Kms variant to life: arn:aws:kms:::key/<key_id> patterns (empty-account form, wildcards allowed in the id, alias/<name> reserved as parse-only) now parse, validate, serialize back, and are matched by KMS statements against the requested key id carried in Args::object. Statements without KMS resources keep the legacy match-every-key behaviour, as do call sites that pass no key resource, so nothing changes until the admin/SSE authorization paths start passing key ids. Statement validation rejects KMS resources on non-KMS statements, and bucket policy validation rejects KMS actions and resources outright while stored policies keep deserializing; evaluation skips pure-KMS bucket policy statements with a warning. A kms:Decrypt action is added for the upcoming SSE-KMS read path. Refs rustfs/backlog#1582 (part of rustfs/backlog#1562) --- crates/policy/src/policy.rs | 6 + crates/policy/src/policy/action.rs | 3 + crates/policy/src/policy/policy.rs | 385 ++++++++++++++++++++ crates/policy/src/policy/resource.rs | 103 +++++- crates/policy/src/policy/statement.rs | 116 +++++- crates/policy/tests/policy_eval_proptest.rs | 153 ++++++++ 6 files changed, 751 insertions(+), 15 deletions(-) diff --git a/crates/policy/src/policy.rs b/crates/policy/src/policy.rs index 0c01b9729..651dae0ca 100644 --- a/crates/policy/src/policy.rs +++ b/crates/policy/src/policy.rs @@ -72,4 +72,10 @@ pub enum Error { #[error("invalid resource, type: '{0}', pattern: '{1}'")] InvalidResource(String, String), + + #[error("KMS resources require a statement whose actions are all KMS actions")] + KmsResourceWithNonKmsAction, + + #[error("bucket policies do not support KMS actions or resources")] + KmsUnsupportedInBucketPolicy, } diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index 9d0790dae..fbefb87f1 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -728,6 +728,8 @@ pub enum KmsAction { ListKeysAction, #[strum(serialize = "kms:DescribeKey")] DescribeKeyAction, + #[strum(serialize = "kms:Decrypt")] + DecryptAction, } #[cfg(test)] @@ -764,6 +766,7 @@ mod tests { ("kms:RotateKey", KmsAction::RotateKeyAction), ("kms:ListKeys", KmsAction::ListKeysAction), ("kms:DescribeKey", KmsAction::DescribeKeyAction), + ("kms:Decrypt", KmsAction::DecryptAction), ] { let action = Action::try_from(raw).expect("Should parse KMS action"); assert_eq!(action, Action::KmsAction(expected)); diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index b2e766f5a..7188d3228 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -1760,6 +1760,391 @@ mod test { ); } + fn kms_args<'a>( + action: Action, + key_id: &'a str, + conditions: &'a HashMap<String, Vec<String>>, + claims: &'a HashMap<String, Value>, + ) -> Args<'a> { + Args { + account: "testuser", + groups: &None, + action, + bucket: "", + conditions, + is_owner: false, + object: key_id, + claims, + deny_only: false, + } + } + + #[tokio::test] + async fn test_kms_statement_with_key_resource_scopes_by_key() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DisableKey"], + "Resource": ["arn:aws:kms:::key/key-a"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DisableKeyAction); + + assert!( + policy.is_allowed(&kms_args(action, "key-a", &conditions, &claims)).await, + "the granted key must be allowed" + ); + assert!( + !policy.is_allowed(&kms_args(action, "key-b", &conditions, &claims)).await, + "a key outside the granted resource must be denied" + ); + assert!( + !policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::EnableKeyAction), "key-a", &conditions, &claims)) + .await, + "an action outside the grant must stay denied even for the granted key" + ); + assert!( + policy.is_allowed(&kms_args(action, "", &conditions, &claims)).await, + "call sites that do not pass a key resource keep the legacy match-every-key behaviour" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_with_wildcard_key_resource() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:GenerateDataKey"], + "Resource": ["arn:aws:kms:::key/app-*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::GenerateDataKeyAction); + + assert!( + policy + .is_allowed(&kms_args(action, "app-primary", &conditions, &claims)) + .await + ); + assert!( + !policy + .is_allowed(&kms_args(action, "backup-primary", &conditions, &claims)) + .await + ); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_without_resource_matches_every_key() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + + for key_id in ["", "key-a", "any-other-key"] { + assert!( + policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::DisableKeyAction), key_id, &conditions, &claims)) + .await, + "resource-less KMS statement must keep matching every key (key_id: {key_id:?})" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_kms_deny_with_wildcard_resource_overrides_allow() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:*"], + "Resource": ["arn:aws:kms:::key/key-a"] + }, + { + "Effect": "Deny", + "Action": ["kms:DisableKey"], + "Resource": ["arn:aws:kms:::key/*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + + assert!( + !policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::DisableKeyAction), "key-a", &conditions, &claims)) + .await, + "a wildcard Deny must override the narrower Allow" + ); + assert!( + policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::RotateKeyAction), "key-a", &conditions, &claims)) + .await, + "actions outside the Deny keep the Allow" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_with_not_resource_excludes_keys() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DescribeKey"], + "NotResource": ["arn:aws:kms:::key/prod-*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DescribeKeyAction); + + assert!(policy.is_allowed(&kms_args(action, "dev-key", &conditions, &claims)).await); + assert!(!policy.is_allowed(&kms_args(action, "prod-key", &conditions, &claims)).await); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_with_s3_resource_is_treated_as_unscoped() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + // Statements combining KMS actions with S3 resources predate KMS resource + // support and were always evaluated as if unscoped. Pin that they still + // match every key (a warning is logged during evaluation). + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DisableKey"], + "Resource": ["arn:aws:s3:::somebucket/*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DisableKeyAction); + + for key_id in ["", "key-a"] { + assert!( + policy.is_allowed(&kms_args(action, key_id, &conditions, &claims)).await, + "malformed KMS statement with S3 resources must keep matching every key (key_id: {key_id:?})" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_kms_alias_resource_parses_but_matches_no_key() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DescribeKey"], + "Resource": ["arn:aws:kms:::alias/app-alias"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DescribeKeyAction); + + assert!( + !policy.is_allowed(&kms_args(action, "app-alias", &conditions, &claims)).await, + "alias patterns are parse-only until alias resolution lands and must not match key requests" + ); + assert!( + policy.is_allowed(&kms_args(action, "", &conditions, &claims)).await, + "call sites without a key resource keep the legacy match-every-key behaviour" + ); + + Ok(()) + } + + #[test] + fn test_kms_resource_policy_round_trips() { + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:GenerateDataKey", "kms:Decrypt"], + "Resource": ["arn:aws:kms:::key/app-*", "arn:aws:kms:::alias/app-alias"] + } + ] +}"#, + ) + .expect("KMS resource policy should parse"); + + let json = serde_json::to_string(&policy).expect("policy should serialize"); + let round_trip = Policy::parse_config(json.as_bytes()).expect("serialized KMS resource policy should re-parse"); + assert_eq!(round_trip.statements[0].resources, policy.statements[0].resources); + assert_eq!(round_trip.statements[0].actions, policy.statements[0].actions); + + let value: serde_json::Value = serde_json::from_str(&json).expect("JSON valid"); + let resources: Vec<_> = value["Statement"][0]["Resource"] + .as_array() + .expect("Resource should serialize as an array") + .iter() + .map(|resource| resource.as_str().expect("Resource entries should be strings")) + .collect(); + assert_eq!(resources, vec!["arn:aws:kms:::key/app-*", "arn:aws:kms:::alias/app-alias"]); + } + + #[test] + fn test_kms_resource_with_non_kms_action_is_invalid() { + for action in ["s3:GetObject", "admin:ServerInfo", "sts:AssumeRole"] { + let data = format!( + r#"{{ + "Version": "2012-10-17", + "Statement": [ + {{ + "Effect": "Allow", + "Action": ["{action}"], + "Resource": ["arn:aws:kms:::key/key-a"] + }} + ] +}}"# + ); + + let result = Policy::parse_config(data.as_bytes()); + assert!( + matches!(result.as_ref().unwrap_err(), Error::PolicyError(IamError::KmsResourceWithNonKmsAction)), + "{action} with a KMS resource should fail with KmsResourceWithNonKmsAction, got: {result:?}" + ); + } + } + + #[test] + fn test_bucket_policy_with_kms_statement_is_invalid() { + for statement in [ + r#"{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["kms:GenerateDataKey"],"Resource":["arn:aws:s3:::bucket/*"]}"#, + r#"{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject"],"Resource":["arn:aws:kms:::key/key-a"]}"#, + r#"{"Effect":"Allow","Principal":{"AWS":"*"},"NotAction":["kms:*"],"Resource":["arn:aws:s3:::bucket/*"]}"#, + ] { + let data = format!(r#"{{"Version":"2012-10-17","Statement":[{statement}]}}"#); + let policy: BucketPolicy = + serde_json::from_str(&data).expect("bucket policy with KMS content should still deserialize"); + let result = policy.is_valid(); + assert!( + matches!(result.as_ref().unwrap_err(), Error::PolicyError(IamError::KmsUnsupportedInBucketPolicy)), + "bucket policy statement {statement} should fail with KmsUnsupportedInBucketPolicy, got: {result:?}" + ); + } + } + + #[tokio::test] + async fn test_stored_bucket_policy_with_kms_statement_loads_and_is_ignored() -> Result<()> { + // Policies stored before KMS statements were rejected at validation must keep + // deserializing, and their KMS statements must not affect bucket traffic. + let bucket_policy: BucketPolicy = serde_json::from_str( + r#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["kms:GenerateDataKey"], + "Resource": ["arn:aws:s3:::bucket/*"] + }, + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::bucket/*"] + } + ] +}"#, + )?; + + let conditions = HashMap::new(); + let args = BucketPolicyArgs { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::GetObjectAction), + bucket: "bucket", + conditions: &conditions, + is_owner: false, + object: "a.txt", + }; + assert!( + bucket_policy.is_allowed(&args).await, + "the S3 statement must keep working alongside an ignored KMS statement" + ); + + let put_args = BucketPolicyArgs { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::PutObjectAction), + bucket: "bucket", + conditions: &conditions, + is_owner: false, + object: "a.txt", + }; + assert!( + !bucket_policy.is_allowed(&put_args).await, + "the ignored KMS statement must not grant anything" + ); + + Ok(()) + } + #[test] fn test_mixed_action_families_are_invalid_even_with_resource() { let data = r#" diff --git a/crates/policy/src/policy/resource.rs b/crates/policy/src/policy/resource.rs index 2b6c03e72..262b13f4e 100644 --- a/crates/policy/src/policy/resource.rs +++ b/crates/policy/src/policy/resource.rs @@ -40,7 +40,7 @@ impl Serialize for ResourceSet { for resource in &self.0 { let resource_str = match resource { Resource::S3(value) => format!("{}{}", Resource::S3_PREFIX, value), - Resource::Kms(value) => value.clone(), + Resource::Kms(value) => format!("{}{}", Resource::KMS_PREFIX, value), }; seq.serialize_element(&resource_str)?; } @@ -171,6 +171,20 @@ pub enum Resource { impl Resource { pub const S3_PREFIX: &'static str = "arn:aws:s3:::"; + /// KMS ARNs use the same empty-account form as [`Self::S3_PREFIX`]; the suffix is + /// `key/<key_id>` (wildcards allowed in the id), `alias/<name>`, or a bare `*`. + pub const KMS_PREFIX: &'static str = "arn:aws:kms:::"; + /// Resource-type segment for key ids; request-side KMS resource strings are + /// `key/<key_id>` so they line up with these patterns. + pub const KMS_KEY_SEGMENT: &'static str = "key/"; + /// Resource-type segment reserved for key aliases. Alias patterns parse and + /// validate, but requests are always evaluated against `key/<key_id>` strings, + /// so an alias pattern matches nothing until alias resolution lands. + pub const KMS_ALIAS_SEGMENT: &'static str = "alias/"; + + pub fn is_kms(&self) -> bool { + matches!(self, Resource::Kms(_)) + } pub async fn is_match(&self, resource: &str, conditions: &HashMap<String, Vec<String>>) -> bool { self.is_match_with_resolver(resource, conditions, None).await @@ -228,10 +242,13 @@ impl Resource { impl TryFrom<&str> for Resource { type Error = Error; fn try_from(value: &str) -> std::result::Result<Self, Self::Error> { - let Some(value) = value.strip_prefix(Self::S3_PREFIX) else { + let resource = if let Some(suffix) = value.strip_prefix(Self::S3_PREFIX) { + Resource::S3(suffix.into()) + } else if let Some(suffix) = value.strip_prefix(Self::KMS_PREFIX) { + Resource::Kms(suffix.into()) + } else { return Err(IamError::InvalidResource("unknown".into(), value.into()).into()); }; - let resource = Resource::S3(value.into()); resource.is_valid()?; Ok(resource) @@ -248,13 +265,17 @@ impl Validator for Resource { } } Self::Kms(pattern) => { - if pattern.is_empty() + // A bare `*` matches every key resource; anything else must carry a + // resource-type segment. Key ids never contain separators (the KMS + // backends reject them), while alias names may nest ("alias/aws/s3"). + let well_formed = pattern == "*" || pattern - .char_indices() - .find(|&(_, c)| c == '/' || c == '\\' || c == '.') - .map(|(i, _)| i) - .is_some() - { + .strip_prefix(Self::KMS_KEY_SEGMENT) + .is_some_and(|id| !id.is_empty() && !id.contains('/') && !id.contains('\\')) + || pattern + .strip_prefix(Self::KMS_ALIAS_SEGMENT) + .is_some_and(|name| !name.is_empty() && !name.contains('\\')); + if !well_formed { return Err(IamError::InvalidResource("kms".into(), pattern.into()).into()); } } @@ -270,7 +291,7 @@ impl Serialize for Resource { { match self { Resource::S3(s) => serializer.serialize_str(&format!("{}{}", Self::S3_PREFIX, s)), - Resource::Kms(s) => serializer.serialize_str(s), + Resource::Kms(s) => serializer.serialize_str(&format!("{}{}", Self::KMS_PREFIX, s)), } } } @@ -308,9 +329,71 @@ mod tests { #[test_case("arn:aws:s3:::mybucket","mybucket/myobject" => false; "15")] #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/evil.txt" => false; "16")] #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/safe/../../victim-bucket/evil.txt" => false; "17")] + #[test_case("arn:aws:kms:::key/mykey","key/mykey" => true; "kms exact key")] + #[test_case("arn:aws:kms:::key/mykey","key/otherkey" => false; "kms wrong key")] + #[test_case("arn:aws:kms:::key/*","key/mykey" => true; "kms key wildcard")] + #[test_case("arn:aws:kms:::key/app-*","key/app-primary" => true; "kms key prefix wildcard")] + #[test_case("arn:aws:kms:::key/app-*","key/backup-primary" => false; "kms key prefix mismatch")] + #[test_case("arn:aws:kms:::key/mykey?","key/mykey1" => true; "kms key question mark")] + #[test_case("arn:aws:kms:::*","key/mykey" => true; "kms bare star")] + #[test_case("arn:aws:kms:::key/mykey","key/mykey/../otherkey" => false; "kms traversal cleaned")] + #[test_case("arn:aws:kms:::alias/myalias","key/myalias" => false; "kms alias never matches key")] + #[test_case("arn:aws:kms:::alias/*","key/mykey" => false; "kms alias wildcard never matches key")] + #[test_case("arn:aws:kms:::key/mykey","mykey" => false; "kms bare id lacks key segment")] fn test_resource_is_match(resource: &str, object: &str) -> bool { let resource: Resource = resource.try_into().unwrap(); pollster::block_on(resource.is_match(object, &HashMap::new())) } + + #[test_case("arn:aws:kms:::key/mykey" => true; "key id parses")] + #[test_case("arn:aws:kms:::key/*" => true; "key wildcard parses")] + #[test_case("arn:aws:kms:::key/app-key.v2" => true; "key id with dot parses")] + #[test_case("arn:aws:kms:::alias/myalias" => true; "alias parses")] + #[test_case("arn:aws:kms:::alias/aws/s3" => true; "nested alias parses")] + #[test_case("arn:aws:kms:::*" => true; "bare star parses")] + #[test_case("arn:aws:kms:::" => false; "empty suffix rejected")] + #[test_case("arn:aws:kms:::key/" => false; "empty key id rejected")] + #[test_case("arn:aws:kms:::alias/" => false; "empty alias name rejected")] + #[test_case("arn:aws:kms:::mykey" => false; "missing resource type segment rejected")] + #[test_case("arn:aws:kms:::key/a/b" => false; "separator in key id rejected")] + #[test_case("arn:aws:kms:::key/a\\b" => false; "backslash in key id rejected")] + #[test_case("arn:aws:kms:us-east-1:123456789012:key/mykey" => false; "region and account form rejected")] + fn test_kms_resource_parse(resource: &str) -> bool { + Resource::try_from(resource).is_ok() + } + + #[test] + fn test_kms_resource_serialization_round_trip() { + for raw in [ + "arn:aws:kms:::key/mykey", + "arn:aws:kms:::key/*", + "arn:aws:kms:::alias/myalias", + "arn:aws:kms:::*", + ] { + let resource = Resource::try_from(raw).expect("KMS resource should parse"); + assert!(resource.is_kms()); + + let json = serde_json::to_string(&resource).expect("KMS resource should serialize"); + assert_eq!(json, format!("\"{raw}\""), "serialization must write back the full ARN"); + + let round_trip: Resource = serde_json::from_str(&json).expect("serialized KMS resource should deserialize"); + assert_eq!(round_trip, resource); + } + } + + #[test] + fn test_kms_resource_set_serialization_round_trip() { + use crate::policy::resource::ResourceSet; + + let set: ResourceSet = + serde_json::from_str(r#"["arn:aws:kms:::key/app-*","arn:aws:kms:::alias/myalias"]"#).expect("set should parse"); + assert_eq!(set.len(), 2); + + let json = serde_json::to_string(&set).expect("set should serialize"); + assert_eq!(json, r#"["arn:aws:kms:::key/app-*","arn:aws:kms:::alias/myalias"]"#); + + let round_trip: ResourceSet = serde_json::from_str(&json).expect("serialized set should deserialize"); + assert_eq!(round_trip, set); + } } diff --git a/crates/policy/src/policy/statement.rs b/crates/policy/src/policy/statement.rs index 861c188e1..a45466713 100644 --- a/crates/policy/src/policy/statement.rs +++ b/crates/policy/src/policy/statement.rs @@ -16,6 +16,7 @@ use super::{ ActionSet, Args, BucketPolicyArgs, Effect, Error as IamError, Functions, ID, Principal, ResourceSet, Validator, action::{Action, S3Action}, function::key_name::{KeyName, S3KeyName}, + resource::Resource, variables::{VariableContext, VariableResolver}, }; use crate::error::{Error, Result}; @@ -173,13 +174,79 @@ impl Statement { Some(ActionFamily::Mixed) } + /// Resource scope check for KMS statements, which match `arn:aws:kms:::key/<key_id>` + /// patterns instead of the bucket/object path grammar used by S3 statements. + /// + /// Call-site contract (wired up by the admin/SSE authorization paths): the requested + /// key identifier (pre-alias-resolution) travels in `args.object` with `args.bucket` + /// left empty. An empty `args.object` means the caller did not scope the request to a + /// key, which preserves the legacy match-every-key behaviour. + async fn kms_key_scope_matches(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool { + let kms_resources: Vec<&Resource> = self.resources.iter().filter(|resource| resource.is_kms()).collect(); + let kms_not_resources: Vec<&Resource> = self.not_resources.iter().filter(|resource| resource.is_kms()).collect(); + + if kms_resources.len() != self.resources.len() || kms_not_resources.len() != self.not_resources.len() { + // Statements combining KMS actions with S3 resources predate KMS resource + // support and were always evaluated as if unscoped; keep that behaviour + // but surface it, since the S3 patterns never constrain key access. + tracing::warn!( + sid = %self.sid.0, + "KMS statement carries non-KMS resources; they are ignored and the statement matches every key" + ); + } + + if kms_resources.is_empty() && kms_not_resources.is_empty() { + // No KMS resources: the statement scopes by action only (legacy form). + return true; + } + + if args.object.is_empty() { + // Call sites that do not pass a key resource keep the pre-resource-scoping + // behaviour where any key matches. + return true; + } + + let requested = format!("{}{}", Resource::KMS_KEY_SEGMENT, args.object); + + if !kms_resources.is_empty() { + let mut matched = false; + for resource in kms_resources { + if resource + .is_match_with_resolver(&requested, args.conditions, Some(resolver)) + .await + { + matched = true; + break; + } + } + if !matched { + return false; + } + } + + for resource in kms_not_resources { + if resource + .is_match_with_resolver(&requested, args.conditions, Some(resolver)) + .await + { + return false; + } + } + + true + } + /// Returns true when this statement would reach `conditions.evaluate_with_resolver` in - /// [`Statement::is_allowed`] (including the KMS shortcut path). Does not evaluate conditions. + /// [`Statement::is_allowed`] (including the KMS resource path). Does not evaluate conditions. pub(crate) async fn request_reaches_condition_eval(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool { if (!self.actions.is_match(&args.action) && !self.actions.is_empty()) || self.not_actions.is_match(&args.action) { return false; } + if self.is_kms() { + return self.kms_key_scope_matches(args, resolver).await; + } + let resource = build_resource( &args.action, args.bucket, @@ -187,10 +254,6 @@ impl Statement { self.conditions.references_key_name(&KeyName::S3(S3KeyName::S3Prefix)), ); - if self.is_kms() && (resource == "/" || self.resources.is_empty()) { - return true; - } - if self.resources.is_empty() && self.not_resources.is_empty() && !self.is_admin() && !self.is_sts() { return false; } @@ -273,6 +336,19 @@ impl Validator for Statement { return Err(IamError::BothResourceAndNotResource.into()); } + // KMS resources only make sense on pure-KMS statements. The reverse + // combination (KMS actions with S3 resources) predates KMS resources, + // may already be stored, and stays loadable; evaluation treats it as + // unscoped and warns. + let has_kms_resource = self + .resources + .iter() + .chain(self.not_resources.iter()) + .any(|resource| resource.is_kms()); + if has_kms_resource && !matches!(action_family, Some(ActionFamily::Kms)) { + return Err(IamError::KmsResourceWithNonKmsAction.into()); + } + self.actions.is_valid()?; self.not_actions.is_valid()?; self.resources.is_valid()?; @@ -323,6 +399,18 @@ pub struct BPStatement { impl BPStatement { /// Returns true when this statement would reach `conditions.evaluate` in [`BPStatement::is_allowed`]. pub(crate) async fn request_reaches_condition_eval(&self, args: &BucketPolicyArgs<'_>) -> bool { + if !self.actions.is_empty() && self.actions.iter().all(|action| matches!(action, Action::KmsAction(_))) { + // Bucket policies cannot grant or deny KMS access; such statements are + // rejected at validation but may exist in policies stored before that + // check. Skip them so they never influence bucket traffic. Statements + // mixing KMS with S3 actions keep evaluating their S3 actions as before. + tracing::warn!( + sid = %self.sid.0, + "ignoring bucket policy statement with KMS actions during evaluation" + ); + return false; + } + if !self.principal.is_match(args.account) { return false; } @@ -379,6 +467,24 @@ impl Validator for BPStatement { return Err(IamError::BothActionAndNotAction.into()); } + // Bucket policies govern S3 access; KMS grants belong in identity policies. + // Rejected here (PutBucketPolicy) only: deserialization stays permissive so + // stored policies from before this check keep loading, and evaluation skips + // pure-KMS statements with a warning. + let has_kms_action = self + .actions + .iter() + .chain(self.not_actions.iter()) + .any(|action| matches!(action, Action::KmsAction(_))); + let has_kms_resource = self + .resources + .iter() + .chain(self.not_resources.iter()) + .any(|resource| resource.is_kms()); + if has_kms_action || has_kms_resource { + return Err(IamError::KmsUnsupportedInBucketPolicy.into()); + } + if self.resources.is_empty() && self.not_resources.is_empty() { return Err(IamError::NonResource.into()); } diff --git a/crates/policy/tests/policy_eval_proptest.rs b/crates/policy/tests/policy_eval_proptest.rs index 6ca75e0fc..3de57a0a4 100644 --- a/crates/policy/tests/policy_eval_proptest.rs +++ b/crates/policy/tests/policy_eval_proptest.rs @@ -26,6 +26,12 @@ //! policy is also allowed by the widened policy; //! (c) an empty policy denies every non-owner request (default deny). //! +//! Properties (d)-(g) extend the same invariants to KMS key resources +//! (`arn:aws:kms:::key/<key_id>`, backlog#1582): Deny-first over key scopes, +//! wildcard/resource-less supersets implying concrete key grants, exact key +//! scoping without cross-key leaks, and the legacy resource-less +//! match-every-key compatibility pin. +//! //! Pure evaluation: no IO, no global state, parallel-safe. Statements are built //! from JSON exactly like production policies arriving via PutPolicy. Generated //! bucket/key/action pools avoid wildcard metacharacters so resource patterns @@ -47,10 +53,23 @@ const OBJECT_ACTIONS: &[&str] = &[ "s3:PutObjectTagging", ]; +/// Key-scoped KMS actions safe to pair with an `arn:aws:kms:::key/<id>` resource. +const KMS_KEY_ACTIONS: &[&str] = &[ + "kms:GenerateDataKey", + "kms:Decrypt", + "kms:DisableKey", + "kms:RotateKey", + "kms:DescribeKey", +]; + fn statement_json(effect: &str, action: &str, resource: &str) -> String { format!(r#"{{"Effect":"{effect}","Action":["{action}"],"Resource":["{resource}"]}}"#) } +fn resourceless_statement_json(effect: &str, action: &str) -> String { + format!(r#"{{"Effect":"{effect}","Action":["{action}"]}}"#) +} + fn policy_from_statements(statements: &[String]) -> Policy { let json = format!(r#"{{"Version":"2012-10-17","Statement":[{}]}}"#, statements.join(",")); serde_json::from_str(&json).expect("generated policy JSON should parse") @@ -88,6 +107,22 @@ fn action_strategy() -> impl Strategy<Value = &'static str> { proptest::sample::select(OBJECT_ACTIONS) } +/// Strategy: a KMS key id without wildcard metacharacters or separators. +fn key_id_strategy() -> impl Strategy<Value = String> { + "[a-z][a-z0-9-]{2,11}" +} + +/// Strategy: one action name from the key-scoped KMS action pool. +fn kms_action_strategy() -> impl Strategy<Value = &'static str> { + proptest::sample::select(KMS_KEY_ACTIONS) +} + +/// KMS evaluation contract: the requested key id travels in `args.object` with an +/// empty bucket (see `Statement::kms_key_scope_matches`). +fn is_allowed_for_key(policy: &Policy, action: &str, key_id: &str) -> bool { + is_allowed(policy, action, "", key_id) +} + proptest! { /// (a) Deny anywhere wins: a Deny statement matching the request denies it, /// no matter how many broad Allow statements surround it or at which index @@ -205,4 +240,122 @@ proptest! { "Policy::default() must deny {action} on {bucket}/{key}" ); } + + /// (d) KMS Deny anywhere wins: a Deny scoped to the exact key (or `key/*`) + /// denies the request no matter how many broad KMS Allow statements + /// (resource-less or `key/*`-scoped) surround it. + #[test] + fn kms_explicit_deny_anywhere_denies( + key_id in key_id_strategy(), + action in kms_action_strategy(), + allow_count in 0usize..4, + deny_pos_seed in 0usize..16, + broad in proptest::bool::ANY, + wildcard_deny in proptest::bool::ANY, + ) { + let mut statements: Vec<String> = (0..allow_count) + .map(|_| { + if broad { + resourceless_statement_json("Allow", "kms:*") + } else { + statement_json("Allow", "kms:*", "arn:aws:kms:::key/*") + } + }) + .collect(); + + let deny_resource = if wildcard_deny { + "arn:aws:kms:::key/*".to_string() + } else { + format!("arn:aws:kms:::key/{key_id}") + }; + let deny = statement_json("Deny", action, &deny_resource); + let deny_pos = deny_pos_seed % (statements.len() + 1); + statements.insert(deny_pos, deny); + + let policy = policy_from_statements(&statements); + + if allow_count > 0 { + let mut allows_only = statements.clone(); + allows_only.remove(deny_pos); + let allow_policy = policy_from_statements(&allows_only); + prop_assert!( + is_allowed_for_key(&allow_policy, action, &key_id), + "sanity: the KMS Allow statements alone should permit {action} on key {key_id}" + ); + } + + prop_assert!( + !is_allowed_for_key(&policy, action, &key_id), + "explicit KMS Deny at index {deny_pos} of {} statements must deny {action} on key {key_id}", + statements.len() + ); + } + + /// (e) KMS wildcard superset implies the concrete key grant: whatever an + /// exact `key/<id>` Allow permits is also permitted by `key/*`, by a bare + /// `arn:aws:kms:::*`, and by the legacy resource-less statement form. + #[test] + fn kms_wildcard_superset_implies_concrete_match( + key_id in key_id_strategy(), + action in kms_action_strategy(), + ) { + let narrow = policy_from_statements(&[statement_json( + "Allow", + action, + &format!("arn:aws:kms:::key/{key_id}"), + )]); + let widened = policy_from_statements(&[statement_json("Allow", "kms:*", "arn:aws:kms:::key/*")]); + let star = policy_from_statements(&[statement_json("Allow", "kms:*", "arn:aws:kms:::*")]); + let resourceless = policy_from_statements(&[resourceless_statement_json("Allow", "kms:*")]); + + prop_assert!(is_allowed_for_key(&narrow, action, &key_id), "narrow KMS policy must allow its own grant"); + prop_assert!(is_allowed_for_key(&widened, action, &key_id), "kms:* on key/* must imply the concrete grant"); + prop_assert!(is_allowed_for_key(&star, action, &key_id), "kms:* on arn:aws:kms:::* must imply the concrete grant"); + prop_assert!( + is_allowed_for_key(&resourceless, action, &key_id), + "the legacy resource-less KMS statement must imply the concrete grant" + ); + } + + /// (f) Key scoping is exact: an Allow on `key/<a>` never leaks to a + /// different key id, while the compatibility contract keeps unscoped + /// requests (no key id passed) matching. + #[test] + fn kms_key_scope_does_not_leak_across_keys( + key_a in key_id_strategy(), + key_b in key_id_strategy(), + action in kms_action_strategy(), + ) { + prop_assume!(key_a != key_b); + + let policy = policy_from_statements(&[statement_json( + "Allow", + action, + &format!("arn:aws:kms:::key/{key_a}"), + )]); + + prop_assert!(is_allowed_for_key(&policy, action, &key_a), "the granted key must be allowed"); + prop_assert!( + !is_allowed_for_key(&policy, action, &key_b), + "an Allow scoped to key {key_a} must not leak to key {key_b}" + ); + prop_assert!( + is_allowed_for_key(&policy, action, ""), + "call sites that pass no key resource keep the legacy match-every-key behaviour" + ); + } + + /// (g) Legacy compatibility pin: resource-less KMS statements match every + /// generated key id, exactly as before KMS resources existed. + #[test] + fn kms_resourceless_statement_matches_every_key( + key_id in key_id_strategy(), + action in kms_action_strategy(), + ) { + let policy = policy_from_statements(&[resourceless_statement_json("Allow", action)]); + prop_assert!( + is_allowed_for_key(&policy, action, &key_id), + "resource-less KMS statement must keep matching {action} on key {key_id}" + ); + } } From 739efaaea1603e7ae3659c13ab88c3cf4b05214f Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 10:05:47 +0800 Subject: [PATCH 50/52] feat(kms): restore local backend key material from sealed backup bundles (#5522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): add link_durably primitive and restore-marker startup guard The local backend gains the two pieces the bundle restore path builds on: a no-clobber hard-link publish primitive whose AlreadyExists case is idempotent only for byte-identical content, and a fail-closed startup guard that refuses to open a key directory holding a restore cutover marker. The durable commit protocol and the key-id containment check become pub(crate) so the restore module reuses them instead of copies. * feat(kms): record a master-key verifier and pre-seal decrypt probe in export Fill the manifest's master_key_verifier slot with an opaque one-way value (scheme-prefixed, bound to the backup id and the KDF salt) so a restore can detect a wrong operator-supplied master key before touching any target state, and probe-decrypt every artifact as stored under the backup KEK before the manifest may seal — digest equality alone only proves the ciphertext landed intact. The payload decryption tail is factored out and shared with the restore side so producer and consumer cannot drift on the framing. Also drops the stale KmsClient test import orphaned by the backend refactor (#5501); the test suite did not compile without this. * feat(kms): restore local backend key material from sealed backup bundles The consumer side of the Local bundle export, as a four-phase protocol: - Dry-run: full in-memory bundle decode (digest and AEAD verification of every artifact), KDF-drift detection against the compiled-in derivation, deployment and injected-generation checks (strictly lower is rejected, equal stays allowed for repeated drills), master-key verifier check, and target conflict enumeration - with zero writes. - Staging: artifacts are committed durably into the .restore-staging/ subdirectory (invisible to the backend's key scan and orphan-temp matcher) and every record is decryption-probed with the derived master key both in memory before staging and again from the staged bytes. - Commit marker + cutover: the durably published .restore-commit.json marker is the single commit point; cutover publishes staged files via link_durably (salt first, keys after), then durably removes the marker and drops staging. - Crash re-entry: before the marker the target top level is untouched and a re-run starts over; with the marker published, backend startup fails closed and a re-run with the same bundle rolls forward while abort_local_restore rolls back. Every interruption converges to the complete old or complete new state. Restore never goes through LocalKmsClient::new (which would mint a fresh salt); the only write mode is the explicit restore-into-empty-target policy, where an orphan salt or a foreign marker already counts as non-empty. The bundle source stays strictly read-only. Refs rustfs/backlog#1572 --- crates/kms/src/backends/local.rs | 139 +- crates/kms/src/backup/local_export.rs | 126 +- crates/kms/src/backup/local_restore.rs | 1970 ++++++++++++++++++++++++ crates/kms/src/backup/mod.rs | 12 +- 4 files changed, 2217 insertions(+), 30 deletions(-) create mode 100644 crates/kms/src/backup/local_restore.rs diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 1a12b8593..490e5bdac 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -46,7 +46,10 @@ use zeroize::Zeroizing; /// `key_dir` is accepted, so identifiers already in use by existing deployments keep /// resolving. Only separators, traversal and the degenerate cases are refused, which is /// what stops `key_dir.join(...)` from escaping. -fn validate_key_id(key_id: &str) -> Result<()> { +/// +/// pub(crate) because the backup restore path applies the same containment +/// rule to key identifiers recovered from bundle artifacts. +pub(crate) fn validate_key_id(key_id: &str) -> Result<()> { if key_id.is_empty() { return Err(KmsError::invalid_key("key identifier must not be empty")); } @@ -68,7 +71,15 @@ fn validate_key_id(key_id: &str) -> Result<()> { } } -const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt"; +// The salt and restore-marker file names are pub(crate) so the backup/restore +// modules (`crate::backup`) address the exact on-disk names instead of copies +// that could drift. +pub(crate) const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt"; +/// Commit marker of an in-progress Local restore cutover (see +/// `crate::backup::local_restore`). Its presence means the key directory is +/// mid-cutover: startup must fail closed until the restore is rolled forward +/// or explicitly aborted. +pub(crate) const LOCAL_RESTORE_COMMIT_MARKER_FILE: &str = ".restore-commit.json"; // The KDF parameters are pub(crate) so the backup manifest contract // (`crate::backup`) records the exact compiled-in derivation instead of a // copy that could drift. @@ -87,7 +98,11 @@ pub(crate) const LOCAL_KMS_ARGON2_P_COST: u32 = 1; /// `foo.tmp-<uuid>` is stored as `foo.tmp-<uuid>.key` — so the `.key` guard /// plus the exact hyphenated-UUID check makes it impossible to match an /// authoritative file. -fn is_orphan_commit_temp_name(file_name: &str) -> bool { +/// +/// pub(crate) because the backup restore path applies the same classification +/// when it re-enters an interrupted run: a leftover commit temp is never +/// authoritative state, so it does not make a target non-empty. +pub(crate) fn is_orphan_commit_temp_name(file_name: &str) -> bool { if file_name.ends_with(".key") { return false; } @@ -110,12 +125,15 @@ fn is_orphan_commit_temp_name(file_name: &str) -> bool { /// /// This intentionally mirrors ecstore's fsync helpers without depending on the /// ecstore crate: the KMS backend stays decoupled from storage internals. -mod durable_file { +/// +/// pub(crate) because the backup restore path (`crate::backup::local_restore`) +/// commits staged files and its cutover marker through the same protocol. +pub(crate) mod durable_file { use std::io::{self, Write}; use std::path::{Path, PathBuf}; /// How the fully written temp file becomes visible under its final name. - pub(super) enum Publish { + pub(crate) enum Publish { /// Atomically replace whatever is at the destination via `rename`. Replace, /// Publish via `hard_link`, failing with [`CommitError::AlreadyExists`] @@ -124,7 +142,7 @@ mod durable_file { } #[derive(Debug)] - pub(super) enum CommitError { + pub(crate) enum CommitError { AlreadyExists, Io(io::Error), /// Test-only simulated crash: the protocol stops after the given step @@ -158,14 +176,14 @@ mod durable_file { /// to prove that every interrupted prefix recovers to either the complete /// old state or the complete new state. #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum CommitStep { + pub(crate) enum CommitStep { TempWritten, FileSynced, Published, DirSynced, } - pub(super) async fn commit( + pub(crate) async fn commit( temp_path: PathBuf, final_path: PathBuf, content: Vec<u8>, @@ -179,7 +197,7 @@ mod durable_file { /// Remove a published file durably: without the parent directory fsync a /// deleted key could resurface after power loss. - pub(super) async fn remove_durably(path: PathBuf) -> io::Result<()> { + pub(crate) async fn remove_durably(path: PathBuf) -> io::Result<()> { tokio::task::spawn_blocking(move || { std::fs::remove_file(&path)?; let parent = path @@ -191,6 +209,41 @@ mod durable_file { .map_err(io::Error::other)? } + /// Publish an already-durable file under a second name via `hard_link`, + /// then fsync the destination's parent directory. + /// + /// This is the restore cutover primitive: the source (a staged file that + /// went through [`commit`]) is already durable, so linking plus a parent + /// fsync is a complete publish. `AlreadyExists` is idempotent success only + /// when the destination content is byte-identical to the source — that is + /// exactly the re-entry case of a cutover interrupted after this link — + /// and a hard failure otherwise, so the primitive can never clobber or + /// silently accept foreign state. + pub(crate) async fn link_durably(source: PathBuf, dest: PathBuf) -> io::Result<()> { + tokio::task::spawn_blocking(move || { + match std::fs::hard_link(&source, &dest) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + let existing = std::fs::read(&dest)?; + let staged = std::fs::read(&source)?; + if existing != staged { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("destination {} already exists with different content", dest.display()), + )); + } + } + Err(error) => return Err(error), + } + let parent = dest + .parent() + .ok_or_else(|| io::Error::other("destination has no parent directory"))?; + fsync_dir(parent) + }) + .await + .map_err(io::Error::other)? + } + fn commit_blocking( temp_path: &Path, final_path: &Path, @@ -355,7 +408,7 @@ mod durable_file { /// Test-only failpoints simulating a crash after a given commit step. /// Armed per directory so parallel tests never affect each other. #[cfg(test)] - pub(super) mod failpoint { + pub(crate) mod failpoint { use super::CommitStep; use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -460,6 +513,11 @@ impl LocalKmsClient { debug!(path = ?config.key_dir, "KMS key directory created"); } + // The restore-marker guard must run before anything else touches the + // directory (in particular before salt load/creation): a directory + // mid-cutover holds an arbitrary mix of old and new state. + Self::ensure_no_restore_marker(&config).await?; + // Initialize master cipher if master key is provided let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key { let salt = Self::load_or_create_master_key_salt(&config).await?; @@ -491,6 +549,7 @@ impl LocalKmsClient { if !fs::try_exists(&config.key_dir).await? { return Err(KmsError::configuration_error("Local KMS key directory does not exist")); } + Self::ensure_no_restore_marker(&config).await?; let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key { let legacy_key = Self::derive_legacy_master_key(master_key)?; @@ -567,8 +626,36 @@ impl LocalKmsClient { Self::master_key_salt_path(&self.config) } + /// Operator-configured master key string, exposed for the backup export + /// module so it can record a one-way verifier in the bundle manifest. + /// Never log or persist this value. + pub(crate) fn configured_master_key(&self) -> Option<&str> { + self.config.master_key.as_deref() + } + + /// Fail closed while a restore cutover marker is present: the directory + /// then holds an arbitrary mix of pre-restore and restored state, and the + /// only valid next steps are re-running the restore with the same bundle + /// (roll forward) or explicitly aborting it. This mirrors the missing-salt + /// guard: startup must never paper over a half-applied restore. + async fn ensure_no_restore_marker(config: &LocalConfig) -> Result<()> { + let marker = config.key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + if fs::try_exists(&marker).await? { + return Err(KmsError::configuration_error(format!( + "Local KMS key directory has an unfinished restore (marker {} present); \ + re-run the restore with the same bundle to roll it forward or abort it explicitly", + marker.display() + ))); + } + Ok(()) + } + /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. - fn derive_master_key(master_key: &str, salt: &[u8]) -> Result<Key<Aes256Gcm>> { + /// + /// pub(crate) because the backup restore path derives the same key from + /// the operator-supplied master key and the bundled salt for its verifier + /// check and staged decryption probe. + pub(crate) fn derive_master_key(master_key: &str, salt: &[u8]) -> Result<Key<Aes256Gcm>> { let params = Params::new( LOCAL_KMS_ARGON2_M_COST_KIB, LOCAL_KMS_ARGON2_T_COST, @@ -585,7 +672,7 @@ impl LocalKmsClient { Ok(key) } - fn derive_legacy_master_key(master_key: &str) -> Result<Key<Aes256Gcm>> { + pub(crate) fn derive_legacy_master_key(master_key: &str) -> Result<Key<Aes256Gcm>> { let mut hasher = Sha256::new(); hasher.update(master_key.as_bytes()); hasher.update(b"rustfs-kms-local"); @@ -2453,6 +2540,34 @@ mod tests { assert!(!is_orphan_commit_temp_name("mykey.tmp-")); } + #[tokio::test] + async fn link_durably_publishes_no_clobber_and_is_content_idempotent() { + let temp_dir = TempDir::new().expect("temp dir"); + let source = temp_dir.path().join("staged"); + let dest = temp_dir.path().join("published"); + fs::write(&source, b"staged-content").await.expect("write source"); + + durable_file::link_durably(source.clone(), dest.clone()) + .await + .expect("first link must succeed"); + assert_eq!(fs::read(&dest).await.expect("read dest"), b"staged-content"); + + // Re-entry with identical content is idempotent success — exactly the + // resumed-cutover case. + durable_file::link_durably(source.clone(), dest.clone()) + .await + .expect("re-linking identical content must be idempotent"); + + // Existing content that differs is a hard failure, never a clobber. + let foreign = temp_dir.path().join("foreign"); + fs::write(&foreign, b"different-content").await.expect("write foreign"); + let error = durable_file::link_durably(foreign, dest.clone()) + .await + .expect_err("differing content must not be clobbered"); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert_eq!(fs::read(&dest).await.expect("dest unchanged"), b"staged-content"); + } + #[tokio::test] async fn durable_commit_fsyncs_every_write_path() { use durable_file::fsync_recorder; diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs index 00bb905e8..c0f393057 100644 --- a/crates/kms/src/backup/local_export.rs +++ b/crates/kms/src/backup/local_export.rs @@ -14,9 +14,9 @@ //! Local backend backup export: sealed, KEK-protected bundle production. //! -//! This is the producer side only; restore lives in a follow-up change. The -//! admin API is not wired here either — callers construct the request and -//! supply the backup KEK explicitly. +//! This is the producer side; the consumer side lives in +//! [`crate::backup::local_restore`]. The admin API is not wired here — +//! callers construct the request and supply the backup KEK explicitly. //! //! # Bundle layout //! @@ -63,6 +63,7 @@ use aes_gcm::{ use jiff::Zoned; use rand::RngExt; use serde::Deserialize; +use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; use tokio::fs; use tokio::io::AsyncWriteExt; @@ -76,6 +77,12 @@ const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc"; const AEAD_NONCE_LEN: usize = 12; /// Domain-separation context for the artifact AAD binding. const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1"; +/// Domain-separation context for the master-key verifier. +const MASTER_KEY_VERIFIER_CONTEXT: &str = "rustfs-kms-local-master-key-verifier:v1"; +/// Verifier scheme prefix for the Argon2id (salted) derivation. +pub(crate) const MASTER_KEY_VERIFIER_ARGON2ID_PREFIX: &str = "argon2id-v1:"; +/// Verifier scheme prefix for the legacy pre-beta.9 SHA-256 derivation. +pub(crate) const MASTER_KEY_VERIFIER_LEGACY_PREFIX: &str = "legacy-sha256-v1:"; /// Caller-supplied backup KEK: a trust root deliberately separate from the /// business KMS hierarchy (it must not be a key that is itself part of the @@ -208,7 +215,15 @@ pub async fn export_local_backup( )); } - let manifest = build_and_write_bundle(kek, request, &snapshot).await?; + // The verifier lets a restore detect a wrong operator-supplied master key + // before touching any target state; dev-mode directories have no master + // key to verify. + let master_key_verifier = match client.configured_master_key() { + Some(master_key) => Some(compute_master_key_verifier(master_key, snapshot.salt.as_deref(), &request.backup_id)?), + None => None, + }; + + let manifest = build_and_write_bundle(kek, request, &snapshot, master_key_verifier).await?; Ok(manifest) } @@ -262,6 +277,21 @@ pub async fn decrypt_bundle_artifact( Err(error) => return Err(error.into()), }; + decrypt_artifact_payload(&manifest.backup_id, manifest.snapshot_generation, descriptor, kek, &payload) +} + +/// Verify and decrypt one artifact payload already read into memory. +/// +/// Fail-closed order: declared length, encrypted digest, nonce framing, then +/// AEAD authentication. Shared by [`decrypt_bundle_artifact`] and the export +/// pre-seal probe so producer and consumer can never drift on the framing. +fn decrypt_artifact_payload( + backup_id: &str, + snapshot_generation: u64, + descriptor: &ArtifactDescriptor, + kek: &BackupKek, + payload: &[u8], +) -> Result<Zeroizing<Vec<u8>>> { if (payload.len() as u64) < descriptor.len { return Err(BackupError::truncated(format!( "artifact '{}' is {} bytes, manifest declares {}", @@ -280,7 +310,7 @@ pub async fn decrypt_bundle_artifact( )) .into()); } - if ContentDigest::sha256_of(&payload) != descriptor.encrypted_digest { + if ContentDigest::sha256_of(payload) != descriptor.encrypted_digest { return Err(BackupError::corrupted(format!("artifact '{}' does not match its manifest digest", descriptor.path)).into()); } if payload.len() < AEAD_NONCE_LEN { @@ -290,7 +320,7 @@ pub async fn decrypt_bundle_artifact( let (nonce_bytes, ciphertext) = payload.split_at(AEAD_NONCE_LEN); let mut nonce = [0u8; AEAD_NONCE_LEN]; nonce.copy_from_slice(nonce_bytes); - let aad = artifact_aad(&manifest.backup_id, manifest.snapshot_generation, &descriptor.path); + let aad = artifact_aad(backup_id, snapshot_generation, &descriptor.path); let plaintext = kek .cipher() .decrypt( @@ -360,6 +390,7 @@ async fn build_and_write_bundle( kek: &BackupKek, request: &LocalBackupExportRequest, snapshot: &CollectedSnapshot, + master_key_verifier: Option<String>, ) -> Result<BackupManifest> { let mut artifacts = Vec::with_capacity(snapshot.records.len() + 1); for record in &snapshot.records { @@ -392,7 +423,7 @@ async fn build_and_write_bundle( snapshot_generation: request.snapshot_generation, backup_kek: kek.descriptor(), artifacts, - local_kdf: Some(local_kdf_descriptor(snapshot)), + local_kdf: Some(local_kdf_descriptor(snapshot, master_key_verifier)), key_versions: None, capability_discovery: None, completeness: CompletenessState::InProgress, @@ -448,13 +479,25 @@ async fn encrypt_and_write_artifact( ))); } - Ok(ArtifactDescriptor { + let descriptor = ArtifactDescriptor { kind, path: artifact_path.to_string(), len: payload.len() as u64, aead_algorithm: AeadAlgorithm::Aes256Gcm, encrypted_digest: digest, - }) + }; + + // Pre-seal decryption probe: digest equality only proves the ciphertext + // landed intact; this proves the stored artifact actually opens under the + // backup KEK and AAD binding before the manifest may reference it. + let reopened = decrypt_artifact_payload(&request.backup_id, request.snapshot_generation, &descriptor, kek, &written)?; + if reopened.as_slice() != plaintext { + return Err(KmsError::internal_error(format!( + "bundle artifact '{artifact_path}' failed the pre-seal decryption probe" + ))); + } + + Ok(descriptor) } /// AAD binding an artifact to its bundle identity and path. A JSON tuple @@ -464,6 +507,29 @@ fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) .expect("AAD tuple of strings and integers always serializes") } +/// Compute the opaque one-way master-key verifier recorded in the manifest: +/// `<scheme-prefix>` + `hex(SHA-256(json(context, backup_id) || derived_key))`. +/// +/// The derivation follows the directory's KDF state: Argon2id over the +/// persistent salt when one exists, the legacy SHA-256 derivation otherwise. +/// The verifier is not an offline-guessing oracle: computing a candidate +/// requires the salt, which exists only inside the KEK-sealed bundle, and a +/// party holding the KEK already has the strictly stronger oracle of the +/// artifact AEAD tags — while every guess still pays the full Argon2id cost. +/// Binding `backup_id` prevents cross-bundle fingerprint correlation. +pub(crate) fn compute_master_key_verifier(master_key: &str, salt: Option<&[u8]>, backup_id: &str) -> Result<String> { + let framing = serde_json::to_vec(&(MASTER_KEY_VERIFIER_CONTEXT, backup_id)) + .expect("verifier framing tuple of strings always serializes"); + let (prefix, derived) = match salt { + Some(salt) => (MASTER_KEY_VERIFIER_ARGON2ID_PREFIX, LocalKmsClient::derive_master_key(master_key, salt)?), + None => (MASTER_KEY_VERIFIER_LEGACY_PREFIX, LocalKmsClient::derive_legacy_master_key(master_key)?), + }; + let mut hasher = Sha256::new(); + hasher.update(&framing); + hasher.update(derived.as_slice()); + Ok(format!("{prefix}{}", hex::encode(hasher.finalize()))) +} + /// The bundle-level protection label is the weakest state observed across /// records: any plaintext-dev-only record marks the whole bundle, then any /// legacy-unspecified marker (unknown until read), and only a uniformly @@ -484,7 +550,7 @@ fn weakest_observed_protection(records: &[CollectedRecord]) -> AtRestProtection } } -fn local_kdf_descriptor(snapshot: &CollectedSnapshot) -> LocalKdfDescriptor { +fn local_kdf_descriptor(snapshot: &CollectedSnapshot, master_key_verifier: Option<String>) -> LocalKdfDescriptor { let mut modes = Vec::new(); for (marker, mode) in [ (StoredKeyProtection::EncryptedMasterKey, AtRestProtection::EncryptedMasterKey), @@ -508,9 +574,7 @@ fn local_kdf_descriptor(snapshot: &CollectedSnapshot) -> LocalKdfDescriptor { LocalKdfDescriptor { derivation, protection_modes: modes, - // The verifier shape is left to the restore change; the schema keeps - // it optional so bundles without one stay valid. - master_key_verifier: None, + master_key_verifier, } } @@ -543,8 +607,9 @@ async fn write_new_file(path: &Path, bytes: &[u8]) -> Result<()> { /// Fsync a directory so freshly created bundle entries survive power loss. /// No-op on non-Unix platforms where directories cannot be opened for -/// syncing (mirrors the local backend's durable commit helper). -async fn fsync_dir(path: &Path) -> Result<()> { +/// syncing (mirrors the local backend's durable commit helper). Shared with +/// the restore module for the same durability points on the target side. +pub(crate) async fn fsync_dir(path: &Path) -> Result<()> { #[cfg(unix)] { let path = path.to_path_buf(); @@ -725,6 +790,37 @@ mod tests { assert_eq!(decrypted.as_slice(), source_record.as_slice()); } + #[tokio::test] + async fn export_records_a_master_key_verifier_matching_recomputation() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("verified", "AES_256", None).await.expect("create key"); + + let bundle = TempDir::new().expect("bundle dir"); + let manifest = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect("export should succeed"); + + let verifier = manifest + .local_kdf + .as_ref() + .expect("local kdf descriptor") + .master_key_verifier + .as_deref() + .expect("encrypted bundles must record a verifier"); + assert!(verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX), "got {verifier:?}"); + + // The verifier is a pure function of (master key, salt, backup id): + // the restore side recomputes it from the operator-supplied key and + // the bundled salt. + let salt = fs::read(client.master_key_salt_file()).await.expect("salt"); + let recomputed = compute_master_key_verifier("test-master-key", Some(&salt), "backup-0001").expect("recompute"); + assert_eq!(recomputed, verifier); + let wrong_key = compute_master_key_verifier("wrong-master-key", Some(&salt), "backup-0001").expect("recompute"); + assert_ne!(wrong_key, verifier, "a different master key must change the verifier"); + let other_bundle = compute_master_key_verifier("test-master-key", Some(&salt), "backup-0002").expect("recompute"); + assert_ne!(other_bundle, verifier, "verifiers must be bundle-bound"); + } + #[tokio::test] async fn export_fence_blocks_writers_until_released() { let (client, _key_dir) = encrypted_client().await; diff --git a/crates/kms/src/backup/local_restore.rs b/crates/kms/src/backup/local_restore.rs new file mode 100644 index 000000000..c496c16bc --- /dev/null +++ b/crates/kms/src/backup/local_restore.rs @@ -0,0 +1,1970 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Local backend restore: consume a sealed backup bundle into a key directory. +//! +//! The consumer side of [`crate::backup::local_export`]. The admin API is not +//! wired here — callers construct the request and supply the backup KEK and +//! the operator-re-supplied master key explicitly. The bundle source is +//! strictly read-only throughout: restore never writes to or deletes from it. +//! +//! # Four-phase protocol +//! +//! 1. **Dry-run (zero writes)** — [`dry_run_local_restore`] decodes the full +//! bundle chain in memory, checks the KDF descriptor against this build, +//! enumerates target conflicts, and verifies the operator-supplied master +//! key against the manifest's one-way verifier. It writes nothing. +//! 2. **Staging** — decrypted artifacts are committed durably into the +//! `.restore-staging/` subdirectory of the target. The leading dot and the +//! directory shape keep it invisible to the backend's key scan and its +//! orphan-temp matcher, so startup recovery can never delete staged state. +//! Every record is decryption-probed with the derived master key before +//! and after staging. +//! 3. **Commit marker + cutover** — the durably published +//! `.restore-commit.json` marker is the single commit point. Cutover +//! publishes each staged file into the target top level via the no-clobber +//! [`durable_file::link_durably`] primitive (salt first, keys after), then +//! durably removes the marker and drops the staging directory. +//! 4. **Crash re-entry** — before the marker, the target top level is +//! untouched and a re-run starts over; with the marker published, backend +//! startup fails closed and re-running the restore with the same bundle +//! rolls the cutover forward, while [`abort_local_restore`] rolls it back. +//! Every interruption converges to the complete old or complete new state. +//! +//! # Deliberately out of scope +//! +//! The `explicit-remap` conflict policy is not implemented: remapping stable +//! key ids requires proving that object envelopes migrate in lockstep, and a +//! bulk rekey is a non-goal of this change. Local has no rotation history +//! (backlog#1565), so per-key version and deletion-state regression cannot +//! become write operations under the empty-target policy; the snapshot +//! generation check freezes the injected observe-and-reject-strictly-lower +//! semantics for the admin layer. + +use crate::backends::local::{ + LOCAL_KMS_MASTER_KEY_SALT_FILE, LOCAL_KMS_MASTER_KEY_SALT_LEN, LOCAL_RESTORE_COMMIT_MARKER_FILE, LocalKmsClient, + StoredKeyProtection, durable_file, is_orphan_commit_temp_name, validate_key_id, +}; +use crate::backup::capability::AtRestProtection; +use crate::backup::dry_run::{ + ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, +}; +use crate::backup::error::BackupError; +use crate::backup::local_export::{ + BackupKek, MASTER_KEY_VERIFIER_ARGON2ID_PREFIX, MASTER_KEY_VERIFIER_LEGACY_PREFIX, compute_master_key_verifier, + decrypt_bundle_artifact, fsync_dir, read_local_bundle_manifest, +}; +use crate::backup::manifest::{ArtifactKind, BackupManifest, ContentDigest, LocalKeyDerivation}; +use crate::error::{KmsError, Result}; +use aes_gcm::{ + Aes256Gcm, Nonce, + aead::{Aead, KeyInit}, +}; +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use tokio::fs; +use tracing::warn; +use zeroize::Zeroizing; + +/// Name of the staging subdirectory inside the restore target. +const RESTORE_STAGING_DIR: &str = ".restore-staging"; +/// Format version of the cutover commit marker. +const RESTORE_MARKER_FORMAT_VERSION: u32 = 1; +/// Sentinel key id for conflicts that concern the whole target rather than +/// one key (the snapshot generation). +const WHOLE_TARGET_CONFLICT_ID: &str = "*"; + +/// How restore resolves conflicts with existing target state. +/// +/// Silent overwrite or merge is never an option; `explicit-remap` is +/// deliberately not offered (see the module docs). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RestoreConflictPolicy { + /// Never write to the target. The default: any actual restore attempt is + /// refused, so mutating a key directory always requires the explicit + /// opt-in below. Dry-runs are unaffected. + #[default] + Fail, + /// Restore only into an empty target. A target counts as non-empty when + /// it holds any key file, a KDF salt (an orphan salt would poison every + /// key created later), a marker of a different restore, or any other + /// unexpected entry; only leftovers of an interrupted run of this same + /// restore are tolerated. + RestoreIntoEmptyTarget, +} + +/// Parameters of one restore (or dry-run) attempt. +/// +/// `observed_generation` is injected by the caller, mirroring the export +/// request's `snapshot_generation`: the admin layer owns where the target's +/// highest observed generation is tracked; this module freezes only the +/// strictly-lower-is-rejected semantics (equal generations stay allowed so +/// drills can be repeated). +pub struct LocalRestoreRequest { + /// Bundle directory produced by the Local export. Read-only. + pub bundle_dir: PathBuf, + /// Key directory to restore into. + pub target_key_dir: PathBuf, + /// Deployment identity of the restore target; must match the manifest. + pub target_deployment_identity: String, + /// Highest snapshot generation the target has already observed, when the + /// caller tracks one. + pub observed_generation: Option<u64>, + /// Operator-re-supplied master key string. The master key is outside the + /// backup domain, so it must arrive out of band; required whenever the + /// bundle carries encrypted-at-rest records. + pub master_key: Option<String>, + /// Conflict policy; see [`RestoreConflictPolicy`]. + pub conflict_policy: RestoreConflictPolicy, + /// Unix permission bits for restored files, mirroring + /// `LocalConfig::file_permissions`. + pub file_permissions: Option<u32>, +} + +impl std::fmt::Debug for LocalRestoreRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalRestoreRequest") + .field("bundle_dir", &self.bundle_dir) + .field("target_key_dir", &self.target_key_dir) + .field("target_deployment_identity", &self.target_deployment_identity) + .field("observed_generation", &self.observed_generation) + .field("master_key", &self.master_key.as_ref().map(|_| "<redacted>")) + .field("conflict_policy", &self.conflict_policy) + .field("file_permissions", &self.file_permissions) + .finish() + } +} + +impl LocalRestoreRequest { + fn validate(&self) -> Result<()> { + if self.target_deployment_identity.is_empty() { + return Err(KmsError::validation_error("restore target_deployment_identity must not be empty")); + } + Ok(()) + } +} + +/// Serializable outcome of a completed restore. Identifiers only; never key +/// material or secrets. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalRestoreReport { + /// Identifier of the restored bundle. + pub backup_id: String, + /// Snapshot generation the target now holds. + pub snapshot_generation: u64, + /// Stable key ids restored into the target. + pub restored_key_ids: Vec<String>, + /// Whether the master-key KDF salt was restored. + pub salt_restored: bool, + /// Whether this run rolled forward a previously interrupted cutover. + pub resumed: bool, +} + +/// The cutover commit marker: its durable publication is the single commit +/// point of a restore. It names the exact bundle and the exact top-level +/// files the cutover publishes, so re-entry can verify it is completing the +/// same restore and an abort knows precisely what to take back. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RestoreCommitMarker { + format_version: u32, + backup_id: String, + manifest_digest: ContentDigest, + /// Final top-level file names in link order (salt first, then keys). + files: Vec<String>, +} + +impl RestoreCommitMarker { + fn for_bundle(decoded: &DecodedBundle) -> Self { + let mut files = Vec::with_capacity(decoded.records.len() + 1); + if decoded.salt.is_some() { + files.push(LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string()); + } + files.extend(decoded.records.iter().map(|record| record.file_name.clone())); + Self { + format_version: RESTORE_MARKER_FORMAT_VERSION, + backup_id: decoded.manifest.backup_id.clone(), + manifest_digest: decoded.manifest.manifest_digest.clone(), + files, + } + } + + /// Strict decode: an unreadable or out-of-contract marker fails closed — + /// abort and roll-forward both refuse to guess what a foreign or damaged + /// marker was committing. + fn decode(bytes: &[u8]) -> Result<Self> { + let marker: Self = serde_json::from_slice(bytes) + .map_err(|error| BackupError::corrupted(format!("restore commit marker does not decode: {error}")))?; + if marker.format_version != RESTORE_MARKER_FORMAT_VERSION { + return Err(BackupError::corrupted(format!( + "restore commit marker has unknown format version {}", + marker.format_version + )) + .into()); + } + if marker.files.is_empty() { + return Err(BackupError::corrupted("restore commit marker lists no files").into()); + } + for name in &marker.files { + if name == LOCAL_KMS_MASTER_KEY_SALT_FILE { + continue; + } + match name.strip_suffix(".key") { + Some(stem) if validate_key_id(stem).is_ok() => {} + _ => { + return Err(BackupError::corrupted(format!( + "restore commit marker lists a file name outside the restore contract: {name:?}" + )) + .into()); + } + } + } + Ok(marker) + } + + fn matches_bundle(&self, manifest: &BackupManifest) -> bool { + self.backup_id == manifest.backup_id && self.manifest_digest == manifest.manifest_digest + } +} + +/// One key record decoded out of the bundle, fully re-verified. +struct DecodedRecord { + key_id: String, + /// Final on-disk file name (`<key_id>.key`). + file_name: String, + protection: StoredKeyProtection, + /// Whether the record's material is AEAD-protected under the master key + /// (explicit marker, or a legacy record with a nonce). + effectively_encrypted: bool, + /// Base64-decoded `encrypted_key_material` field. + material: Zeroizing<Vec<u8>>, + nonce: Vec<u8>, + /// Raw record bytes exactly as exported; what staging writes. + plaintext: Zeroizing<Vec<u8>>, +} + +/// The fully decoded and cross-checked bundle content. +struct DecodedBundle { + manifest: BackupManifest, + records: Vec<DecodedRecord>, + salt: Option<Zeroizing<Vec<u8>>>, +} + +impl DecodedBundle { + fn any_record_encrypted(&self) -> bool { + self.records.iter().any(|record| record.effectively_encrypted) + } +} + +/// Minimal projection of a stored key record, mirroring the export-side +/// probe. Unknown fields are ignored on purpose: the record is restored +/// byte-identical, so restore must not constrain its schema. +#[derive(Deserialize)] +struct RestoredRecordProbe { + key_id: String, + #[serde(default)] + at_rest_protection: StoredKeyProtection, + encrypted_key_material: String, + #[serde(default)] + nonce: Vec<u8>, +} + +/// Zero-write restore preflight: evaluate the bundle against the target and +/// report every blocker, conflict, and external dependency mismatch. +/// +/// The bundle is decoded fully in memory (including AEAD verification of +/// every artifact); the target directory is only read. Environment failures +/// (unreadable directories) surface as errors, verdicts about the bundle or +/// the target surface inside the report. +pub async fn dry_run_local_restore(kek: &BackupKek, request: &LocalRestoreRequest) -> Result<RestoreDryRunReport> { + request.validate()?; + let mut report = RestoreDryRunReport { + backup_id: String::new(), + target_deployment_identity: request.target_deployment_identity.clone(), + blockers: Vec::new(), + conflicts: Vec::new(), + external_mismatches: Vec::new(), + }; + + let decoded = match decode_bundle(&request.bundle_dir, kek).await { + Ok(decoded) => decoded, + Err(error) => match blocker_for(&error) { + Some(blocker) => { + report.blockers.push(blocker); + return Ok(report); + } + None => return Err(error), + }, + }; + report.backup_id = decoded.manifest.backup_id.clone(); + + if let Some(detail) = kdf_drift_detail(&decoded.manifest) { + report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::UnsupportedBackend, + detail, + }); + } + if decoded.manifest.deployment_identity != request.target_deployment_identity { + report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::DeploymentMismatch, + detail: format!( + "bundle was produced by deployment '{}', restore target is '{}'", + decoded.manifest.deployment_identity, request.target_deployment_identity + ), + }); + } + if let Some(observed) = request.observed_generation + && decoded.manifest.snapshot_generation < observed + { + report.conflicts.push(RestoreConflict { + key_id: WHOLE_TARGET_CONFLICT_ID.to_string(), + kind: RestoreConflictKind::GenerationRegression, + detail: format!( + "target observed snapshot generation {observed}, bundle carries {}; equal generations stay allowed for repeated drills", + decoded.manifest.snapshot_generation + ), + }); + } + + match check_master_key(&decoded, request.master_key.as_deref())? { + MasterKeyCheck::Ok => {} + MasterKeyCheck::MissingPassphrase(detail) => report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::ExternalDependencyUnavailable, + detail, + }), + MasterKeyCheck::UnknownVerifierScheme(detail) => report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::BundleCorrupted, + detail, + }), + MasterKeyCheck::Mismatch { expected, observed } => report.external_mismatches.push(ExternalDependencyMismatch { + dependency: "local KMS master key (operator-supplied)".to_string(), + expected, + observed, + }), + } + + report + .conflicts + .extend(enumerate_target_conflicts(&request.target_key_dir, &decoded.manifest).await?); + Ok(report) +} + +/// Restore a sealed Local bundle into the target key directory. +/// +/// Fail-fast counterpart of the dry-run: the first violated precondition +/// aborts with its typed error before the target top level is touched. Only +/// [`RestoreConflictPolicy::RestoreIntoEmptyTarget`] ever writes. +pub async fn restore_local_backup(kek: &BackupKek, request: &LocalRestoreRequest) -> Result<LocalRestoreReport> { + request.validate()?; + if request.conflict_policy == RestoreConflictPolicy::Fail { + return Err(KmsError::invalid_operation( + "restore conflict policy 'fail' never writes to the target; review the dry-run report and \ + re-run with the 'restore-into-empty-target' policy to proceed", + )); + } + + let decoded = decode_bundle(&request.bundle_dir, kek).await?; + if let Some(detail) = kdf_drift_detail(&decoded.manifest) { + return Err(KmsError::unsupported_capability("local-restore", detail)); + } + if decoded.manifest.deployment_identity != request.target_deployment_identity { + return Err(KmsError::invalid_operation(format!( + "bundle was produced by deployment '{}' but the restore target is '{}'", + decoded.manifest.deployment_identity, request.target_deployment_identity + ))); + } + if let Some(observed) = request.observed_generation + && decoded.manifest.snapshot_generation < observed + { + return Err(KmsError::invalid_operation(format!( + "restore would regress the snapshot generation: target observed {observed}, bundle carries {}", + decoded.manifest.snapshot_generation + ))); + } + match check_master_key(&decoded, request.master_key.as_deref())? { + MasterKeyCheck::Ok => {} + MasterKeyCheck::MissingPassphrase(detail) => return Err(KmsError::invalid_operation(detail)), + MasterKeyCheck::UnknownVerifierScheme(detail) => return Err(BackupError::corrupted(detail).into()), + MasterKeyCheck::Mismatch { .. } => { + return Err(KmsError::validation_error( + "operator-supplied master key does not match the bundle's master-key verifier", + )); + } + } + + // Decryption-probe every record in memory before anything is written: a + // wrong master key on a verifier-less legacy bundle must fail here, with + // the target untouched. + let ciphers = derive_ciphers(request.master_key.as_deref(), decoded.salt.as_deref().map(Vec::as_slice))?; + for record in &decoded.records { + probe_record_decryption(record, &ciphers)?; + } + + let mode = inspect_target_for_restore(&request.target_key_dir, &decoded.manifest).await?; + let expected_marker = RestoreCommitMarker::for_bundle(&decoded); + let resumed = match &mode { + TargetMode::Fresh { .. } => false, + TargetMode::RollForward(published) => { + if *published != expected_marker { + return Err(BackupError::corrupted( + "published restore marker names the same bundle but a different file set; \ + abort the interrupted restore before retrying", + ) + .into()); + } + true + } + }; + + // Phase B: staging. + let staging = request.target_key_dir.join(RESTORE_STAGING_DIR); + if let TargetMode::Fresh { stale_entries } = &mode { + // A fresh run owns no prior state: drop staging leftovers of an + // earlier interrupted attempt and any orphan commit temps, then start + // over from the bundle. + if fs::try_exists(&staging).await? { + remove_dir_all_durably(&request.target_key_dir, &staging).await?; + } + for stale in stale_entries { + durable_file::remove_durably(request.target_key_dir.join(stale)).await?; + } + } + fs::create_dir_all(&staging).await?; + if let Some(salt) = &decoded.salt { + stage_file(&staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, salt, request.file_permissions).await?; + } + for record in &decoded.records { + stage_file(&staging, &record.file_name, &record.plaintext, request.file_permissions).await?; + } + // Make the staging directory entry itself durable before the commit + // point: the marker must never survive a crash that its staged files did + // not. + fsync_dir(&request.target_key_dir).await?; + probe_staged_bundle(&staging, &decoded, &ciphers).await?; + + // Phase C: commit point, then cutover. + if !resumed { + publish_marker(&request.target_key_dir, &expected_marker, request.file_permissions).await?; + } + cutover_and_finish(&request.target_key_dir, &staging, &expected_marker).await?; + + Ok(LocalRestoreReport { + backup_id: decoded.manifest.backup_id.clone(), + snapshot_generation: decoded.manifest.snapshot_generation, + restored_key_ids: decoded.records.iter().map(|record| record.key_id.clone()).collect(), + salt_restored: decoded.salt.is_some(), + resumed, + }) +} + +/// Explicitly abort an interrupted restore, returning the target to its +/// pre-restore (empty) state. +/// +/// With a published marker this takes back exactly the files the marker +/// names, then the marker, then the staging directory. Without a marker only +/// staging leftovers exist and are removed. Fails closed on a marker it +/// cannot strictly decode. +pub async fn abort_local_restore(target_key_dir: &Path) -> Result<()> { + let marker_path = target_key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + let staging = target_key_dir.join(RESTORE_STAGING_DIR); + match fs::read(&marker_path).await { + Ok(bytes) => { + let marker = RestoreCommitMarker::decode(&bytes)?; + for file_name in &marker.files { + let path = target_key_dir.join(file_name); + if fs::try_exists(&path).await? { + durable_file::remove_durably(path).await?; + } + } + durable_file::remove_durably(marker_path).await?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if !fs::try_exists(&staging).await? { + return Err(KmsError::invalid_operation(format!( + "no restore is in progress under {}", + target_key_dir.display() + ))); + } + } + Err(error) => return Err(error.into()), + } + if fs::try_exists(&staging).await? { + remove_dir_all_durably(target_key_dir, &staging).await?; + } + Ok(()) +} + +/// Decode and cross-verify the complete bundle in memory. +/// +/// Every artifact is read, digest-verified, and AEAD-authenticated; every key +/// record is re-parsed, identity-checked against its artifact path, and +/// checked for protection consistency with the manifest's KDF descriptor. +async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result<DecodedBundle> { + let manifest = read_local_bundle_manifest(bundle_dir).await?; + let descriptor = manifest + .local_kdf + .clone() + .ok_or_else(|| BackupError::corrupted("local bundle manifest carries no KDF descriptor"))?; + + let mut records = Vec::new(); + let mut salt: Option<Zeroizing<Vec<u8>>> = None; + for artifact in &manifest.artifacts { + match artifact.kind { + ArtifactKind::KeyMaterial => { + let plaintext = decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?; + records.push(decode_key_record(artifact.path.as_str(), plaintext, &descriptor.protection_modes)?); + } + ArtifactKind::MasterKeySalt => { + if salt.is_some() { + return Err(BackupError::corrupted("bundle carries more than one master-key salt artifact").into()); + } + let plaintext = decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?; + if plaintext.len() != LOCAL_KMS_MASTER_KEY_SALT_LEN { + return Err(BackupError::corrupted(format!( + "bundled master-key salt is {} bytes, expected {}", + plaintext.len(), + LOCAL_KMS_MASTER_KEY_SALT_LEN + )) + .into()); + } + salt = Some(plaintext); + } + // No producer emits these for a Local bundle yet; restoring a + // bundle that carries state this implementation cannot apply + // would silently drop it, so fail closed instead. + other => { + return Err(KmsError::unsupported_capability( + "local-restore", + format!("bundle carries artifact kind {other:?}, which this restore implementation cannot apply"), + )); + } + } + } + + // The salt must be exactly as coherent with the KDF descriptor as the + // export invariants promise; anything else is a broken or tampered + // bundle. + match (&descriptor.derivation, salt.is_some()) { + (LocalKeyDerivation::Argon2id { .. }, false) => { + return Err( + BackupError::corrupted("KDF descriptor declares Argon2id but the bundle carries no salt artifact").into(), + ); + } + (LocalKeyDerivation::LegacySha256, true) => { + return Err( + BackupError::corrupted("KDF descriptor declares the legacy derivation but the bundle carries a salt").into(), + ); + } + _ => {} + } + + Ok(DecodedBundle { manifest, records, salt }) +} + +fn decode_key_record( + artifact_path: &str, + plaintext: Zeroizing<Vec<u8>>, + allowed_modes: &[AtRestProtection], +) -> Result<DecodedRecord> { + let stem = Path::new(artifact_path) + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_suffix(".key.enc")) + .ok_or_else(|| { + BackupError::corrupted(format!( + "key material artifact path {artifact_path:?} does not follow the '<key_id>.key.enc' layout" + )) + })? + .to_string(); + validate_key_id(&stem)?; + + let probe: RestoredRecordProbe = serde_json::from_slice(&plaintext) + .map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' does not deserialize: {error}")))?; + if probe.key_id != stem { + return Err(BackupError::corrupted(format!( + "bundled key record identity mismatch: artifact names {stem:?}, record names {:?}", + probe.key_id + )) + .into()); + } + if probe.encrypted_key_material.is_empty() { + return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into()); + } + let material = + Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| { + BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}")) + })?); + if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) { + return Err(BackupError::corrupted(format!( + "bundled key record '{stem}' declares protection {:?}, absent from the manifest's KDF descriptor", + probe.at_rest_protection + )) + .into()); + } + + let effectively_encrypted = match probe.at_rest_protection { + StoredKeyProtection::EncryptedMasterKey => true, + StoredKeyProtection::PlaintextDevOnly => false, + StoredKeyProtection::LegacyUnspecified => !probe.nonce.is_empty(), + }; + if effectively_encrypted && probe.nonce.len() != 12 { + return Err(BackupError::corrupted(format!( + "bundled key record '{stem}' has an invalid nonce length ({} bytes, expected 12)", + probe.nonce.len() + )) + .into()); + } + + Ok(DecodedRecord { + file_name: format!("{stem}.key"), + key_id: stem, + protection: probe.at_rest_protection, + effectively_encrypted, + material, + nonce: probe.nonce, + plaintext, + }) +} + +fn protection_mode(protection: StoredKeyProtection) -> AtRestProtection { + match protection { + StoredKeyProtection::EncryptedMasterKey => AtRestProtection::EncryptedMasterKey, + StoredKeyProtection::PlaintextDevOnly => AtRestProtection::PlaintextDevOnly, + StoredKeyProtection::LegacyUnspecified => AtRestProtection::LegacyUnspecified, + } +} + +/// Detail message when the bundle's recorded KDF differs from what this build +/// compiles in; restoring such a bundle would produce keys this build cannot +/// decrypt. +fn kdf_drift_detail(manifest: &BackupManifest) -> Option<String> { + match &manifest.local_kdf.as_ref()?.derivation { + derivation @ LocalKeyDerivation::Argon2id { .. } => { + let current = LocalKeyDerivation::current_argon2id(); + (*derivation != current).then(|| { + format!("bundle KDF descriptor {derivation:?} does not match this build's compiled-in derivation {current:?}") + }) + } + LocalKeyDerivation::LegacySha256 => None, + } +} + +enum MasterKeyCheck { + Ok, + MissingPassphrase(String), + UnknownVerifierScheme(String), + Mismatch { expected: String, observed: String }, +} + +/// Verify the operator-supplied master key against the manifest's one-way +/// verifier, and require one whenever the bundle carries encrypted records. +/// +/// Bundles without a verifier (produced before the verifier landed) fall +/// back to the staging-phase decryption probe; unknown verifier schemes fail +/// closed. +fn check_master_key(decoded: &DecodedBundle, master_key: Option<&str>) -> Result<MasterKeyCheck> { + let Some(master_key) = master_key else { + if decoded.any_record_encrypted() { + return Ok(MasterKeyCheck::MissingPassphrase( + "bundle carries encrypted-at-rest key records; the operator must re-supply the master key before restore" + .to_string(), + )); + } + return Ok(MasterKeyCheck::Ok); + }; + + let verifier = decoded + .manifest + .local_kdf + .as_ref() + .and_then(|descriptor| descriptor.master_key_verifier.as_deref()); + let Some(verifier) = verifier else { + return Ok(MasterKeyCheck::Ok); + }; + + let observed = if verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX) { + let Some(salt) = decoded.salt.as_deref() else { + return Err( + BackupError::corrupted("manifest carries an Argon2id master-key verifier but the bundle has no salt").into(), + ); + }; + compute_master_key_verifier(master_key, Some(salt), &decoded.manifest.backup_id)? + } else if verifier.starts_with(MASTER_KEY_VERIFIER_LEGACY_PREFIX) { + compute_master_key_verifier(master_key, None, &decoded.manifest.backup_id)? + } else { + return Ok(MasterKeyCheck::UnknownVerifierScheme( + "manifest carries a master-key verifier with an unknown scheme prefix; failing closed".to_string(), + )); + }; + + if observed != verifier { + return Ok(MasterKeyCheck::Mismatch { + expected: verifier.to_string(), + observed, + }); + } + Ok(MasterKeyCheck::Ok) +} + +/// AEAD ciphers derived from the operator-supplied master key, mirroring the +/// backend's current-plus-legacy pair. +struct DerivedCiphers { + current: Option<Aes256Gcm>, + legacy: Option<Aes256Gcm>, +} + +fn derive_ciphers(master_key: Option<&str>, salt: Option<&[u8]>) -> Result<DerivedCiphers> { + let Some(master_key) = master_key else { + return Ok(DerivedCiphers { + current: None, + legacy: None, + }); + }; + let legacy = Aes256Gcm::new(&LocalKmsClient::derive_legacy_master_key(master_key)?); + let current = match salt { + Some(salt) => Aes256Gcm::new(&LocalKmsClient::derive_master_key(master_key, salt)?), + None => legacy.clone(), + }; + Ok(DerivedCiphers { + current: Some(current), + legacy: Some(legacy), + }) +} + +/// Decryption probe for one record, mirroring the backend's read path: the +/// current cipher first, with the legacy fallback only for legacy-unmarked +/// records. Local has no rotation history, so probing every record is the +/// full-coverage probe. +fn probe_record_decryption(record: &DecodedRecord, ciphers: &DerivedCiphers) -> Result<()> { + if !record.effectively_encrypted { + return Ok(()); + } + let Some(current) = ciphers.current.as_ref() else { + return Err(KmsError::invalid_operation(format!( + "bundled key record '{}' is encrypted at rest and requires the operator-supplied master key", + record.key_id + ))); + }; + let mut nonce = [0u8; 12]; + nonce.copy_from_slice(&record.nonce); + let nonce = Nonce::from(nonce); + match current.decrypt(&nonce, record.material.as_slice()) { + Ok(material) => { + drop(Zeroizing::new(material)); + Ok(()) + } + Err(_) if record.protection == StoredKeyProtection::LegacyUnspecified => { + let legacy = ciphers + .legacy + .as_ref() + .expect("legacy cipher exists whenever the current cipher does"); + let material = legacy + .decrypt(&nonce, record.material.as_slice()) + .map_err(|_| KmsError::material_authentication_failed(&record.key_id))?; + drop(Zeroizing::new(material)); + Ok(()) + } + Err(_) => Err(KmsError::material_authentication_failed(&record.key_id)), + } +} + +/// Re-run the decryption probe against the bytes that actually landed in +/// staging, so cutover only ever publishes state proven readable from disk. +async fn probe_staged_bundle(staging: &Path, decoded: &DecodedBundle, ciphers: &DerivedCiphers) -> Result<()> { + if let Some(salt) = &decoded.salt { + let staged_salt = fs::read(staging.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)).await?; + if staged_salt.as_slice() != salt.as_slice() { + return Err(BackupError::corrupted("staged master-key salt does not match the bundle").into()); + } + } + for record in &decoded.records { + let staged = Zeroizing::new(fs::read(staging.join(&record.file_name)).await?); + if staged.as_slice() != record.plaintext.as_slice() { + return Err( + BackupError::corrupted(format!("staged key record '{}' does not match the bundle", record.key_id)).into(), + ); + } + let staged_record = decode_key_record( + &format!("staged/{}.enc", record.file_name), + staged, + &decoded + .manifest + .local_kdf + .as_ref() + .expect("decoded bundle always carries a KDF descriptor") + .protection_modes, + )?; + probe_record_decryption(&staged_record, ciphers)?; + } + Ok(()) +} + +enum TargetMode { + /// The target is empty apart from removable leftovers of an interrupted + /// earlier attempt (staging, orphan commit temps). + Fresh { stale_entries: Vec<String> }, + /// A published marker of this same bundle: complete its cutover. + RollForward(RestoreCommitMarker), +} + +/// Fail-fast target inspection for an actual restore. +async fn inspect_target_for_restore(target: &Path, manifest: &BackupManifest) -> Result<TargetMode> { + if !fs::try_exists(target).await? { + return Ok(TargetMode::Fresh { + stale_entries: Vec::new(), + }); + } + + let marker = read_marker(target).await?; + if let Some(marker) = marker { + if !marker.matches_bundle(manifest) { + return Err(KmsError::invalid_operation(format!( + "target key directory holds a restore marker for a different bundle ('{}'); \ + roll that restore forward with its own bundle or abort it explicitly", + marker.backup_id + ))); + } + // Everything at the top level must belong to this restore: the + // marker's files (partially linked) and our own working entries. + let allowed: Vec<&str> = marker.files.iter().map(String::as_str).collect(); + for entry in list_target_entries(target).await? { + if !allowed.contains(&entry.as_str()) && !is_orphan_commit_temp_name(&entry) { + return Err(KmsError::invalid_operation(format!( + "target key directory holds an entry outside the interrupted restore: {entry:?}; \ + clean it up manually before resuming" + ))); + } + } + return Ok(TargetMode::RollForward(marker)); + } + + let mut stale_entries = Vec::new(); + for entry in list_target_entries(target).await? { + if is_orphan_commit_temp_name(&entry) { + // Unpublished commit temps are never authoritative; a re-run may + // remove them exactly as startup recovery would. + stale_entries.push(entry); + continue; + } + let detail = if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE { + "an orphan KDF salt would poison every key created after the restore; clean it up manually" + } else { + "restore only ever writes into an empty target" + }; + return Err(KmsError::invalid_operation(format!( + "target key directory is not empty (found {entry:?}); {detail}" + ))); + } + Ok(TargetMode::Fresh { stale_entries }) +} + +/// Top-level entries of the target, excluding the staging directory and the +/// commit marker (both handled separately). +async fn list_target_entries(target: &Path) -> Result<Vec<String>> { + let mut names = Vec::new(); + let mut entries = fs::read_dir(target).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry + .file_name() + .to_str() + .ok_or_else(|| KmsError::invalid_operation("target key directory holds a non-UTF-8 entry; clean it up manually"))? + .to_string(); + if name == RESTORE_STAGING_DIR || name == LOCAL_RESTORE_COMMIT_MARKER_FILE { + continue; + } + names.push(name); + } + names.sort(); + Ok(names) +} + +async fn read_marker(target: &Path) -> Result<Option<RestoreCommitMarker>> { + match fs::read(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)).await { + Ok(bytes) => Ok(Some(RestoreCommitMarker::decode(&bytes)?)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } +} + +/// Conflict enumeration for the dry-run: every target entry that would block +/// the restore, without failing on the first one. +async fn enumerate_target_conflicts(target: &Path, manifest: &BackupManifest) -> Result<Vec<RestoreConflict>> { + if !fs::try_exists(target).await? { + return Ok(Vec::new()); + } + let mut conflicts = Vec::new(); + + let marker = match read_marker(target).await { + Ok(marker) => marker, + // A marker that does not strictly decode blocks exactly like a + // foreign one; the dry-run reports it instead of failing. + Err(error) => { + conflicts.push(RestoreConflict { + key_id: LOCAL_RESTORE_COMMIT_MARKER_FILE.to_string(), + kind: RestoreConflictKind::KeyAlreadyExists, + detail: format!("target holds an undecodable restore marker: {error}"), + }); + None + } + }; + let own_files: Vec<String> = match &marker { + Some(marker) if marker.matches_bundle(manifest) => marker.files.clone(), + Some(marker) => { + conflicts.push(RestoreConflict { + key_id: LOCAL_RESTORE_COMMIT_MARKER_FILE.to_string(), + kind: RestoreConflictKind::KeyAlreadyExists, + detail: format!("target holds a restore marker for a different bundle ('{}')", marker.backup_id), + }); + Vec::new() + } + None => Vec::new(), + }; + + for entry in list_target_entries(target).await? { + if own_files.contains(&entry) || is_orphan_commit_temp_name(&entry) { + continue; + } + let (key_id, detail) = if let Some(stem) = entry.strip_suffix(".key") { + (stem.to_string(), format!("target already has key file {entry:?}")) + } else if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE { + ( + entry.clone(), + "target holds an orphan master-key KDF salt; it would poison every key created after the restore".to_string(), + ) + } else { + (entry.clone(), format!("target holds an unexpected entry {entry:?}")) + }; + conflicts.push(RestoreConflict { + key_id, + kind: RestoreConflictKind::KeyAlreadyExists, + detail, + }); + } + Ok(conflicts) +} + +/// Durably commit one staged file. Idempotent across roll-forward re-entry: +/// an already staged file is accepted only byte-identical. +async fn stage_file(staging: &Path, file_name: &str, content: &[u8], permissions: Option<u32>) -> Result<()> { + let final_path = staging.join(file_name); + match fs::read(&final_path).await { + Ok(existing) => { + let existing = Zeroizing::new(existing); + if existing.as_slice() == content { + return Ok(()); + } + return Err(BackupError::corrupted(format!( + "staging already holds '{file_name}' with different content; abort the interrupted restore before retrying" + )) + .into()); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let temp_path = staging.join(format!("{file_name}.tmp-{}", uuid::Uuid::new_v4())); + durable_file::commit(temp_path, final_path, content.to_vec(), permissions, durable_file::Publish::NoClobber) + .await + .map_err(KmsError::from) +} + +/// Durably publish the commit marker. This is the restore's single commit +/// point: before it, the target top level is untouched; after it, the only +/// ways out are roll-forward or explicit abort. +async fn publish_marker(target: &Path, marker: &RestoreCommitMarker, permissions: Option<u32>) -> Result<()> { + let content = serde_json::to_vec_pretty(marker) + .map_err(|error| KmsError::internal_error(format!("restore commit marker serialization failed: {error}")))?; + let final_path = target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + let temp_path = target.join(format!("{LOCAL_RESTORE_COMMIT_MARKER_FILE}.tmp-{}", uuid::Uuid::new_v4())); + durable_file::commit(temp_path, final_path, content, permissions, durable_file::Publish::NoClobber) + .await + .map_err(KmsError::from) +} + +/// The atomic cutover: publish every staged file under its final name (salt +/// first, keys after — the marker froze that order), durably remove the +/// marker, then drop the staging directory. +async fn cutover_and_finish(target: &Path, staging: &Path, marker: &RestoreCommitMarker) -> Result<()> { + for file_name in &marker.files { + durable_file::link_durably(staging.join(file_name), target.join(file_name)) + .await + .map_err(|error| KmsError::io_error(format!("restore cutover failed publishing '{file_name}': {error}")))?; + } + durable_file::remove_durably(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)) + .await + .map_err(|error| KmsError::io_error(format!("restore cutover failed removing the commit marker: {error}")))?; + // The restore is complete once the marker is gone; a leftover staging + // directory is inert (its entries are hard links of the published files), + // so its removal is cleanup, not protocol. + if let Err(error) = remove_dir_all_durably(target, staging).await { + warn!(%error, "restored Local KMS key directory kept an inert staging directory"); + } + Ok(()) +} + +async fn remove_dir_all_durably(parent: &Path, dir: &Path) -> Result<()> { + let dir = dir.to_path_buf(); + tokio::task::spawn_blocking(move || std::fs::remove_dir_all(&dir)) + .await + .map_err(|error| KmsError::io_error(error.to_string()))??; + fsync_dir(parent).await +} + +fn blocker_for(error: &KmsError) -> Option<RestoreBlocker> { + match error { + KmsError::Backup(inner) => Some(RestoreBlocker::from(inner)), + KmsError::UnsupportedCapability { .. } => Some(RestoreBlocker { + code: RestoreBlockerCode::UnsupportedBackend, + detail: error.to_string(), + }), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backup::local_export::{LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, export_local_backup}; + use crate::config::LocalConfig; + use sha2::{Digest, Sha256}; + use std::time::SystemTime; + use tempfile::TempDir; + + const TEST_MASTER_KEY: &str = "test-master-key"; + const DEPLOYMENT: &str = "deployment-test"; + const BACKUP_ID: &str = "backup-0001"; + const SNAPSHOT_GENERATION: u64 = 7; + + fn local_config(dir: &Path, master_key: Option<&str>) -> LocalConfig { + LocalConfig { + key_dir: dir.to_path_buf(), + master_key: master_key.map(str::to_string), + file_permissions: Some(0o600), + } + } + + async fn source_with_keys(master_key: Option<&str>, keys: &[&str]) -> (LocalKmsClient, TempDir) { + let temp = TempDir::new().expect("temp dir"); + let client = LocalKmsClient::new(local_config(temp.path(), master_key)) + .await + .expect("client should initialize"); + for key in keys { + client.create_key(key, "AES_256", None).await.expect("create key"); + } + (client, temp) + } + + fn test_kek() -> BackupKek { + BackupKek::new("backup-kek-test", 1, [0x42; 32]).expect("kek") + } + + async fn export_bundle(client: &LocalKmsClient, dir: &Path) -> (PathBuf, BackupManifest) { + let destination = dir.join("bundle"); + let manifest = export_local_backup( + client, + &test_kek(), + &LocalBackupExportRequest { + backup_id: BACKUP_ID.to_string(), + deployment_identity: DEPLOYMENT.to_string(), + rustfs_version: "1.0.0-test".to_string(), + snapshot_generation: SNAPSHOT_GENERATION, + destination: destination.clone(), + }, + ) + .await + .expect("export should succeed"); + (destination, manifest) + } + + fn restore_request(bundle: &Path, target: &Path) -> LocalRestoreRequest { + LocalRestoreRequest { + bundle_dir: bundle.to_path_buf(), + target_key_dir: target.to_path_buf(), + target_deployment_identity: DEPLOYMENT.to_string(), + observed_generation: None, + master_key: Some(TEST_MASTER_KEY.to_string()), + conflict_policy: RestoreConflictPolicy::RestoreIntoEmptyTarget, + file_permissions: Some(0o600), + } + } + + /// Recursive (path, content SHA-256, mtime) snapshot; equality across an + /// operation proves it performed no persistent write at all. + fn snapshot_tree(dir: &Path) -> Vec<(String, Vec<u8>, SystemTime)> { + fn walk(root: &Path, dir: &Path, out: &mut Vec<(String, Vec<u8>, SystemTime)>) { + for entry in std::fs::read_dir(dir).expect("read dir") { + let entry = entry.expect("dir entry"); + let path = entry.path(); + if path.is_dir() { + walk(root, &path, out); + continue; + } + let relative = path.strip_prefix(root).expect("under root").to_string_lossy().into_owned(); + let content = std::fs::read(&path).expect("read file"); + let mtime = entry.metadata().expect("metadata").modified().expect("mtime"); + out.push((relative, Sha256::digest(&content).to_vec(), mtime)); + } + } + let mut out = Vec::new(); + walk(dir, dir, &mut out); + out.sort(); + out + } + + async fn top_level_names(dir: &Path) -> Vec<String> { + let mut names = Vec::new(); + let mut entries = fs::read_dir(dir).await.expect("read dir"); + while let Some(entry) = entries.next_entry().await.expect("dir entry") { + names.push(entry.file_name().to_string_lossy().into_owned()); + } + names.sort(); + names + } + + /// Re-seal the bundle manifest after mutating it, keeping the digest + /// contract intact so only the mutated aspect is under test. + async fn reseal_manifest(bundle_dir: &Path, mutate: impl FnOnce(&mut BackupManifest)) { + let path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE); + let mut manifest = BackupManifest::decode(&fs::read(&path).await.expect("manifest bytes")).expect("manifest decodes"); + mutate(&mut manifest); + let sealed = manifest.seal().expect("re-seal"); + fs::write(&path, sealed.encode().expect("encode")) + .await + .expect("write manifest"); + } + + /// The full acceptance check for a restored directory: raw files + /// byte-identical to the source, and every record decodable through the + /// read-only export constructor with byte-identical material. + async fn assert_restored_matches_source(source: &LocalKmsClient, target: &Path, keys: &[&str]) { + for key in keys { + let source_bytes = fs::read(source.key_directory().join(format!("{key}.key"))) + .await + .expect("source record"); + let restored_bytes = fs::read(target.join(format!("{key}.key"))).await.expect("restored record"); + assert_eq!(source_bytes, restored_bytes, "record {key} must restore byte-for-byte"); + } + let source_salt = fs::read(source.master_key_salt_file()).await.expect("source salt"); + let restored_salt = fs::read(target.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)) + .await + .expect("restored salt"); + assert_eq!(source_salt, restored_salt, "salt must restore byte-for-byte"); + + let restored = LocalKmsClient::new_for_key_export(local_config(target, Some(TEST_MASTER_KEY))) + .await + .expect("restored directory must open read-only"); + for key in keys { + let source_material = source.decrypt_key_material_for_export(key).await.expect("source material"); + let restored_material = restored + .decrypt_key_material_for_export(key) + .await + .expect("restored material"); + assert_eq!( + source_material.as_ref(), + restored_material.as_ref(), + "key material for {key} must survive the restore byte-for-byte" + ); + } + } + + #[tokio::test] + async fn dry_run_is_zero_write_and_permits_clean_restore() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + fs::create_dir_all(&target).await.expect("create target"); + + let bundle_before = snapshot_tree(&bundle); + let target_before = snapshot_tree(&target); + let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("dry run should succeed"); + + assert!(report.restore_permitted(), "clean bundle and empty target must permit: {report:?}"); + assert_eq!(report.backup_id, BACKUP_ID); + assert_eq!(report.target_deployment_identity, DEPLOYMENT); + assert_eq!(snapshot_tree(&bundle), bundle_before, "dry run must not touch the bundle"); + assert_eq!(snapshot_tree(&target), target_before, "dry run must not touch the target"); + + // A missing target directory stays missing. + let missing = work.path().join("missing-target"); + let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, &missing)) + .await + .expect("dry run should succeed"); + assert!(report.restore_permitted()); + assert!(!missing.exists(), "dry run must not create the target directory"); + } + + #[tokio::test] + async fn dry_run_flags_deployment_mismatch_and_generation_regression() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.target_deployment_identity = "deployment-other".to_string(); + request.observed_generation = Some(SNAPSHOT_GENERATION + 1); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::DeploymentMismatch), + "expected a deployment mismatch blocker: {report:?}" + ); + assert!( + report + .conflicts + .iter() + .any(|conflict| conflict.kind == RestoreConflictKind::GenerationRegression), + "expected a generation regression conflict: {report:?}" + ); + + // Equal generations are allowed so drills can repeat. + let mut request = restore_request(&bundle, &target); + request.observed_generation = Some(SNAPSHOT_GENERATION); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!(report.restore_permitted(), "equal generation must not conflict: {report:?}"); + } + + #[tokio::test] + async fn dry_run_checks_the_operator_master_key() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let verifier = manifest + .local_kdf + .as_ref() + .and_then(|kdf| kdf.master_key_verifier.as_deref()) + .expect("encrypted bundle must carry a verifier"); + assert!(verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX), "got {verifier:?}"); + + // No passphrase: encrypted records make the master key a required + // external dependency. + let mut request = restore_request(&bundle, &target); + request.master_key = None; + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::ExternalDependencyUnavailable), + "expected a missing-master-key blocker: {report:?}" + ); + + // Wrong passphrase: reported as an external dependency mismatch. + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert_eq!(report.external_mismatches.len(), 1, "{report:?}"); + assert_eq!(report.external_mismatches[0].expected, verifier); + assert!(!report.restore_permitted()); + } + + #[tokio::test] + async fn dry_run_enumerates_existing_target_state() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + + // A populated target: a key file, the salt, and a stray file. + let (_occupant, target_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["occupied"]).await; + fs::write(target_dir.path().join("operator-notes.txt"), b"leftover") + .await + .expect("stray file"); + + let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, target_dir.path())) + .await + .expect("dry run"); + let conflicting_ids: Vec<&str> = report.conflicts.iter().map(|conflict| conflict.key_id.as_str()).collect(); + assert!(conflicting_ids.contains(&"occupied"), "{report:?}"); + assert!(conflicting_ids.contains(&LOCAL_KMS_MASTER_KEY_SALT_FILE), "{report:?}"); + assert!(conflicting_ids.contains(&"operator-notes.txt"), "{report:?}"); + assert!(!report.restore_permitted()); + } + + #[tokio::test] + async fn restore_into_empty_target_round_trips_byte_for_byte() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let bundle_before = snapshot_tree(&bundle); + let report = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("restore should succeed"); + + assert_eq!( + report, + LocalRestoreReport { + backup_id: BACKUP_ID.to_string(), + snapshot_generation: SNAPSHOT_GENERATION, + restored_key_ids: vec!["alpha".to_string(), "beta".to_string()], + salt_restored: true, + resumed: false, + } + ); + assert_eq!(snapshot_tree(&bundle), bundle_before, "the bundle source must stay byte-identical"); + assert_eq!( + top_level_names(&target).await, + vec![ + LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string(), + "alpha.key".to_string(), + "beta.key".to_string() + ], + "cutover must leave no marker, staging, or temp files behind" + ); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + + // The restored directory boots as a normal backend directory. + let restored = LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))) + .await + .expect("restored directory must pass startup validation"); + restored.describe_key("alpha", None).await.expect("restored key describes"); + } + + #[tokio::test] + async fn default_fail_policy_never_writes() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.conflict_policy = RestoreConflictPolicy::Fail; + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("the default policy must refuse to write"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(!target.exists(), "the fail policy must not create the target"); + } + + #[tokio::test] + async fn non_empty_targets_are_rejected() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + + // Existing key material. + let (_occupant, occupied) = source_with_keys(Some(TEST_MASTER_KEY), &["occupied"]).await; + let before = snapshot_tree(occupied.path()); + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, occupied.path())) + .await + .expect_err("occupied target must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert_eq!(snapshot_tree(occupied.path()), before, "the rejected target must stay untouched"); + + // An orphan salt alone makes the target non-empty: a foreign salt + // would poison every key created after the restore. + let orphan = TempDir::new().expect("orphan dir"); + fs::write(orphan.path().join(LOCAL_KMS_MASTER_KEY_SALT_FILE), [7u8; 16]) + .await + .expect("seed orphan salt"); + let before = snapshot_tree(orphan.path()); + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, orphan.path())) + .await + .expect_err("orphan salt must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("salt"), "got {error}"); + assert_eq!(snapshot_tree(orphan.path()), before); + + // A marker of a different restore. + let foreign = TempDir::new().expect("foreign dir"); + let mut digest = manifest.manifest_digest.clone(); + digest.hex = digest.hex.chars().rev().collect(); + let foreign_marker = RestoreCommitMarker { + format_version: RESTORE_MARKER_FORMAT_VERSION, + backup_id: "backup-foreign".to_string(), + manifest_digest: digest, + files: vec!["foreign.key".to_string()], + }; + fs::write( + foreign.path().join(LOCAL_RESTORE_COMMIT_MARKER_FILE), + serde_json::to_vec_pretty(&foreign_marker).expect("marker json"), + ) + .await + .expect("seed foreign marker"); + let before = snapshot_tree(foreign.path()); + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, foreign.path())) + .await + .expect_err("foreign marker must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("different bundle"), "got {error}"); + assert_eq!(snapshot_tree(foreign.path()), before); + } + + #[tokio::test] + async fn wrong_passphrase_fails_via_verifier_before_any_target_write() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("wrong master key must be rejected by the verifier"); + assert!(matches!(error, KmsError::ValidationError { .. }), "got {error:?}"); + assert!(!target.exists(), "the verifier must reject before any target write"); + } + + #[tokio::test] + async fn wrong_passphrase_fails_via_probe_when_bundle_has_no_verifier() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + // Simulate a bundle produced before the verifier landed. + reseal_manifest(&bundle, |manifest| { + manifest + .local_kdf + .as_mut() + .expect("local bundle has a KDF descriptor") + .master_key_verifier = None; + }) + .await; + + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("wrong master key must be caught by the decryption probe"); + assert!(matches!(error, KmsError::MaterialAuthenticationFailed { .. }), "got {error:?}"); + assert!(!target.exists(), "the in-memory probe must reject before any target write"); + + // The right passphrase still restores the verifier-less bundle. + restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("verifier-less bundle must restore with the right master key"); + assert_restored_matches_source(&client, &target, &["alpha"]).await; + } + + #[tokio::test] + async fn bundle_defects_fail_closed_on_restore() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["victim"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + let request = restore_request(&bundle, &target); + + let artifact_file = bundle.join(&manifest.artifacts[0].path); + let original_artifact = std::fs::read(&artifact_file).expect("artifact bytes"); + + // Tampered artifact byte. + let mut tampered = original_artifact.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0x01; + std::fs::write(&artifact_file, &tampered).expect("write tampered"); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("tampered artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted), + "{report:?}" + ); + + // Truncated artifact. + std::fs::write(&artifact_file, &original_artifact[..original_artifact.len() - 4]).expect("truncate"); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("truncated artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); + std::fs::write(&artifact_file, &original_artifact).expect("restore artifact"); + + // Wrong KEK identity and wrong KEK material. + let wrong_id = BackupKek::new("other-kek", 1, [0x42; 32]).expect("kek"); + let error = restore_local_backup(&wrong_id, &request) + .await + .expect_err("wrong KEK id must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); + let wrong_material = BackupKek::new("backup-kek-test", 1, [0x24; 32]).expect("kek"); + let error = restore_local_backup(&wrong_material, &request) + .await + .expect_err("wrong KEK material must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + + // Missing manifest: the bundle never sealed. + std::fs::remove_file(bundle.join(LOCAL_BUNDLE_MANIFEST_FILE)).expect("remove manifest"); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("unsealed bundle must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::IncompleteBundle { .. })), "got {error:?}"); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::IncompleteBundle), + "{report:?}" + ); + + assert!(!target.exists(), "no rejected bundle may touch the target"); + } + + #[tokio::test] + async fn kdf_parameter_drift_is_rejected() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + reseal_manifest(&bundle, |manifest| { + let descriptor = manifest.local_kdf.as_mut().expect("local bundle has a KDF descriptor"); + match &mut descriptor.derivation { + LocalKeyDerivation::Argon2id { memory_kib, .. } => *memory_kib *= 2, + other => panic!("expected an Argon2id descriptor, got {other:?}"), + } + }) + .await; + + let request = restore_request(&bundle, &target); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("a KDF parameter drift must be rejected"); + assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "got {error:?}"); + assert!(!target.exists()); + + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::UnsupportedBackend), + "{report:?}" + ); + } + + #[tokio::test] + async fn unknown_verifier_scheme_fails_closed() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + reseal_manifest(&bundle, |manifest| { + manifest + .local_kdf + .as_mut() + .expect("local bundle has a KDF descriptor") + .master_key_verifier = Some("post-quantum-v2:0123abcd".to_string()); + }) + .await; + + let request = restore_request(&bundle, &target); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("an unknown verifier scheme must fail closed"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + assert!(!target.exists()); + + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted), + "{report:?}" + ); + } + + #[tokio::test] + async fn generation_regression_is_rejected_and_equal_is_allowed() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.observed_generation = Some(SNAPSHOT_GENERATION + 1); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("a strictly lower bundle generation must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(!target.exists()); + + request.observed_generation = Some(SNAPSHOT_GENERATION); + restore_local_backup(&test_kek(), &request) + .await + .expect("an equal generation must stay restorable for repeated drills"); + assert_restored_matches_source(&client, &target, &["alpha"]).await; + } + + #[tokio::test] + async fn dev_mode_bundle_round_trips_without_passphrase() { + let (client, _source_dir) = source_with_keys(None, &["dev-key"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.master_key_verifier, None, "dev-mode bundles have no master key to verify"); + + let mut request = restore_request(&bundle, &target); + request.master_key = None; + let report = restore_local_backup(&test_kek(), &request) + .await + .expect("dev-mode restore must succeed without a passphrase"); + assert!(!report.salt_restored); + assert_eq!(report.restored_key_ids, vec!["dev-key".to_string()]); + + let source_bytes = fs::read(client.key_directory().join("dev-key.key")).await.expect("source"); + let restored_bytes = fs::read(target.join("dev-key.key")).await.expect("restored"); + assert_eq!(source_bytes, restored_bytes); + + let restored = LocalKmsClient::new(local_config(&target, None)) + .await + .expect("restored dev directory must boot"); + restored.describe_key("dev-key", None).await.expect("restored key describes"); + } + + #[tokio::test] + async fn legacy_sha256_bundle_round_trips() { + // A pre-beta.9 directory: legacy SHA-256 encrypted record, no salt, + // no protection marker (same fixture as the backend's beta.5 test; + // the material decrypts to 32 x 0x42 under this master key). + let source_dir = TempDir::new().expect("legacy dir"); + let master_key = "beta5-test-master-key"; + let record = serde_json::json!({ + "key_id": "beta5-key", + "version": 1u32, + "algorithm": "AES_256", + "usage": "EncryptDecrypt", + "status": "Active", + "description": serde_json::Value::Null, + "metadata": std::collections::HashMap::<String, String>::new(), + "created_at": "2024-01-01T00:00:00+00:00", + "rotated_at": serde_json::Value::Null, + "created_by": "beta5-fixture", + "encrypted_key_material": "xjwGa4Lj4qzKg6XQl8s2btyFkPHPChMAkjqs268TFGyvFUv8WjDD5HQCUDLViZmt", + "nonce": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + }); + fs::write( + source_dir.path().join("beta5-key.key"), + serde_json::to_vec_pretty(&record).expect("record json"), + ) + .await + .expect("write legacy record"); + + // Export through the read-only constructor so no Argon2 salt gets + // created: the bundle must record the legacy derivation. + let client = LocalKmsClient::new_for_key_export(local_config(source_dir.path(), Some(master_key))) + .await + .expect("open legacy dir read-only"); + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.derivation, LocalKeyDerivation::LegacySha256); + let verifier = kdf.master_key_verifier.as_deref().expect("legacy verifier"); + assert!(verifier.starts_with(MASTER_KEY_VERIFIER_LEGACY_PREFIX), "got {verifier:?}"); + + let target = work.path().join("target"); + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("the legacy verifier must reject a wrong master key"); + assert!(matches!(error, KmsError::ValidationError { .. }), "got {error:?}"); + assert!(!target.exists()); + + request.master_key = Some(master_key.to_string()); + let report = restore_local_backup(&test_kek(), &request).await.expect("legacy restore"); + assert!(!report.salt_restored, "a legacy bundle carries no salt"); + + // The restored directory boots (creating a fresh Argon2 salt is + // legitimate for legacy records) and the material still decrypts via + // the legacy fallback. + let restored = LocalKmsClient::new(local_config(&target, Some(master_key))) + .await + .expect("restored legacy directory must boot"); + let material = restored + .decrypt_key_material_for_export("beta5-key") + .await + .expect("legacy material must decrypt after restore"); + assert_eq!(material.as_ref(), &[0x42; 32]); + } + + #[tokio::test] + async fn crash_during_staging_keeps_top_level_untouched_and_rerun_converges() { + use durable_file::{CommitStep, failpoint}; + + for step in [ + CommitStep::TempWritten, + CommitStep::FileSynced, + CommitStep::Published, + CommitStep::DirSynced, + ] { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + let request = restore_request(&bundle, &target); + + failpoint::arm(&target, step); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("armed staging commit must simulate a crash"); + failpoint::disarm(&target); + assert!(error.to_string().contains("injected crash"), "step {step:?}: {error}"); + + // Top level: only the staging directory, never a marker or a key. + assert_eq!( + top_level_names(&target).await, + vec![RESTORE_STAGING_DIR.to_string()], + "step {step:?}: a staging crash must leave the top level untouched" + ); + + restore_local_backup(&test_kek(), &request) + .await + .unwrap_or_else(|error| panic!("step {step:?}: re-run must converge: {error}")); + assert_restored_matches_source(&client, &target, &["alpha"]).await; + } + } + + /// Drive the phases manually up to the marker commit, crash the commit at + /// every step, and prove both re-entry outcomes: an unpublished marker + /// temp is removable leftover state, a published marker rolls forward. + #[tokio::test] + async fn crash_around_marker_publish_recovers() { + use durable_file::{CommitStep, failpoint}; + + for step in [ + CommitStep::TempWritten, + CommitStep::FileSynced, + CommitStep::Published, + CommitStep::DirSynced, + ] { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + let request = restore_request(&bundle, &target); + + // Stage everything through the real phase functions. + let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); + let staging = target.join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("create staging"); + let salt = decoded.salt.as_ref().expect("bundle salt"); + stage_file(&staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, salt, Some(0o600)) + .await + .expect("stage salt"); + for record in &decoded.records { + stage_file(&staging, &record.file_name, &record.plaintext, Some(0o600)) + .await + .expect("stage record"); + } + let marker = RestoreCommitMarker::for_bundle(&decoded); + + failpoint::arm(&target, step); + let error = publish_marker(&target, &marker, Some(0o600)) + .await + .expect_err("armed marker commit must simulate a crash"); + failpoint::disarm(&target); + assert!(error.to_string().contains("injected crash"), "step {step:?}: {error}"); + + let marker_path = target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + let published = matches!(step, CommitStep::Published | CommitStep::DirSynced); + assert_eq!(marker_path.exists(), published, "step {step:?}"); + if !published { + // The unpublished marker temp is exactly the shape startup + // recovery and a restore re-run both classify as removable. + let temp = top_level_names(&target) + .await + .into_iter() + .find(|name| is_orphan_commit_temp_name(name)); + assert!(temp.is_some(), "step {step:?}: expected a leftover marker temp"); + } else { + // A published marker makes backend startup fail closed. + let error = match LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))).await { + Ok(_) => panic!("step {step:?}: startup must fail closed on a published marker"), + Err(error) => error, + }; + assert!(matches!(error, KmsError::ConfigurationError { .. }), "step {step:?}: {error:?}"); + assert!(error.to_string().contains("unfinished restore"), "step {step:?}: {error}"); + } + + let report = restore_local_backup(&test_kek(), &request) + .await + .unwrap_or_else(|error| panic!("step {step:?}: re-run must converge: {error}")); + assert_eq!(report.resumed, published, "step {step:?}"); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + } + } + + #[tokio::test] + async fn crash_mid_cutover_fails_startup_then_rolls_forward_or_aborts() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + + // Reconstruct the exact state of a cutover killed after k of n links. + let build_partial_cutover = |target: PathBuf| { + let bundle = bundle.clone(); + async move { + let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); + let staging = target.join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("create staging"); + stage_file( + &staging, + LOCAL_KMS_MASTER_KEY_SALT_FILE, + decoded.salt.as_ref().expect("bundle salt"), + Some(0o600), + ) + .await + .expect("stage salt"); + for record in &decoded.records { + stage_file(&staging, &record.file_name, &record.plaintext, Some(0o600)) + .await + .expect("stage record"); + } + let marker = RestoreCommitMarker::for_bundle(&decoded); + publish_marker(&target, &marker, Some(0o600)).await.expect("publish marker"); + // First link only (the salt), then "power loss". + durable_file::link_durably( + staging.join(LOCAL_KMS_MASTER_KEY_SALT_FILE), + target.join(LOCAL_KMS_MASTER_KEY_SALT_FILE), + ) + .await + .expect("link salt"); + } + }; + + // Outcome 1: roll forward with the same bundle. + let target = work.path().join("target-forward"); + build_partial_cutover(target.clone()).await; + let error = match LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))).await { + Ok(_) => panic!("startup must fail closed mid-cutover"), + Err(error) => error, + }; + assert!(error.to_string().contains("unfinished restore"), "got {error}"); + assert!(matches!( + LocalKmsClient::new_for_key_export(local_config(&target, Some(TEST_MASTER_KEY))).await, + Err(KmsError::ConfigurationError { .. }) + )); + let report = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("roll-forward must converge"); + assert!(report.resumed); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + + // Outcome 2: explicit abort returns the complete old (empty) state. + let target = work.path().join("target-abort"); + build_partial_cutover(target.clone()).await; + abort_local_restore(&target).await.expect("abort must succeed"); + assert_eq!( + top_level_names(&target).await, + Vec::<String>::new(), + "abort must take back every published file, the marker, and staging" + ); + // The aborted target is a healthy empty directory again: both a fresh + // backend start and a fresh restore work. + restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("a fresh restore after abort must succeed"); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + } + + #[tokio::test] + async fn roll_forward_requires_identical_file_set() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); + fs::create_dir_all(target.join(RESTORE_STAGING_DIR)) + .await + .expect("create staging"); + let mut marker = RestoreCommitMarker::for_bundle(&decoded); + marker.files.pop(); + publish_marker(&target, &marker, Some(0o600)).await.expect("publish marker"); + + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect_err("a marker with a diverging file set must fail closed"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + assert!(error.to_string().contains("different file set"), "got {error}"); + } + + #[tokio::test] + async fn completed_restore_tolerates_inert_staging_leftover() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("restore"); + + // A crash between marker removal and staging cleanup leaves an inert + // staging directory; startup must ignore it entirely. + let staging = target.join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("recreate staging"); + fs::write(staging.join("leftover"), b"inert").await.expect("seed leftover"); + + let restored = LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))) + .await + .expect("startup must ignore a leftover staging directory"); + restored.describe_key("alpha", None).await.expect("key describes"); + assert!(staging.exists(), "startup must not touch the staging leftover"); + } + + #[tokio::test] + async fn startup_removes_unpublished_marker_temp() { + let (client, source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + drop(client); + let temp_name = format!("{LOCAL_RESTORE_COMMIT_MARKER_FILE}.tmp-{}", uuid::Uuid::new_v4()); + let temp_path = source_dir.path().join(&temp_name); + fs::write(&temp_path, b"unpublished marker").await.expect("seed marker temp"); + + let client = LocalKmsClient::new(local_config(source_dir.path(), Some(TEST_MASTER_KEY))) + .await + .expect("an unpublished marker temp must not block startup"); + assert!(!temp_path.exists(), "startup recovery must remove the unpublished marker temp"); + client.describe_key("alpha", None).await.expect("key describes"); + } + + #[tokio::test] + async fn abort_without_restore_in_progress_errors_and_staging_only_is_cleaned() { + let empty = TempDir::new().expect("empty dir"); + let error = abort_local_restore(empty.path()) + .await + .expect_err("nothing to abort must be an error"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let staging_only = TempDir::new().expect("staging dir"); + let staging = staging_only.path().join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("create staging"); + fs::write(staging.join("partial.key"), b"partial") + .await + .expect("seed partial"); + abort_local_restore(staging_only.path()) + .await + .expect("a pre-marker abort only removes staging"); + assert_eq!(top_level_names(staging_only.path()).await, Vec::<String>::new()); + } +} diff --git a/crates/kms/src/backup/mod.rs b/crates/kms/src/backup/mod.rs index d3f9134e9..53b0fa741 100644 --- a/crates/kms/src/backup/mod.rs +++ b/crates/kms/src/backup/mod.rs @@ -16,9 +16,10 @@ //! //! The contract side defines the versioned backup manifest, the per-backend //! responsibility matrix, typed failure modes, and the restore dry-run -//! report. [`local_export`] implements the producer side for the Local -//! backend as a crate-internal API; restore orchestration and the admin API -//! build on these pieces in follow-up changes. +//! report. [`local_export`] implements the producer side and +//! [`local_restore`] the consumer side for the Local backend as +//! crate-internal APIs; the admin API builds on these pieces in follow-up +//! changes. //! //! # Bundle model //! @@ -51,6 +52,7 @@ mod capability; mod dry_run; mod error; pub mod local_export; +pub mod local_restore; mod manifest; pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; @@ -62,6 +64,10 @@ pub use local_export::{ BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup, read_local_bundle_manifest, }; +pub use local_restore::{ + LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore, + restore_local_backup, +}; pub use manifest::{ AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, From 9080ea8ea0c1751fc77f0d9c313cc7e87abe98fc Mon Sep 17 00:00:00 2001 From: Zhengchao An <anzhengchao@gmail.com> Date: Sat, 1 Aug 2026 10:14:32 +0800 Subject: [PATCH 51/52] test(object-lock): cover unretained version cleanup (#5504) Co-authored-by: cxymds <cxymds@gmail.com> --- .../src/object_lock/object_lock_test.rs | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/e2e_test/src/object_lock/object_lock_test.rs b/crates/e2e_test/src/object_lock/object_lock_test.rs index 7cb04cf4d..0748ad499 100644 --- a/crates/e2e_test/src/object_lock/object_lock_test.rs +++ b/crates/e2e_test/src/object_lock/object_lock_test.rs @@ -26,6 +26,7 @@ use super::common::*; use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::{ByteStream, DateTimeFormat}; use aws_sdk_s3::types::{ CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus, @@ -2120,6 +2121,127 @@ async fn test_multipart_default_retention_fixed_at_create() { // Versioning Auto-Enable Tests // ============================================================================ +#[tokio::test] +#[serial] +async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() { + init_logging(); + info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)"); + + let mut env = ObjectLockTestEnvironment::new() + .await + .expect("failed to create Object Lock test environment"); + env.start_rustfs().await.expect("failed to start RustFS"); + + let bucket = "test-object-lock-delete-cleanup"; + let key = "unretained-object"; + + env.create_object_lock_bucket(bucket) + .await + .expect("failed to create Object Lock bucket"); + let client = env.s3_client(); + + let put_response = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"unretained data")) + .send() + .await + .expect("failed to upload unretained object"); + let object_version_id = put_response + .version_id() + .expect("Object Lock buckets must create versioned objects") + .to_string(); + + let delete_response = client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("failed to create delete marker"); + assert_eq!(delete_response.delete_marker(), Some(true)); + let delete_marker_version_id = delete_response + .version_id() + .expect("Deleting without a version ID must create a delete marker") + .to_string(); + + let get_error = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect_err("GET must not return an object hidden by a delete marker"); + assert_eq!(get_error.raw_response().map(|response| response.status().as_u16()), Some(404)); + assert_eq!(get_error.as_service_error().and_then(|error| error.code()), Some("NoSuchKey")); + + let listed_objects = client + .list_objects_v2() + .bucket(bucket) + .send() + .await + .expect("failed to list current objects"); + assert!( + listed_objects.contents().iter().all(|object| object.key() != Some(key)), + "ListObjectsV2 must hide objects whose latest version is a delete marker" + ); + + let listed_versions = client + .list_object_versions() + .bucket(bucket) + .send() + .await + .expect("failed to list object versions"); + assert!( + listed_versions + .versions() + .iter() + .any(|version| version.key() == Some(key) && version.version_id() == Some(object_version_id.as_str())), + "The data version must remain until it is explicitly deleted" + ); + assert!( + listed_versions + .delete_markers() + .iter() + .any(|marker| marker.key() == Some(key) && marker.version_id() == Some(delete_marker_version_id.as_str())), + "ListObjectVersions must expose the delete marker" + ); + + client + .delete_object() + .bucket(bucket) + .key(key) + .version_id(object_version_id) + .send() + .await + .expect("failed to delete the data version"); + client + .delete_object() + .bucket(bucket) + .key(key) + .version_id(delete_marker_version_id) + .send() + .await + .expect("failed to delete the delete marker"); + + let remaining_versions = client + .list_object_versions() + .bucket(bucket) + .send() + .await + .expect("failed to list versions after cleanup"); + assert!(remaining_versions.versions().is_empty()); + assert!(remaining_versions.delete_markers().is_empty()); + + client + .delete_bucket() + .bucket(bucket) + .send() + .await + .expect("Deleting every version must remove xl.meta so the bucket can be deleted normally"); +} + #[tokio::test] #[serial] async fn test_versioning_auto_enabled_with_object_lock() { From 4b6b6f14bde8bca9823e026c9a37551f159a099e Mon Sep 17 00:00:00 2001 From: houseme <housemecn@gmail.com> Date: Sat, 1 Aug 2026 10:43:52 +0800 Subject: [PATCH 52/52] feat(hotpath): use mimalloc in inner counting allocator (#5523) * feat(hotpath): use mimalloc in counting allocator * fix(hotpath): adapt mimalloc for allocation counting --------- Co-authored-by: overtrue <anzhengchao@gmail.com> --- rustfs/src/main.rs | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 74d652be3..75f9c81c4 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -12,9 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +use std::alloc::{GlobalAlloc, Layout}; + +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +#[derive(Default)] +struct DefaultMiMalloc; + +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +// SAFETY: allocation and deallocation are forwarded unchanged to MiMalloc, so +// MiMalloc's GlobalAlloc guarantees apply to every returned pointer and layout. +#[allow(unsafe_code)] +unsafe impl GlobalAlloc for DefaultMiMalloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: the caller upholds GlobalAlloc's contract for layout. + unsafe { mimalloc::MiMalloc.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: ptr and layout came from this allocator and are forwarded unchanged. + unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) } + } +} + #[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] #[global_allocator] -static GLOBAL: hotpath::CountingAllocator = hotpath::CountingAllocator::new(); +static GLOBAL: hotpath::CountingAllocator<DefaultMiMalloc> = hotpath::CountingAllocator::new(); #[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))] #[global_allocator] @@ -25,3 +48,15 @@ fn main() { rustfs::startup_entrypoint::run_process(); } + +#[cfg(all(test, feature = "hotpath", feature = "hotpath-alloc"))] +mod tests { + #[test] + #[allow(unsafe_code)] + fn hotpath_allocator_uses_mimalloc() { + let allocation = Box::new([0_u8; 64]); + + // SAFETY: the live Box pointer is valid to inspect for heap ownership. + assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) }); + } +}