feat(admin): expose KMS backup and restore behind explicit guards (#5579)

* feat(kms): add backup and restore admin API

Wires the merged KMS backup contract, Local export and Local restore into
the admin API: export a sealed bundle, run a zero-write restore preflight,
execute a confirmed restore, roll an interrupted restore back, and report
subsystem readiness.

- Dedicated kms:Backup / kms:Restore actions, recorded in the admin route
  matrix. Neither is reachable through any other KMS action.
- Restore requires two independent confirmations: an echo of the bundle
  manifest's backup id, and an explicitly named conflict policy (the
  default never writes).
- The backup KEK comes from the environment and is refused when it reuses
  a secret of the configured backend, compared both as the literal value
  and as raw key bytes.
- No endpoint accepts a path: bundles are addressed by a validated name
  under a configured root, and the restore target is always the server's
  own configured key directory.
- Bundles now carry a sanitized configuration artifact built as an
  allowlist projection, so a future backend credential field cannot leak
  into a bundle by default. Restore verifies it and never applies it.
- Audit entries go through the existing KMS admin wiring and carry
  identifiers only.

* test(kms): pin the backup admin API gates

Fixes the test KEK to a real 32-byte value and drives the export refusal
from the configured backend rather than from the handle that happens to
be available, so a Local handle cannot export on behalf of a backend
whose material RustFS does not own.

Cargo.lock

@@ -9125,6 +9125,7 @@ dependencies = [
  "url",
  "urlencoding",
  "uuid",
+ "zeroize",
  "zip",
  "zstd",
 ]

crates/kms/src/backends/local.rs

@@ -1765,6 +1765,10 @@ impl KmsBackend for LocalKmsBackend {
         self.client.health_check().await.map(|_| true)
     }
 
+    fn local_backup_client(&self) -> Option<&LocalKmsClient> {
+        Some(&self.client)
+    }
+
     fn capabilities(&self) -> BackendCapabilities {
         // Rotation stays unadvertised until historical key versions can be
         // retained (see LocalKmsClient::rotate_key); without version history

crates/kms/src/backends/mod.rs

@@ -241,6 +241,19 @@ pub trait KmsBackend: Send + Sync {
     async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
         Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
     }
+
+    /// The running client to export a full-material backup bundle from.
+    ///
+    /// Only the Local backend owns key material RustFS is allowed to export in
+    /// full (see [`crate::backup::BackupResponsibility`]); every other backend
+    /// keeps its cryptographic root outside RustFS and returns `None` here.
+    ///
+    /// The export must run against the *running* client so that its fence
+    /// actually blocks concurrent create/delete work — a second client opened
+    /// on the same key directory would fence nothing.
+    fn local_backup_client(&self) -> Option<&local::LocalKmsClient> {
+        None
+    }
 }
 
 /// Outcome of [`KmsBackend::remove_expired_key`].

crates/kms/src/backup/local_export.rs

@@ -74,6 +74,7 @@ pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json";
 const ARTIFACTS_DIR: &str = "artifacts";
 const KEYS_DIR: &str = "artifacts/keys";
 const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc";
+const CONFIG_ARTIFACT_PATH: &str = "artifacts/kms-config.json.enc";
 pub(crate) const AEAD_NONCE_LEN: usize = 12;
 /// Domain-separation context for the artifact AAD binding.
 const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1";
@@ -141,6 +142,15 @@ pub struct LocalBackupExportRequest {
     pub snapshot_generation: u64,
     /// Bundle output directory; must not exist yet or must be empty.
     pub destination: PathBuf,
+    /// Serialized sanitized KMS configuration to seal into the bundle, if the
+    /// caller produced one.
+    ///
+    /// The persisted `KmsConfig` carries plaintext credentials (Vault token,
+    /// AppRole secret id, Local master key), so the sanitized projection is
+    /// owned by the admin layer and this module only seals the bytes it is
+    /// handed. The artifact is evidence for an operator decision: restore
+    /// verifies it opens but never applies a configuration.
+    pub sanitized_config: Option<Vec<u8>>,
 }
 
 impl LocalBackupExportRequest {
@@ -410,6 +420,10 @@ async fn build_and_write_bundle(
         let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::MasterKeySalt, SALT_ARTIFACT_PATH, salt).await?;
         artifacts.push(descriptor);
     }
+    if let Some(config) = &request.sanitized_config {
+        let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::KmsConfig, CONFIG_ARTIFACT_PATH, config).await?;
+        artifacts.push(descriptor);
+    }
 
     // Make the artifact directory entries durable before sealing: the sealed
     // manifest must never survive a crash that its artifacts did not.
@@ -673,6 +687,7 @@ mod tests {
             rustfs_version: "1.0.0-test".to_string(),
             snapshot_generation: 7,
             destination,
+            sanitized_config: None,
         }
     }
 

crates/kms/src/backup/local_restore.rs

@@ -522,6 +522,7 @@ async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result<DecodedBund
 
     let mut records = Vec::new();
     let mut salt: Option<Zeroizing<Vec<u8>>> = None;
+    let mut config_seen = false;
     for artifact in &manifest.artifacts {
         match artifact.kind {
             ArtifactKind::KeyMaterial => {
@@ -543,6 +544,18 @@ async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result<DecodedBund
                 }
                 salt = Some(plaintext);
             }
+            // Configuration is evidence, not restorable state: the artifact is
+            // verified so a tampered bundle still fails closed, but restore
+            // never writes a configuration into the target. Presenting the
+            // difference against the running configuration, and any decision to
+            // adopt it, belongs to the admin layer.
+            ArtifactKind::KmsConfig => {
+                if config_seen {
+                    return Err(BackupError::corrupted("bundle carries more than one KMS configuration artifact").into());
+                }
+                config_seen = true;
+                decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?;
+            }
             // No producer emits these for a Local bundle yet; restoring a
             // bundle that carries state this implementation cannot apply
             // would silently drop it, so fail closed instead.
@@ -1072,6 +1085,14 @@ mod tests {
     }
 
     async fn export_bundle(client: &LocalKmsClient, dir: &Path) -> (PathBuf, BackupManifest) {
+        export_bundle_with_config(client, dir, None).await
+    }
+
+    async fn export_bundle_with_config(
+        client: &LocalKmsClient,
+        dir: &Path,
+        sanitized_config: Option<Vec<u8>>,
+    ) -> (PathBuf, BackupManifest) {
         let destination = dir.join("bundle");
         let manifest = export_local_backup(
             client,
@@ -1082,6 +1103,7 @@ mod tests {
                 rustfs_version: "1.0.0-test".to_string(),
                 snapshot_generation: SNAPSHOT_GENERATION,
                 destination: destination.clone(),
+                sanitized_config,
             },
         )
         .await
@@ -1967,4 +1989,71 @@ mod tests {
             .expect("a pre-marker abort only removes staging");
         assert_eq!(top_level_names(staging_only.path()).await, Vec::<String>::new());
     }
+
+    /// A bundle carrying the sanitized configuration artifact stays fully
+    /// restorable, and the configuration never becomes target state: restore
+    /// verifies it and moves on, leaving the decision to the admin layer.
+    #[tokio::test]
+    async fn a_config_artifact_is_verified_but_never_restored() {
+        let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await;
+        let bundle_parent = TempDir::new().expect("bundle parent");
+        let config_bytes = br#"{"backend":"local","note":"sanitized"}"#.to_vec();
+        let (bundle, manifest) = export_bundle_with_config(&client, bundle_parent.path(), Some(config_bytes.clone())).await;
+        drop(client);
+
+        assert!(
+            manifest
+                .artifacts
+                .iter()
+                .any(|artifact| artifact.kind == ArtifactKind::KmsConfig),
+            "the bundle must carry the configuration artifact"
+        );
+
+        let target = TempDir::new().expect("target");
+        let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, target.path()))
+            .await
+            .expect("dry-run should evaluate a bundle with a config artifact");
+        assert!(report.blockers.is_empty(), "unexpected blockers: {:?}", report.blockers);
+
+        let mut request = restore_request(&bundle, target.path());
+        request.conflict_policy = RestoreConflictPolicy::RestoreIntoEmptyTarget;
+        let report = restore_local_backup(&test_kek(), &request)
+            .await
+            .expect("restore should succeed");
+        assert_eq!(report.restored_key_ids, vec!["alpha".to_string()]);
+
+        // Only key material and the salt land in the target; the configuration
+        // is evidence that stays in the bundle.
+        let names = top_level_names(target.path()).await;
+        assert!(names.contains(&"alpha.key".to_string()), "{names:?}");
+        for name in &names {
+            assert!(
+                !name.contains("config"),
+                "restore must not write a configuration into the key directory: {names:?}"
+            );
+        }
+        let restored = std::fs::read(target.path().join("alpha.key")).expect("restored record");
+        assert_ne!(restored, config_bytes);
+
+        // A tampered config artifact must still fail the bundle closed.
+        let descriptor = manifest
+            .artifacts
+            .iter()
+            .find(|artifact| artifact.kind == ArtifactKind::KmsConfig)
+            .expect("config artifact");
+        let artifact_path = bundle.join(&descriptor.path);
+        let mut payload = std::fs::read(&artifact_path).expect("artifact bytes");
+        let last = payload.len() - 1;
+        payload[last] ^= 0xff;
+        std::fs::write(&artifact_path, &payload).expect("tamper artifact");
+
+        let fresh_target = TempDir::new().expect("target");
+        dry_run_local_restore(&test_kek(), &restore_request(&bundle, fresh_target.path()))
+            .await
+            .expect("dry-run reports rather than errors")
+            .blockers
+            .iter()
+            .find(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted)
+            .expect("a tampered config artifact must be reported as corruption");
+    }
 }

crates/kms/src/manager.rs

@@ -448,6 +448,15 @@ impl KmsManager {
     pub(crate) fn backend(&self) -> Arc<dyn KmsBackend> {
         self.backend.clone()
     }
+
+    /// The running client a full-material backup can be exported from, or
+    /// `None` for backends whose cryptographic root lives outside RustFS.
+    ///
+    /// See [`KmsBackend::local_backup_client`] for why the export must use the
+    /// running client rather than a freshly opened one.
+    pub fn local_backup_client(&self) -> Option<&crate::backends::local::LocalKmsClient> {
+        self.backend.local_backup_client()
+    }
 }
 
 #[cfg(test)]

crates/policy/src/policy/action.rs

@@ -730,6 +730,13 @@ pub enum KmsAction {
     DescribeKeyAction,
     #[strum(serialize = "kms:Decrypt")]
     DecryptAction,
+    /// Export a KMS backup bundle. Separate from every key action because a
+    /// bundle carries the material of every key at once.
+    #[strum(serialize = "kms:Backup")]
+    BackupAction,
+    /// Preflight or execute a KMS restore.
+    #[strum(serialize = "kms:Restore")]
+    RestoreAction,
 }
 
 #[cfg(test)]
@@ -767,6 +774,8 @@ mod tests {
             ("kms:ListKeys", KmsAction::ListKeysAction),
             ("kms:DescribeKey", KmsAction::DescribeKeyAction),
             ("kms:Decrypt", KmsAction::DecryptAction),
+            ("kms:Backup", KmsAction::BackupAction),
+            ("kms:Restore", KmsAction::RestoreAction),
         ] {
             let action = Action::try_from(raw).expect("Should parse KMS action");
             assert_eq!(action, Action::KmsAction(expected));

rustfs/Cargo.toml

@@ -288,6 +288,7 @@ atoi = { workspace = true }
 atomic_enum = { workspace = true }
 async_zip = { workspace = true, default-features = false, features = ["tokio", "deflate"] }
 base64 = { workspace = true }
+zeroize = { workspace = true }
 hmac = { workspace = true }
 sha2 = { workspace = true }
 rsa = { workspace = true, features = ["sha2"] }

rustfs/src/admin/handlers/kms.rs

@@ -14,7 +14,7 @@
 
 //! KMS admin handlers for HTTP API
 
-use super::{kms_dynamic, kms_key_lifecycle, kms_keys, kms_management};
+use super::{kms_backup, kms_dynamic, kms_key_lifecycle, kms_keys, kms_management};
 use crate::admin::router::{AdminOperation, S3Router};
 
 pub fn register_kms_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -22,5 +22,6 @@ pub fn register_kms_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<(
     kms_dynamic::register_kms_dynamic_route(r)?;
     kms_keys::register_kms_key_route(r)?;
     kms_key_lifecycle::register_kms_key_lifecycle_route(r)?;
+    kms_backup::register_kms_backup_route(r)?;
     Ok(())
 }

rustfs/src/admin/handlers/kms_audit.rs

@@ -98,6 +98,16 @@ pub(super) enum KmsAdminOperation {
     Start,
     /// Stop of the KMS service.
     Stop,
+    /// Export of a backup bundle.
+    Backup,
+    /// Readiness lookup for the backup subsystem.
+    BackupStatus,
+    /// Zero-write restore preflight.
+    RestoreDryRun,
+    /// Execution of a restore.
+    Restore,
+    /// Roll-back of an interrupted restore.
+    RestoreAbort,
 }
 
 impl KmsAdminOperation {
@@ -109,6 +119,11 @@ impl KmsAdminOperation {
             Self::Reconfigure => "Reconfigure",
             Self::Start => "Start",
             Self::Stop => "Stop",
+            Self::Backup => "Backup",
+            Self::BackupStatus => "BackupStatus",
+            Self::RestoreDryRun => "RestoreDryRun",
+            Self::Restore => "Restore",
+            Self::RestoreAbort => "RestoreAbort",
         }
     }
 
@@ -121,6 +136,18 @@ impl KmsAdminOperation {
             Self::Configure | Self::Reconfigure => EventName::KmsServiceConfigured,
             Self::Start => EventName::KmsServiceStarted,
             Self::Stop => EventName::KmsServiceStopped,
+            // A backup reads the material of every key, and a restore
+            // preflight reads a bundle without touching the target: both are
+            // key access. The `kmsOperation` tag distinguishes them, and the
+            // event-name space is a fixed 64-bit mask that is nearly full, so
+            // these reuse the existing access event rather than claiming two
+            // more bits of it.
+            Self::Backup | Self::BackupStatus | Self::RestoreDryRun => EventName::KmsKeyAccessed,
+            // A restore publishes key material into the target directory.
+            Self::Restore => EventName::KmsKeyCreated,
+            // Aborting an interrupted restore removes the material a partial
+            // cutover had already published.
+            Self::RestoreAbort => EventName::KmsKeyDeleted,
         }
     }
 }
@@ -545,6 +572,11 @@ mod tests {
             KmsAdminOperation::Reconfigure,
             KmsAdminOperation::Start,
             KmsAdminOperation::Stop,
+            KmsAdminOperation::Backup,
+            KmsAdminOperation::BackupStatus,
+            KmsAdminOperation::RestoreDryRun,
+            KmsAdminOperation::Restore,
+            KmsAdminOperation::RestoreAbort,
         ] {
             assert!(
                 operation.event().is_kms(),

rustfs/src/admin/handlers/mod.rs

@@ -33,6 +33,7 @@ pub mod inspect_archive;
 pub mod is_admin;
 pub mod kms;
 pub mod kms_audit;
+pub mod kms_backup;
 pub mod kms_dynamic;
 pub mod kms_key_lifecycle;
 pub mod kms_keys;

rustfs/src/admin/route_policy.rs

@@ -54,6 +54,7 @@ const HEALTH_INFO: AdminActionRef = AdminActionRef::new("HealthInfoAdminAction")
 const IMPORT_BUCKET_METADATA: AdminActionRef = AdminActionRef::new("ImportBucketMetadataAction");
 const IMPORT_IAM: AdminActionRef = AdminActionRef::new("ImportIAMAction");
 const INSPECT_DATA: AdminActionRef = AdminActionRef::new("InspectDataAction");
+const KMS_BACKUP: AdminActionRef = AdminActionRef::new("kms:Backup");
 const KMS_CLEAR_CACHE: AdminActionRef = AdminActionRef::new("kms:ClearCache");
 const KMS_CONFIGURE: AdminActionRef = AdminActionRef::new("kms:Configure");
 const KMS_DELETE_KEY: AdminActionRef = AdminActionRef::new("kms:DeleteKey");
@@ -62,6 +63,7 @@ const KMS_DISABLE_KEY: AdminActionRef = AdminActionRef::new("kms:DisableKey");
 const KMS_ENABLE_KEY: AdminActionRef = AdminActionRef::new("kms:EnableKey");
 const KMS_GENERATE_DATA_KEY: AdminActionRef = AdminActionRef::new("kms:GenerateDataKey");
 const KMS_LIST_KEYS: AdminActionRef = AdminActionRef::new("kms:ListKeys");
+const KMS_RESTORE: AdminActionRef = AdminActionRef::new("kms:Restore");
 const KMS_ROTATE_KEY: AdminActionRef = AdminActionRef::new("kms:RotateKey");
 const KMS_SERVICE_CONTROL: AdminActionRef = AdminActionRef::new("kms:ServiceControl");
 const LIST_GROUPS: AdminActionRef = AdminActionRef::new("ListGroupsAdminAction");
@@ -788,6 +790,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         RouteRiskLevel::High,
     ),
     admin(HttpMethod::Post, "/rustfs/admin/v3/kms/keys/rotate", KMS_ROTATE_KEY, RouteRiskLevel::High),
+    // Backup and restore act on the material of every key at once, so they
+    // carry their own actions rather than reusing any per-key one.
+    admin(HttpMethod::Get, "/rustfs/admin/v3/kms/backup", KMS_BACKUP, RouteRiskLevel::Sensitive),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/kms/backup", KMS_BACKUP, RouteRiskLevel::High),
+    admin(
+        HttpMethod::Post,
+        "/rustfs/admin/v3/kms/restore/dry-run",
+        KMS_RESTORE,
+        RouteRiskLevel::Sensitive,
+    ),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/kms/restore", KMS_RESTORE, RouteRiskLevel::High),
+    admin(HttpMethod::Post, "/rustfs/admin/v3/kms/restore/abort", KMS_RESTORE, RouteRiskLevel::High),
     public(
         HttpMethod::Get,
         "/rustfs/admin/v3/oidc/providers",
@@ -1826,6 +1840,41 @@ mod tests {
         assert_action(HttpMethod::Delete, "/rustfs/admin/v3/kms/keys/delete", KMS_DELETE_KEY);
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/keys/cancel-deletion", KMS_DELETE_KEY);
         assert_action(HttpMethod::Get, "/rustfs/admin/v3/kms/keys/{key_id}", KMS_DESCRIBE_KEY);
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/backup", KMS_BACKUP);
+        assert_action(HttpMethod::Get, "/rustfs/admin/v3/kms/backup", KMS_BACKUP);
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/restore", KMS_RESTORE);
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/restore/dry-run", KMS_RESTORE);
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/kms/restore/abort", KMS_RESTORE);
+    }
+
+    /// Backup and restore expose the whole key inventory at once, so no other
+    /// KMS action may reach them: holding `kms:Configure` or a per-key action
+    /// must not be enough.
+    #[test]
+    fn route_policy_isolates_backup_and_restore_from_other_kms_actions() {
+        for (method, path) in [
+            (HttpMethod::Get, "/rustfs/admin/v3/kms/backup"),
+            (HttpMethod::Post, "/rustfs/admin/v3/kms/backup"),
+            (HttpMethod::Post, "/rustfs/admin/v3/kms/restore"),
+            (HttpMethod::Post, "/rustfs/admin/v3/kms/restore/dry-run"),
+            (HttpMethod::Post, "/rustfs/admin/v3/kms/restore/abort"),
+        ] {
+            for action in [
+                SERVER_INFO,
+                KMS_CONFIGURE,
+                KMS_DESCRIBE_KEY,
+                KMS_LIST_KEYS,
+                KMS_SERVICE_CONTROL,
+                KMS_DELETE_KEY,
+            ] {
+                assert_not_action(method, path, action);
+            }
+        }
+
+        // Backup and restore are separate privileges: neither implies the
+        // other.
+        assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/kms/backup", KMS_RESTORE);
+        assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/kms/restore", KMS_BACKUP);
     }
 
     #[test]

rustfs/src/admin/route_registration_test.rs

@@ -343,6 +343,11 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::POST, "/v3/kms/keys/enable"),
         admin_route(Method::POST, "/v3/kms/keys/disable"),
         admin_route(Method::POST, "/v3/kms/keys/rotate"),
+        admin_route(Method::GET, "/v3/kms/backup"),
+        admin_route(Method::POST, "/v3/kms/backup"),
+        admin_route(Method::POST, "/v3/kms/restore/dry-run"),
+        admin_route(Method::POST, "/v3/kms/restore"),
+        admin_route(Method::POST, "/v3/kms/restore/abort"),
         admin_route(Method::GET, "/v3/oidc/providers"),
         admin_route_sample(Method::GET, "/v3/oidc/authorize/{provider_id}", "/v3/oidc/authorize/default"),
         admin_route_sample(Method::GET, "/v3/oidc/callback/{provider_id}", "/v3/oidc/callback/default"),