diff --git a/_typos.toml b/_typos.toml index 779f6c48f..12d50d60e 100644 --- a/_typos.toml +++ b/_typos.toml @@ -47,6 +47,9 @@ consts = "consts" Hashi = "Hashi" # HashiCorp # Accept alternate spelling used in parser/XML comments. unparseable = "unparseable" +# Disaster-recovery objectives: recovery time and recovery point. +RTO = "RTO" +rto = "rto" [files] extend-exclude = [] diff --git a/crates/kms/examples/kms_dr_drill.rs b/crates/kms/examples/kms_dr_drill.rs new file mode 100644 index 000000000..9074bb202 --- /dev/null +++ b/crates/kms/examples/kms_dr_drill.rs @@ -0,0 +1,187 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Operator entry point for the KMS disaster-recovery drill. +//! +//! Runs one rehearsal inside a scratch workspace and writes the evidence +//! bundle. Everything it touches lives under that workspace: it never reads, +//! writes, or restores into a running deployment's key directory. See +//! `docs/operations/kms-disaster-recovery-drill.md` for the procedure and for +//! how to size the dataset so the measured recovery time transfers. +//! +//! Exit status is the verdict: 0 when every check held, 1 otherwise, so a +//! scheduled drill fails its job instead of quietly filing a bad report. + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill}; +use std::path::PathBuf; +use std::process::ExitCode; +use zeroize::Zeroizing; + +const ENV_WORKSPACE: &str = "RUSTFS_KMS_DRILL_WORKSPACE"; +const ENV_DRILL_ID: &str = "RUSTFS_KMS_DRILL_ID"; +const ENV_DISASTER: &str = "RUSTFS_KMS_DRILL_DISASTER"; +const ENV_DEPLOYMENT: &str = "RUSTFS_KMS_DRILL_DEPLOYMENT"; +const ENV_MASTER_KEY: &str = "RUSTFS_KMS_DRILL_MASTER_KEY"; +const ENV_KEYS: &str = "RUSTFS_KMS_DRILL_KEYS"; +const ENV_OBJECTS_PER_KEY: &str = "RUSTFS_KMS_DRILL_OBJECTS_PER_KEY"; +const ENV_OBJECT_BYTES: &str = "RUSTFS_KMS_DRILL_OBJECT_BYTES"; +const ENV_EVIDENCE: &str = "RUSTFS_KMS_DRILL_EVIDENCE"; +const ENV_FILE_PERMISSIONS: &str = "RUSTFS_KMS_DRILL_FILE_PERMISSIONS"; + +// Deliberately the same names the admin backup API reads: drilling with the +// KEK the real backups are sealed under is what proves that KEK is still +// retrievable, which is the part of a recovery no bundle can attest to. +const ENV_KEK: &str = "RUSTFS_KMS_BACKUP_KEK"; +const ENV_KEK_ID: &str = "RUSTFS_KMS_BACKUP_KEK_ID"; +const ENV_KEK_VERSION: &str = "RUSTFS_KMS_BACKUP_KEK_VERSION"; + +fn usage() -> String { + format!( + "KMS disaster-recovery drill.\n\ + \n\ + Required:\n \ + {ENV_WORKSPACE} absolute path to a scratch directory the drill owns\n \ + {ENV_MASTER_KEY} at-rest master key for the throwaway rehearsal deployment\n \ + {ENV_KEK} base64 of the 32-byte backup KEK\n \ + {ENV_KEK_ID} identifier recorded in the bundle manifest\n\ + \n\ + Optional:\n \ + {ENV_KEK_VERSION} backup KEK version (default 1)\n \ + {ENV_DRILL_ID} drill identifier and key-id prefix (default kms-dr-drill)\n \ + {ENV_DEPLOYMENT} deployment identity recorded in the bundle (default kms-dr-drill)\n \ + {ENV_DISASTER} key-directory-lost | master-key-salt-lost | key-record-corrupted\n \ + {ENV_KEYS} master keys to rehearse with (default 4)\n \ + {ENV_OBJECTS_PER_KEY} objects sealed per key (default 2)\n \ + {ENV_OBJECT_BYTES} plaintext bytes per object (default 4096)\n \ + {ENV_FILE_PERMISSIONS} octal key-file mode (default 600)\n \ + {ENV_EVIDENCE} evidence output path (default /evidence.json)" + ) +} + +fn required(name: &str) -> Result { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| format!("{name} is required\n\n{}", usage())) +} + +fn optional_usize(name: &str, default: usize) -> Result { + match std::env::var(name) { + Ok(value) => value + .trim() + .parse() + .map_err(|_| format!("{name} must be an unsigned integer")), + Err(_) => Ok(default), + } +} + +fn disaster_from_env() -> Result { + match std::env::var(ENV_DISASTER).ok().as_deref().map(str::trim) { + None | Some("") | Some("key-directory-lost") => Ok(DrillDisaster::KeyDirectoryLost), + Some("master-key-salt-lost") => Ok(DrillDisaster::MasterKeySaltLost), + Some("key-record-corrupted") => Ok(DrillDisaster::KeyRecordCorrupted), + Some(other) => Err(format!( + "{ENV_DISASTER}={other:?} is not a known disaster; use key-directory-lost, master-key-salt-lost, or key-record-corrupted" + )), + } +} + +fn kek_from_env() -> Result { + let raw = Zeroizing::new(required(ENV_KEK)?); + let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?); + if decoded.len() != 32 { + return Err(format!("{ENV_KEK} must decode to exactly 32 bytes")); + } + let mut material = [0u8; 32]; + material.copy_from_slice(&decoded); + let version = optional_usize(ENV_KEK_VERSION, 1)?; + let version = u32::try_from(version).map_err(|_| format!("{ENV_KEK_VERSION} is out of range"))?; + BackupKek::new(required(ENV_KEK_ID)?, version, material).map_err(|error| error.to_string()) +} + +async fn run() -> Result { + let workspace = PathBuf::from(required(ENV_WORKSPACE)?); + if !workspace.is_absolute() { + return Err(format!("{ENV_WORKSPACE} must be an absolute path")); + } + let drill_id = std::env::var(ENV_DRILL_ID) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "kms-dr-drill".to_string()); + let file_permissions = match std::env::var(ENV_FILE_PERMISSIONS) { + Ok(value) => Some(u32::from_str_radix(value.trim(), 8).map_err(|_| format!("{ENV_FILE_PERMISSIONS} must be octal"))?), + Err(_) => Some(0o600), + }; + + let request = DrillRequest { + deployment_identity: std::env::var(ENV_DEPLOYMENT) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| drill_id.clone()), + drill_id, + workspace: workspace.clone(), + rustfs_version: env!("CARGO_PKG_VERSION").to_string(), + // A drill always restores into a target that has observed nothing, so + // any generation is monotonic; the number is recorded as evidence of + // which snapshot the rehearsal covered. + snapshot_generation: 1, + dataset: DrillDataset { + keys: optional_usize(ENV_KEYS, 4)?, + objects_per_key: optional_usize(ENV_OBJECTS_PER_KEY, 2)?, + object_bytes: optional_usize(ENV_OBJECT_BYTES, 4096)?, + }, + disaster: disaster_from_env()?, + master_key: required(ENV_MASTER_KEY)?, + file_permissions, + }; + + let kek = kek_from_env()?; + let evidence = run_local_drill(&kek, &request).await.map_err(|error| error.to_string())?; + + let evidence_path = std::env::var(ENV_EVIDENCE) + .ok() + .filter(|value| !value.trim().is_empty()) + .map_or_else(|| workspace.join("evidence.json"), PathBuf::from); + let encoded = evidence.encode().map_err(|error| error.to_string())?; + std::fs::write(&evidence_path, &encoded).map_err(|error| format!("cannot write {}: {error}", evidence_path.display()))?; + + eprintln!("verdict: {:?}", evidence.verdict); + eprintln!("disaster: {:?}", evidence.disaster); + eprintln!( + "objects: {}/{} historical objects decrypted after the restore", + evidence.envelope_probes.iter().filter(|probe| probe.verified).count(), + evidence.envelope_probes.len() + ); + eprintln!("rpo: {} ms of work past the snapshot fence was lost", evidence.rpo.rpo_window_millis); + eprintln!("rto: {} ms of recovery work", evidence.rto_millis); + for finding in &evidence.findings { + eprintln!("finding: {finding}"); + } + eprintln!("evidence: {}", evidence_path.display()); + + Ok(evidence.verdict) +} + +#[tokio::main] +async fn main() -> ExitCode { + match run().await { + Ok(DrillVerdict::Passed) => ExitCode::SUCCESS, + Ok(DrillVerdict::Failed) => ExitCode::FAILURE, + Err(message) => { + eprintln!("{message}"); + ExitCode::FAILURE + } + } +} diff --git a/crates/kms/src/backup/drill.rs b/crates/kms/src/backup/drill.rs new file mode 100644 index 000000000..1ffc77d5b --- /dev/null +++ b/crates/kms/src/backup/drill.rs @@ -0,0 +1,1463 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Disaster-recovery drill harness for the Local backend. +//! +//! A backup that has never been restored is a hypothesis. This module turns +//! it into evidence: [`run_local_drill`] rehearses the complete loop — +//! seed a deployment, seal real objects through the production encryption +//! path, export a bundle, destroy the persistence layer, preflight, restore, +//! and then *decrypt the pre-disaster ciphertext again* — and returns a +//! machine-readable [`DrillEvidence`] record of what happened and how long it +//! took. +//! +//! # Why the object probe is the acceptance criterion +//! +//! Manifest digests and artifact counts only prove the bundle is intact. The +//! question a disaster actually asks is whether historical objects are still +//! readable, so the drill keeps the ciphertext and the encryption metadata of +//! every object it sealed before the disaster and, after the restore, decrypts +//! each one through a freshly opened backend. A drill whose probes do not +//! reproduce the original plaintext byte for byte is a failed drill regardless +//! of how clean the bundle looked. +//! +//! # Sandbox only +//! +//! The drill owns everything under its workspace: it creates the deployment it +//! later destroys. It never reads, writes, or points a restore at a running +//! deployment's key directory, and the destructive step is applied to the +//! workspace copy only. Sizing the dataset to production is what makes the +//! recovery-time number transferable — the code path is identical either way. +//! +//! # RPO and RTO +//! +//! The drill writes one key *after* the export fence closed and proves it is +//! absent after the restore: the recovery point is the snapshot generation, +//! not the moment of the disaster, and the drill measures that window instead +//! of asserting it. The recovery time is the sum of the phases an operator +//! actually waits on (quarantine, preflight, restore, verification); human +//! decision time is deliberately excluded and belongs to the runbook. +//! +//! # What this module does not re-test +//! +//! Bundle-side defects (tampered, truncated, wrong-KEK, incomplete, +//! unknown-version) and every commit-point crash are covered exhaustively by +//! the unit tests of [`crate::backup::local_export`] and +//! [`crate::backup::local_restore`]. The drill exercises the operator-facing +//! loop on top of them rather than restating them. +//! +//! The procedure this harness automates, how to read the evidence, and the +//! Vault variant are documented in +//! `docs/operations/kms-disaster-recovery-drill.md`. + +use crate::backends::local::{LOCAL_KMS_MASTER_KEY_SALT_FILE, LOCAL_RESTORE_COMMIT_MARKER_FILE, LocalKmsBackend}; +use crate::backends::{KmsBackend as KmsBackendTrait, local::validate_key_id}; +use crate::backup::capability::BackupBackendKind; +use crate::backup::dry_run::RestoreDryRunReport; +use crate::backup::local_export::{BackupKek, LocalBackupExportRequest, export_local_backup, read_local_bundle_manifest}; +use crate::backup::local_restore::{ + LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, dry_run_local_restore, restore_local_backup, +}; +use crate::backup::manifest::ContentDigest; +use crate::config::{BackendConfig, KmsConfig, LocalConfig}; +use crate::error::{KmsError, Result}; +use crate::manager::KmsManager; +use crate::service::ObjectEncryptionService; +use crate::types::{CreateKeyRequest, EncryptionAlgorithm, EncryptionMetadata, KeyUsage}; +use jiff::Zoned; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Instant, UNIX_EPOCH}; +use tokio::fs; +use tokio::io::AsyncReadExt; + +/// Format version of [`DrillEvidence`], so an evidence consumer can reject a +/// shape it does not know instead of guessing. +pub const DRILL_EVIDENCE_FORMAT_VERSION: u32 = 1; + +/// Bucket name the drill seals its sample objects under. +const DRILL_BUCKET: &str = "rustfs-kms-drill"; +/// Workspace subdirectory holding the deployment the drill destroys. +const LIVE_DIR: &str = "live"; +/// Workspace subdirectory holding the exported bundle. +const BUNDLE_DIR: &str = "bundle"; +/// Workspace subdirectory the damaged key directory is moved aside into. +const QUARANTINE_DIR: &str = "quarantine"; +/// Upper bound on a single drill run, guarding against a dataset that would +/// take an operator by surprise mid-incident-rehearsal. +const MAX_DRILL_OBJECTS: usize = 10_000; + +/// Size of the deployment a drill rehearses against. +/// +/// The numbers are evidence: a recovery time only transfers to production if +/// the drill restored a comparable number of keys. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DrillDataset { + /// Master keys created before the backup. + pub keys: usize, + /// Objects sealed under each key and re-read after the restore. + pub objects_per_key: usize, + /// Plaintext size of each sample object. + pub object_bytes: usize, +} + +impl Default for DrillDataset { + fn default() -> Self { + Self { + keys: 4, + objects_per_key: 2, + object_bytes: 4096, + } + } +} + +/// How the drill destroys the KMS persistence layer. +/// +/// Each variant is one row of the Local disaster matrix. All of them converge +/// on the same recovery procedure — quarantine what survived, restore into a +/// clean directory — which is exactly the point: the operator does not have to +/// diagnose the failure mode before acting. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DrillDisaster { + /// The whole key directory is gone (lost volume, wiped host). + KeyDirectoryLost, + /// The key records survive but the KDF salt is gone, so none of their + /// material can be unwrapped. + MasterKeySaltLost, + /// One key record is truncated in place (torn write, partial media + /// failure). + KeyRecordCorrupted, +} + +/// One drill phase, in execution order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DrillPhase { + /// Create the deployment and its master keys. + Seed, + /// Seal the sample objects that the restore will later have to make + /// readable again. + SealObjects, + /// Export the backup bundle. + Export, + /// Write past the snapshot fence, establishing the recovery point. + PostSnapshotWrite, + /// Destroy the persistence layer. + Disaster, + /// Move what survived aside so the restore target is clean. + Quarantine, + /// Zero-write restore preflight. + DryRun, + /// Execute the restore. + Restore, + /// Decrypt every pre-disaster sample object again. + Verify, + /// Re-run the restore against the recovered target. + RepeatRestore, +} + +impl DrillPhase { + /// Whether the phase is part of the recovery an operator waits on. + /// + /// Seeding, sealing, exporting and the disaster itself are drill + /// scaffolding, not recovery work, so they stay out of the recovery-time + /// total. + fn counts_towards_recovery_time(self) -> bool { + matches!( + self, + DrillPhase::Quarantine | DrillPhase::DryRun | DrillPhase::Restore | DrillPhase::Verify + ) + } +} + +/// Wall-clock cost of one phase. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DrillPhaseTiming { + /// Which phase. + pub phase: DrillPhase, + /// Duration in milliseconds. + pub millis: u64, +} + +/// Outcome of decrypting one pre-disaster sample object after the restore. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvelopeProbe { + /// Master key the object's data key was wrapped under. + pub key_id: String, + /// Bucket the object was sealed under. + pub bucket: String, + /// Object key. + pub object_key: String, + /// Digest of the plaintext as it was before the disaster. + pub plaintext_digest: ContentDigest, + /// Whether the restored deployment reproduced that plaintext exactly. + pub verified: bool, + /// Failure detail when it did not. Identifiers and error text only. + pub detail: Option, +} + +/// What the bundle was, and what it still was after the recovery. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DrillBundleEvidence { + /// Bundle identifier. + pub backup_id: String, + /// Snapshot generation the bundle belongs to. + pub snapshot_generation: u64, + /// Sealed manifest digest recorded at export time. + pub manifest_digest: ContentDigest, + /// Number of artifacts in the bundle. + pub artifact_count: usize, + /// Total bundle size on disk. + pub bundle_bytes: u64, + /// Manifest digest re-read after the whole recovery. + pub manifest_digest_after_recovery: ContentDigest, + /// Whether every byte of the bundle directory survived the recovery + /// unchanged. Restore must treat its source as strictly read-only. + pub source_unchanged: bool, +} + +/// What the disaster destroyed and how the recovery proceeded. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DrillRecoveryEvidence { + /// Top-level names the disaster removed or damaged. + pub destroyed_files: Vec, + /// Where the damaged directory was preserved. Nothing is ever destroyed + /// by the recovery itself: the pre-restore state stays available for + /// forensics and rollback. + pub quarantine_path: PathBuf, + /// Top-level names that were moved into quarantine. + pub quarantined_files: Vec, + /// The preflight report, verbatim. + pub dry_run: RestoreDryRunReport, + /// Whether the target was byte-for-byte identical before and after the + /// preflight. + pub dry_run_zero_write: bool, + /// The restore report, verbatim. Absent when the preflight refused the + /// recovery and the drill therefore never restored. + pub restore: Option, + /// Whether the cutover commit marker was gone once the restore returned. + /// A surviving marker means the target is still mid-cutover. + pub commit_marker_cleared: bool, + /// Whether re-running the same restore against the recovered target was + /// refused. + pub repeat_restore_refused: bool, + /// Whether that refused re-run left the recovered target untouched. + pub repeat_restore_left_target_unchanged: bool, +} + +/// The measured recovery point. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DrillRpoEvidence { + /// When the export sealed the bundle. + #[serde(with = "crate::time_serde::zoned")] + pub snapshot_completed_at: Zoned, + /// When the disaster struck. + #[serde(with = "crate::time_serde::zoned")] + pub disaster_at: Zoned, + /// The data-loss window: work committed inside it is not in the bundle. + pub rpo_window_millis: u64, + /// Keys created after the fence closed, which the bundle cannot contain. + pub post_snapshot_key_ids: Vec, + /// Objects sealed under those keys. + pub post_snapshot_objects: usize, + /// How many of them were readable again after the restore. Anything above + /// zero means the bundle was not a point-in-time snapshot. + pub post_snapshot_objects_recovered: usize, +} + +/// Overall drill outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DrillVerdict { + /// Every check held. + Passed, + /// At least one check did not; see the findings. + Failed, +} + +/// Machine-readable record of one drill run. +/// +/// Identifiers, digests, and durations only: no key material, no master key, +/// no plaintext, no ciphertext. It is meant to be archived next to the +/// incident record as the immutable evidence a drill actually happened and +/// what it proved. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DrillEvidence { + /// Evidence schema version. + pub format_version: u32, + /// Caller-chosen identifier of this run. + pub drill_id: String, + /// Backend the drill exercised. + pub backend: BackupBackendKind, + /// Deployment identity the rehearsal deployment carried. + pub deployment_identity: String, + /// RustFS version recorded in the bundle manifest. + pub rustfs_version: String, + /// Disaster that was simulated. + pub disaster: DrillDisaster, + /// Size of the rehearsed deployment. + pub dataset: DrillDataset, + /// When the run started. + #[serde(with = "crate::time_serde::zoned")] + pub started_at: Zoned, + /// Bundle evidence. + pub bundle: DrillBundleEvidence, + /// Recovery evidence. + pub recovery: DrillRecoveryEvidence, + /// Recovery-point evidence. + pub rpo: DrillRpoEvidence, + /// Per-phase durations. + pub timings: Vec, + /// Recovery time: the phases an operator waits on. + pub rto_millis: u64, + /// One entry per sample object re-read after the restore. + pub envelope_probes: Vec, + /// Every check that did not hold, in the order they were observed. + pub findings: Vec, + /// Whether the drill passed. + pub verdict: DrillVerdict, +} + +impl DrillEvidence { + /// Serialize the evidence for archival. + pub fn encode(&self) -> Result> { + serde_json::to_vec_pretty(self) + .map_err(|error| KmsError::internal_error(format!("drill evidence does not serialize: {error}"))) + } +} + +/// Parameters of one drill run. +pub struct DrillRequest { + /// Identifier of this run; also the prefix of every key the drill creates. + pub drill_id: String, + /// Scratch directory the drill owns entirely. Created if absent; must not + /// already contain a drill workspace. + pub workspace: PathBuf, + /// Deployment identity stamped into the bundle and required to match on + /// restore. + pub deployment_identity: String, + /// RustFS version string recorded in the manifest. + pub rustfs_version: String, + /// Snapshot generation for the exported bundle. + pub snapshot_generation: u64, + /// Size of the rehearsed deployment. + pub dataset: DrillDataset, + /// Disaster to simulate. + pub disaster: DrillDisaster, + /// At-rest master key of the rehearsal deployment. Supply a throwaway + /// value: the drill's own deployment is the only thing it protects. + pub master_key: String, + /// Unix permission bits for key files. + pub file_permissions: Option, +} + +impl std::fmt::Debug for DrillRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DrillRequest") + .field("drill_id", &self.drill_id) + .field("workspace", &self.workspace) + .field("deployment_identity", &self.deployment_identity) + .field("rustfs_version", &self.rustfs_version) + .field("snapshot_generation", &self.snapshot_generation) + .field("dataset", &self.dataset) + .field("disaster", &self.disaster) + .field("master_key", &"") + .field("file_permissions", &self.file_permissions) + .finish() + } +} + +impl DrillRequest { + fn validate(&self) -> Result<()> { + for (field, value) in [ + ("drill_id", &self.drill_id), + ("deployment_identity", &self.deployment_identity), + ("rustfs_version", &self.rustfs_version), + ("master_key", &self.master_key), + ] { + if value.is_empty() { + return Err(KmsError::validation_error(format!("drill {field} must not be empty"))); + } + } + // Key ids are derived from the drill id, so an id that cannot name a + // key file has to fail here rather than half way through seeding. + validate_key_id(&self.drill_id)?; + if !self.workspace.is_absolute() { + return Err(KmsError::validation_error("drill workspace must be an absolute path")); + } + if self.dataset.keys == 0 || self.dataset.objects_per_key == 0 || self.dataset.object_bytes == 0 { + return Err(KmsError::validation_error( + "drill dataset must contain at least one key, one object per key, and one byte per object", + )); + } + let objects = self + .dataset + .keys + .checked_mul(self.dataset.objects_per_key) + .ok_or_else(|| KmsError::validation_error("drill dataset size overflows"))?; + if objects > MAX_DRILL_OBJECTS { + return Err(KmsError::validation_error(format!( + "drill dataset of {objects} objects exceeds the {MAX_DRILL_OBJECTS} object limit" + ))); + } + Ok(()) + } + + fn key_id(&self, index: usize) -> String { + format!("{}-key-{index:04}", self.drill_id) + } + + fn post_snapshot_key_id(&self) -> String { + format!("{}-post-snapshot", self.drill_id) + } + + fn local_config(&self, key_dir: PathBuf) -> KmsConfig { + let mut config = KmsConfig::local(key_dir.clone()); + config.backend_config = BackendConfig::Local(LocalConfig { + key_dir, + master_key: Some(self.master_key.clone()), + file_permissions: self.file_permissions, + }); + // The drill re-opens the backend after the restore to prove the + // material on disk decrypts; a warm cache would prove nothing. + config.enable_cache = false; + // The rehearsal deployment is throwaway by construction: it is created + // inside the drill workspace, destroyed by the drill, and never + // registered as a server's KMS. The production-hygiene guard on where + // a key directory may live therefore does not apply to it — a drill + // workspace under the temp directory is the normal case, not a + // misconfiguration. The one guard that still matters, a non-empty + // at-rest master key, is enforced by `DrillRequest::validate`. + config.allow_insecure_dev_defaults = true; + config + } +} + +/// One object sealed before the disaster, kept so the restore can be asked to +/// make it readable again. +struct SealedObject { + key_id: String, + object_key: String, + plaintext_digest: ContentDigest, + ciphertext: Vec, + metadata: EncryptionMetadata, +} + +/// Collects per-phase wall-clock cost. +struct PhaseClock { + timings: Vec, + started: Instant, +} + +impl PhaseClock { + fn new() -> Self { + Self { + timings: Vec::new(), + started: Instant::now(), + } + } + + /// Close the current phase and open the next one. + fn finish(&mut self, phase: DrillPhase) { + let now = Instant::now(); + self.timings.push(DrillPhaseTiming { + phase, + millis: now.duration_since(self.started).as_millis().min(u128::from(u64::MAX)) as u64, + }); + self.started = now; + } + + fn recovery_time_millis(&self) -> u64 { + self.timings + .iter() + .filter(|timing| timing.phase.counts_towards_recovery_time()) + .map(|timing| timing.millis) + .sum() + } +} + +/// Run one disaster-recovery drill and return its evidence. +/// +/// Environment failures (an unusable workspace, a backend that will not start) +/// are errors. Verdicts about the recovery itself — a probe that did not +/// reproduce its plaintext, a preflight that wrote, a bundle the restore +/// modified — are findings inside the returned evidence, so a failed drill +/// still produces the record of *how* it failed. +pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result { + request.validate()?; + let started_at = Zoned::now(); + let mut clock = PhaseClock::new(); + let mut findings = Vec::new(); + + let live_dir = request.workspace.join(LIVE_DIR); + let bundle_dir = request.workspace.join(BUNDLE_DIR); + let quarantine_dir = request.workspace.join(QUARANTINE_DIR); + prepare_workspace(&request.workspace, &live_dir).await?; + + let config = request.local_config(live_dir.clone()); + + // The seeded deployment and its bundle. The backend is dropped before the + // disaster so no live handle outlives the state it points at. + let (sealed, rpo_sealed, manifest, snapshot_completed_at) = { + let backend = Arc::new(LocalKmsBackend::new(config.clone()).await?); + let service = ObjectEncryptionService::new(KmsManager::new(backend.clone(), config.clone())); + + let mut key_ids = Vec::with_capacity(request.dataset.keys); + for index in 0..request.dataset.keys { + let key_id = request.key_id(index); + create_drill_key(&service, &key_id).await?; + key_ids.push(key_id); + } + clock.finish(DrillPhase::Seed); + + let mut sealed = Vec::with_capacity(request.dataset.keys * request.dataset.objects_per_key); + for key_id in &key_ids { + for index in 0..request.dataset.objects_per_key { + sealed.push(seal_object(&service, key_id, index, request.dataset.object_bytes).await?); + } + } + clock.finish(DrillPhase::SealObjects); + + let client = backend + .local_backup_client() + .ok_or_else(|| KmsError::internal_error("local backend did not expose its backup client"))?; + let manifest = export_local_backup( + client, + kek, + &LocalBackupExportRequest { + backup_id: request.drill_id.clone(), + deployment_identity: request.deployment_identity.clone(), + rustfs_version: request.rustfs_version.clone(), + snapshot_generation: request.snapshot_generation, + destination: bundle_dir.clone(), + sanitized_config: None, + }, + ) + .await?; + let snapshot_completed_at = Zoned::now(); + clock.finish(DrillPhase::Export); + + // Past the fence: this key belongs to the window the bundle cannot + // cover, and the restore must not resurrect it. + let post_snapshot_key = request.post_snapshot_key_id(); + create_drill_key(&service, &post_snapshot_key).await?; + let mut rpo_sealed = Vec::with_capacity(request.dataset.objects_per_key); + for index in 0..request.dataset.objects_per_key { + rpo_sealed.push(seal_object(&service, &post_snapshot_key, index, request.dataset.object_bytes).await?); + } + clock.finish(DrillPhase::PostSnapshotWrite); + + (sealed, rpo_sealed, manifest, snapshot_completed_at) + }; + + let bundle_digest_before = tree_digest(&bundle_dir).await?; + + let destroyed_files = apply_disaster(&live_dir, request.disaster).await?; + let disaster_at = Zoned::now(); + clock.finish(DrillPhase::Disaster); + + // Recovery step one is always the same: preserve what survived, then + // restore into a clean directory. Nothing damaged is deleted. + let quarantined_files = quarantine(&live_dir, &quarantine_dir).await?; + clock.finish(DrillPhase::Quarantine); + + let restore_request = LocalRestoreRequest { + bundle_dir: bundle_dir.clone(), + target_key_dir: live_dir.clone(), + target_deployment_identity: request.deployment_identity.clone(), + observed_generation: None, + master_key: Some(request.master_key.clone()), + conflict_policy: RestoreConflictPolicy::RestoreIntoEmptyTarget, + file_permissions: request.file_permissions, + }; + + let target_before_dry_run = tree_digest(&live_dir).await?; + let dry_run = dry_run_local_restore(kek, &restore_request).await?; + let dry_run_zero_write = tree_digest(&live_dir).await? == target_before_dry_run; + clock.finish(DrillPhase::DryRun); + if !dry_run_zero_write { + findings.push("restore preflight modified the target; a dry-run must be zero-write".to_string()); + } + // A refused preflight ends the drill: proceeding anyway is exactly the + // behavior the restore contract forbids. The evidence is still produced, + // because *why* a rehearsed recovery was refused is the finding. + let permitted = dry_run.restore_permitted(); + if !permitted { + findings.push(format!( + "restore preflight refused the recovery: {} blocker(s), {} conflict(s)", + dry_run.blockers.len(), + dry_run.conflicts.len() + )); + } + + let mut restore_report = None; + let mut commit_marker_cleared = false; + let mut envelope_probes = Vec::new(); + let mut post_snapshot_objects_recovered = 0usize; + let mut repeat_restore_refused = false; + let mut repeat_restore_left_target_unchanged = false; + + if permitted { + let restore = restore_local_backup(kek, &restore_request).await?; + commit_marker_cleared = !fs::try_exists(live_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)).await?; + clock.finish(DrillPhase::Restore); + if !commit_marker_cleared { + findings.push("the cutover commit marker survived a completed restore".to_string()); + } + if restore.backup_id != manifest.backup_id || restore.snapshot_generation != manifest.snapshot_generation { + findings.push(format!( + "restore reported bundle '{}' at generation {}, the exported bundle was '{}' at generation {}", + restore.backup_id, restore.snapshot_generation, manifest.backup_id, manifest.snapshot_generation + )); + } + restore_report = Some(restore); + + // The acceptance criterion: a freshly opened backend over the restored + // directory must reproduce every pre-disaster plaintext. + { + let backend = Arc::new(LocalKmsBackend::new(config.clone()).await?); + let service = ObjectEncryptionService::new(KmsManager::new(backend, config.clone())); + for object in &sealed { + envelope_probes.push(probe_object(&service, object).await); + } + for object in &rpo_sealed { + if probe_object(&service, object).await.verified { + post_snapshot_objects_recovered += 1; + } + } + } + clock.finish(DrillPhase::Verify); + + for probe in &envelope_probes { + if !probe.verified { + findings.push(format!( + "object '{}' under key '{}' did not decrypt to its pre-disaster plaintext", + probe.object_key, probe.key_id + )); + } + } + if post_snapshot_objects_recovered != 0 { + findings.push(format!( + "{post_snapshot_objects_recovered} object(s) written after the snapshot fence were readable again; \ + the bundle is not a point-in-time snapshot" + )); + } + + // A second restore into the recovered target must be refused without + // touching it: repeating a drill, or an operator running the runbook + // twice, must never be able to damage a healthy deployment. + let target_before_repeat = tree_digest(&live_dir).await?; + repeat_restore_refused = restore_local_backup(kek, &restore_request).await.is_err(); + repeat_restore_left_target_unchanged = tree_digest(&live_dir).await? == target_before_repeat; + clock.finish(DrillPhase::RepeatRestore); + if !repeat_restore_refused { + findings.push("re-running the restore against the recovered target was accepted".to_string()); + } + if !repeat_restore_left_target_unchanged { + findings.push("the refused re-run modified the recovered target".to_string()); + } + } + + let manifest_after = read_local_bundle_manifest(&bundle_dir).await?; + let source_unchanged = tree_digest(&bundle_dir).await? == bundle_digest_before; + if !source_unchanged || manifest_after.manifest_digest != manifest.manifest_digest { + findings.push("the bundle changed during the recovery; a restore source must be read-only".to_string()); + } + + let verdict = if findings.is_empty() { + DrillVerdict::Passed + } else { + DrillVerdict::Failed + }; + + Ok(DrillEvidence { + format_version: DRILL_EVIDENCE_FORMAT_VERSION, + drill_id: request.drill_id.clone(), + backend: BackupBackendKind::Local, + deployment_identity: request.deployment_identity.clone(), + rustfs_version: request.rustfs_version.clone(), + disaster: request.disaster, + dataset: request.dataset, + started_at, + bundle: DrillBundleEvidence { + backup_id: manifest.backup_id.clone(), + snapshot_generation: manifest.snapshot_generation, + manifest_digest: manifest.manifest_digest.clone(), + artifact_count: manifest.artifacts.len(), + bundle_bytes: directory_bytes(&bundle_dir).await?, + manifest_digest_after_recovery: manifest_after.manifest_digest, + source_unchanged, + }, + recovery: DrillRecoveryEvidence { + destroyed_files, + quarantine_path: quarantine_dir, + quarantined_files, + dry_run, + dry_run_zero_write, + restore: restore_report, + commit_marker_cleared, + repeat_restore_refused, + repeat_restore_left_target_unchanged, + }, + rpo: DrillRpoEvidence { + rpo_window_millis: elapsed_millis(&snapshot_completed_at, &disaster_at), + snapshot_completed_at, + disaster_at, + post_snapshot_key_ids: vec![request.post_snapshot_key_id()], + post_snapshot_objects: rpo_sealed.len(), + post_snapshot_objects_recovered, + }, + rto_millis: clock.recovery_time_millis(), + timings: clock.timings, + envelope_probes, + findings, + verdict, + }) +} + +async fn prepare_workspace(workspace: &Path, live_dir: &Path) -> Result<()> { + fs::create_dir_all(workspace).await?; + if fs::try_exists(live_dir).await? { + return Err(KmsError::invalid_operation(format!( + "drill workspace {} already holds a rehearsal deployment; point the drill at a fresh directory", + workspace.display() + ))); + } + fs::create_dir_all(live_dir).await?; + Ok(()) +} + +async fn create_drill_key(service: &ObjectEncryptionService, key_id: &str) -> Result<()> { + service + .create_key(CreateKeyRequest { + key_name: Some(key_id.to_string()), + key_usage: KeyUsage::EncryptDecrypt, + description: Some("rustfs kms disaster-recovery drill".to_string()), + policy: None, + tags: HashMap::new(), + origin: Some("kms-dr-drill".to_string()), + }) + .await?; + Ok(()) +} + +/// Seal one sample object through the production encryption path and keep +/// exactly what a real deployment would keep: the ciphertext and its +/// encryption metadata. +async fn seal_object(service: &ObjectEncryptionService, key_id: &str, index: usize, bytes: usize) -> Result { + let object_key = format!("{key_id}/object-{index:04}"); + let plaintext = drill_payload(&object_key, bytes); + let plaintext_digest = ContentDigest::sha256_of(&plaintext); + let result = service + .encrypt_object( + DRILL_BUCKET, + &object_key, + Cursor::new(plaintext), + &EncryptionAlgorithm::Aes256, + Some(key_id), + None, + ) + .await?; + Ok(SealedObject { + key_id: key_id.to_string(), + object_key, + plaintext_digest, + ciphertext: result.ciphertext, + metadata: result.metadata, + }) +} + +/// Ask the restored deployment to unwrap one object's data key and reproduce +/// its plaintext. +async fn probe_object(service: &ObjectEncryptionService, object: &SealedObject) -> EnvelopeProbe { + let mut probe = EnvelopeProbe { + key_id: object.key_id.clone(), + bucket: DRILL_BUCKET.to_string(), + object_key: object.object_key.clone(), + plaintext_digest: object.plaintext_digest.clone(), + verified: false, + detail: None, + }; + + let decrypted = service + .decrypt_object(DRILL_BUCKET, &object.object_key, object.ciphertext.clone(), &object.metadata, None) + .await; + let mut reader = match decrypted { + Ok(reader) => reader, + Err(error) => { + probe.detail = Some(error.to_string()); + return probe; + } + }; + + let mut plaintext = Vec::new(); + if let Err(error) = reader.read_to_end(&mut plaintext).await { + probe.detail = Some(error.to_string()); + return probe; + } + if ContentDigest::sha256_of(&plaintext) == object.plaintext_digest { + probe.verified = true; + } else { + probe.detail = Some("decrypted plaintext does not match the pre-disaster digest".to_string()); + } + probe +} + +/// Deterministic, incompressible-enough sample payload derived from the object +/// key, so a drill never needs a fixture file and two runs of the same drill +/// seal the same bytes. +fn drill_payload(object_key: &str, bytes: usize) -> Vec { + let mut payload = Vec::with_capacity(bytes); + let mut block = Sha256::digest(object_key.as_bytes()); + while payload.len() < bytes { + let take = block.len().min(bytes - payload.len()); + payload.extend_from_slice(&block[..take]); + block = Sha256::digest(block); + } + payload +} + +/// Destroy the persistence layer and report what was destroyed. +async fn apply_disaster(live_dir: &Path, disaster: DrillDisaster) -> Result> { + match disaster { + DrillDisaster::KeyDirectoryLost => { + let names = top_level_names(live_dir).await?; + fs::remove_dir_all(live_dir).await?; + Ok(names) + } + DrillDisaster::MasterKeySaltLost => { + let salt = live_dir.join(LOCAL_KMS_MASTER_KEY_SALT_FILE); + if !fs::try_exists(&salt).await? { + return Err(KmsError::internal_error( + "drill deployment has no master-key salt to destroy; the seed phase did not produce one", + )); + } + fs::remove_file(&salt).await?; + Ok(vec![LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string()]) + } + DrillDisaster::KeyRecordCorrupted => { + let mut records: Vec = top_level_names(live_dir) + .await? + .into_iter() + .filter(|name| name.ends_with(".key")) + .collect(); + records.sort(); + let victim = records + .first() + .ok_or_else(|| KmsError::internal_error("drill deployment has no key record to corrupt"))?; + let path = live_dir.join(victim); + let mut content = fs::read(&path).await?; + content.truncate(content.len() / 2); + fs::write(&path, &content).await?; + Ok(vec![victim.clone()]) + } + } +} + +/// Move whatever survived the disaster aside, leaving a clean restore target. +/// +/// The damaged state is preserved, never deleted: a restore that turns out to +/// be the wrong decision must remain reversible, and forensics need the +/// original bytes. +async fn quarantine(live_dir: &Path, quarantine_dir: &Path) -> Result> { + if !fs::try_exists(live_dir).await? { + fs::create_dir_all(live_dir).await?; + return Ok(Vec::new()); + } + let names = top_level_names(live_dir).await?; + if names.is_empty() { + return Ok(Vec::new()); + } + fs::rename(live_dir, quarantine_dir).await?; + fs::create_dir_all(live_dir).await?; + Ok(names) +} + +async fn top_level_names(dir: &Path) -> Result> { + let mut names = Vec::new(); + let mut entries = match fs::read_dir(dir).await { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(names), + Err(error) => return Err(error.into()), + }; + while let Some(entry) = entries.next_entry().await? { + names.push(entry.file_name().to_string_lossy().into_owned()); + } + names.sort(); + Ok(names) +} + +async fn directory_bytes(dir: &Path) -> Result { + let dir = dir.to_path_buf(); + tokio::task::spawn_blocking(move || { + let mut total = 0u64; + for path in walk_files(&dir)? { + total = total.saturating_add(std::fs::metadata(&path)?.len()); + } + Ok(total) + }) + .await + .map_err(|error| KmsError::internal_error(format!("drill directory sizing failed: {error}")))? +} + +/// Canonical digest over a directory tree: relative path, length, content +/// digest, and modification time of every file. Equality across an operation +/// is proof that the operation performed no persistent write. +async fn tree_digest(root: &Path) -> Result { + let root = root.to_path_buf(); + tokio::task::spawn_blocking(move || { + let mut lines = Vec::new(); + for path in walk_files(&root)? { + let metadata = std::fs::metadata(&path)?; + let modified = metadata + .modified()? + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_nanos()); + let relative = path + .strip_prefix(&root) + .unwrap_or(path.as_path()) + .to_string_lossy() + .into_owned(); + let content = std::fs::read(&path)?; + lines.push(format!( + "{relative}\u{1f}{}\u{1f}{modified}\u{1f}{}", + metadata.len(), + hex::encode(Sha256::digest(&content)) + )); + } + lines.sort(); + Ok(ContentDigest::sha256_of(lines.join("\n").as_bytes())) + }) + .await + .map_err(|error| KmsError::internal_error(format!("drill tree digest failed: {error}")))? +} + +/// Every regular file under `root`, recursively. A missing root is an empty +/// tree, not an error: the drill digests directories the disaster may have +/// removed. +fn walk_files(root: &Path) -> std::io::Result> { + let mut files = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let entries = match std::fs::read_dir(&dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + for entry in entries { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + stack.push(path); + } else { + files.push(path); + } + } + } + Ok(files) +} + +fn elapsed_millis(from: &Zoned, to: &Zoned) -> u64 { + let nanos = to.timestamp().as_nanosecond() - from.timestamp().as_nanosecond(); + u64::try_from(nanos / 1_000_000).unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backends::local::durable_file::{CommitStep, failpoint}; + use crate::backends::local::is_orphan_commit_temp_name; + use crate::backup::capability::{AtRestProtection, BackupResponsibility}; + use crate::backup::local_export::{AEAD_NONCE_LEN, LOCAL_BUNDLE_MANIFEST_FILE, artifact_aad}; + use crate::backup::local_restore::abort_local_restore; + use crate::backup::manifest::{ + AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupManifest, CompletenessState, DigestAlgorithm, + VaultExternalReferences, VaultKvRecordReference, VaultTransitReference, + }; + use crate::backup::vault_restore::{ + VAULT_RECORD_ARTIFACT_DIR, VAULT_RECORD_ARTIFACT_SUFFIX, VaultRestoreClient, VaultRestoreRequest, VaultRestoreTarget, + dry_run_vault_restore, + }; + use crate::config::VaultAuthMethod; + use aes_gcm::{ + Nonce, + aead::{Aead, Payload}, + }; + use tempfile::TempDir; + + /// Deliberately distinctive so the evidence assertion below cannot pass by + /// accident. + const DRILL_MASTER_KEY: &str = "drill-master-key-never-in-evidence"; + const DEPLOYMENT: &str = "deployment-drill"; + + fn test_kek() -> BackupKek { + BackupKek::new("backup-kek-drill", 1, [0x37; 32]).expect("kek") + } + + fn drill_request(workspace: &Path, disaster: DrillDisaster) -> DrillRequest { + DrillRequest { + drill_id: "drill-0001".to_string(), + workspace: workspace.to_path_buf(), + deployment_identity: DEPLOYMENT.to_string(), + rustfs_version: "1.0.0-test".to_string(), + snapshot_generation: 11, + dataset: DrillDataset { + keys: 2, + objects_per_key: 2, + object_bytes: 1024, + }, + disaster, + master_key: DRILL_MASTER_KEY.to_string(), + file_permissions: Some(0o600), + } + } + + /// Open the restored directory and ask it to reproduce one object. + async fn probe_after_restore(config: &KmsConfig, object: &SealedObject) -> EnvelopeProbe { + let backend = Arc::new( + LocalKmsBackend::new(config.clone()) + .await + .expect("restored backend must open"), + ); + let service = ObjectEncryptionService::new(KmsManager::new(backend, config.clone())); + probe_object(&service, object).await + } + + /// Seed a deployment, seal one object, and export a bundle. + /// + /// The interrupted-cutover legs drive the restore themselves — they have + /// to crash it — so they cannot go through [`run_local_drill`], but they + /// use the same production path to produce the state they crash on. + async fn interrupted_cutover_fixture(work: &Path) -> (KmsConfig, LocalRestoreRequest, BackupManifest, SealedObject) { + let request = drill_request(work, DrillDisaster::KeyDirectoryLost); + let source_dir = work.join("source"); + fs::create_dir_all(&source_dir).await.expect("source dir"); + let source_config = request.local_config(source_dir); + let backend = Arc::new(LocalKmsBackend::new(source_config.clone()).await.expect("source backend")); + let service = ObjectEncryptionService::new(KmsManager::new(backend.clone(), source_config.clone())); + + let key_id = request.key_id(0); + create_drill_key(&service, &key_id).await.expect("create key"); + let sealed = seal_object(&service, &key_id, 0, 512).await.expect("seal object"); + + let bundle_dir = work.join("bundle"); + let manifest = export_local_backup( + backend.local_backup_client().expect("local backup client"), + &test_kek(), + &LocalBackupExportRequest { + backup_id: request.drill_id.clone(), + deployment_identity: DEPLOYMENT.to_string(), + rustfs_version: request.rustfs_version.clone(), + snapshot_generation: request.snapshot_generation, + destination: bundle_dir.clone(), + sanitized_config: None, + }, + ) + .await + .expect("export"); + + let target = work.join("target"); + let restore_request = LocalRestoreRequest { + bundle_dir, + target_key_dir: target.clone(), + target_deployment_identity: DEPLOYMENT.to_string(), + observed_generation: None, + master_key: Some(DRILL_MASTER_KEY.to_string()), + conflict_policy: RestoreConflictPolicy::RestoreIntoEmptyTarget, + file_permissions: Some(0o600), + }; + (request.local_config(target), restore_request, manifest, sealed) + } + + /// Crash a restore precisely at its commit point: arming the failpoint on + /// the marker path itself leaves the staging commits untouched, so what + /// survives is a fully staged restore with a published marker and no + /// cutover — the state a power loss produces between the two. + async fn crash_at_commit_point(restore_request: &LocalRestoreRequest) { + let marker_path = restore_request.target_key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + failpoint::arm(&marker_path, CommitStep::DirSynced); + let error = restore_local_backup(&test_kek(), restore_request) + .await + .expect_err("the armed commit point must simulate a crash"); + failpoint::disarm(&marker_path); + assert!(error.to_string().contains("injected crash"), "got {error}"); + assert!(marker_path.exists(), "the commit marker must be published before the crash"); + } + + #[tokio::test] + async fn every_local_disaster_recovers_and_replays_historical_object_deks() { + for disaster in [ + DrillDisaster::KeyDirectoryLost, + DrillDisaster::MasterKeySaltLost, + DrillDisaster::KeyRecordCorrupted, + ] { + let work = TempDir::new().expect("work dir"); + let request = drill_request(work.path(), disaster); + let evidence = run_local_drill(&test_kek(), &request) + .await + .unwrap_or_else(|error| panic!("{disaster:?}: drill failed to run: {error}")); + + assert_eq!(evidence.verdict, DrillVerdict::Passed, "{disaster:?}: {:?}", evidence.findings); + assert!(evidence.findings.is_empty(), "{disaster:?}: {:?}", evidence.findings); + + // The acceptance criterion: every object sealed before the + // disaster decrypts again. + assert_eq!(evidence.envelope_probes.len(), 4, "{disaster:?}"); + assert!(evidence.envelope_probes.iter().all(|probe| probe.verified), "{disaster:?}"); + + // ... and the recovery point holds: work past the fence stays lost. + assert_eq!(evidence.rpo.post_snapshot_objects, 2, "{disaster:?}"); + assert_eq!(evidence.rpo.post_snapshot_objects_recovered, 0, "{disaster:?}"); + assert_eq!(evidence.rpo.post_snapshot_key_ids, vec![request.post_snapshot_key_id()], "{disaster:?}"); + + assert!(evidence.recovery.dry_run_zero_write, "{disaster:?}"); + assert!(evidence.recovery.dry_run.restore_permitted(), "{disaster:?}"); + let restore = evidence.recovery.restore.as_ref().expect("restore report"); + assert_eq!(restore.restored_key_ids.len(), 2, "{disaster:?}: the post-fence key is not in the bundle"); + assert!(restore.salt_restored, "{disaster:?}"); + assert!(!restore.resumed, "{disaster:?}"); + assert!(evidence.recovery.commit_marker_cleared, "{disaster:?}"); + assert!(evidence.recovery.repeat_restore_refused, "{disaster:?}"); + assert!(evidence.recovery.repeat_restore_left_target_unchanged, "{disaster:?}"); + + assert!(evidence.bundle.source_unchanged, "{disaster:?}"); + assert_eq!( + evidence.bundle.manifest_digest, evidence.bundle.manifest_digest_after_recovery, + "{disaster:?}" + ); + assert_eq!(evidence.bundle.snapshot_generation, request.snapshot_generation, "{disaster:?}"); + + assert!(!evidence.recovery.destroyed_files.is_empty(), "{disaster:?}"); + assert_eq!( + evidence.timings.iter().map(|timing| timing.phase).collect::>(), + vec![ + DrillPhase::Seed, + DrillPhase::SealObjects, + DrillPhase::Export, + DrillPhase::PostSnapshotWrite, + DrillPhase::Disaster, + DrillPhase::Quarantine, + DrillPhase::DryRun, + DrillPhase::Restore, + DrillPhase::Verify, + DrillPhase::RepeatRestore, + ], + "{disaster:?}" + ); + } + } + + /// Whatever survives a disaster is preserved, never deleted: a restore + /// that turns out to be the wrong call has to stay reversible. + #[tokio::test] + async fn the_damaged_key_directory_is_preserved_for_rollback() { + let work = TempDir::new().expect("work dir"); + let request = drill_request(work.path(), DrillDisaster::KeyRecordCorrupted); + let evidence = run_local_drill(&test_kek(), &request).await.expect("drill"); + + assert_eq!(evidence.recovery.destroyed_files, vec![format!("{}.key", request.key_id(0))]); + let quarantined = &evidence.recovery.quarantined_files; + assert!(quarantined.contains(&LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string()), "got {quarantined:?}"); + assert!(quarantined.contains(&format!("{}.key", request.key_id(1))), "got {quarantined:?}"); + assert_eq!( + top_level_names(&evidence.recovery.quarantine_path).await.expect("quarantine"), + *quarantined, + "the quarantined directory must still hold exactly what was moved into it" + ); + } + + #[tokio::test] + async fn drill_evidence_is_archivable_and_names_no_secret() { + let work = TempDir::new().expect("work dir"); + let request = drill_request(work.path(), DrillDisaster::KeyDirectoryLost); + let evidence = run_local_drill(&test_kek(), &request).await.expect("drill"); + + let encoded = evidence.encode().expect("evidence must serialize"); + let text = String::from_utf8(encoded.clone()).expect("evidence is utf-8"); + assert!(!text.contains(DRILL_MASTER_KEY), "the evidence must not carry the master key"); + assert!( + !text.contains(&hex::encode([0x37u8; 32])), + "the evidence must not carry backup KEK material" + ); + + let decoded: DrillEvidence = serde_json::from_slice(&encoded).expect("evidence must round-trip"); + assert_eq!(decoded.format_version, DRILL_EVIDENCE_FORMAT_VERSION); + assert_eq!(decoded.verdict, evidence.verdict); + assert_eq!(decoded.bundle, evidence.bundle); + assert_eq!(decoded.envelope_probes, evidence.envelope_probes); + assert_eq!(decoded.recovery.restore, evidence.recovery.restore); + } + + #[tokio::test] + async fn an_interrupted_cutover_leaves_a_marker_bound_to_its_bundle_and_rolls_forward() { + let work = TempDir::new().expect("work dir"); + let (target_config, restore_request, manifest, sealed) = interrupted_cutover_fixture(work.path()).await; + let target = restore_request.target_key_dir.clone(); + crash_at_commit_point(&restore_request).await; + + // The marker is read as the on-disk artifact an operator would find, + // not through the type that wrote it: it must name this exact bundle + // and exactly the files the cutover still owes. + let marker_bytes = fs::read(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)) + .await + .expect("commit marker"); + let marker: serde_json::Value = serde_json::from_slice(&marker_bytes).expect("marker json"); + assert_eq!(marker["backup_id"], serde_json::json!(manifest.backup_id)); + assert_eq!(marker["manifest_digest"]["hex"], serde_json::json!(manifest.manifest_digest.hex)); + let files: Vec = serde_json::from_value(marker["files"].clone()).expect("marker file list"); + assert_eq!(files, vec![LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string(), format!("{}.key", sealed.key_id)]); + + // A directory mid-cutover must not serve requests. + let error = LocalKmsBackend::new(target_config.clone()) + .await + .err() + .expect("startup must fail closed mid-cutover"); + assert!(error.to_string().contains("unfinished restore"), "got {error}"); + + let report = restore_local_backup(&test_kek(), &restore_request) + .await + .expect("roll forward"); + assert!(report.resumed, "re-running the same bundle must roll the cutover forward"); + assert!(!target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE).exists()); + assert!(probe_after_restore(&target_config, &sealed).await.verified); + } + + #[tokio::test] + async fn an_interrupted_cutover_can_be_aborted_back_to_the_pre_restore_state() { + let work = TempDir::new().expect("work dir"); + let (target_config, restore_request, _manifest, sealed) = interrupted_cutover_fixture(work.path()).await; + let target = restore_request.target_key_dir.clone(); + crash_at_commit_point(&restore_request).await; + + abort_local_restore(&target).await.expect("abort must succeed"); + // Only the commit temp of the crashed marker write may survive: it was + // never published, carries no state, and is exactly what the backend's + // startup recovery and a fresh restore both classify as removable. + let leftovers = top_level_names(&target).await.expect("target names"); + assert!( + leftovers.iter().all(|name| is_orphan_commit_temp_name(name)), + "abort must take back the marker, the staged state, and anything already published; left {leftovers:?}" + ); + + // The rolled-back target is a clean slate again, so the drill is + // repeatable rather than a one-shot. + let report = restore_local_backup(&test_kek(), &restore_request) + .await + .expect("a fresh restore after an abort must succeed"); + assert!(!report.resumed); + assert!(probe_after_restore(&target_config, &sealed).await.verified); + } + + #[tokio::test] + async fn a_drill_refuses_a_workspace_that_already_holds_a_rehearsal() { + let work = TempDir::new().expect("work dir"); + let request = drill_request(work.path(), DrillDisaster::KeyDirectoryLost); + run_local_drill(&test_kek(), &request).await.expect("first drill"); + + let error = run_local_drill(&test_kek(), &request) + .await + .expect_err("a used workspace must be refused"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + } + + #[tokio::test] + async fn drill_requests_are_validated_before_anything_is_created() { + let work = TempDir::new().expect("work dir"); + + let mut request = drill_request(work.path(), DrillDisaster::KeyDirectoryLost); + request.master_key = String::new(); + assert!(run_local_drill(&test_kek(), &request).await.is_err()); + + let mut request = drill_request(work.path(), DrillDisaster::KeyDirectoryLost); + request.drill_id = "../escape".to_string(); + assert!(run_local_drill(&test_kek(), &request).await.is_err()); + + let mut request = drill_request(work.path(), DrillDisaster::KeyDirectoryLost); + request.dataset.keys = MAX_DRILL_OBJECTS; + request.dataset.objects_per_key = 2; + assert!(run_local_drill(&test_kek(), &request).await.is_err()); + + assert!( + !work.path().join(LIVE_DIR).exists(), + "a rejected request must not have created a deployment" + ); + } + + #[test] + fn drill_payloads_are_deterministic_and_sized() { + for bytes in [1usize, 31, 32, 33, 4096] { + let payload = drill_payload("bucket/object", bytes); + assert_eq!(payload.len(), bytes); + assert_eq!(payload, drill_payload("bucket/object", bytes)); + } + assert_ne!(drill_payload("a", 64), drill_payload("b", 64)); + } + + /// Seal a minimal Vault bundle: one KV metadata record plus the external + /// references that name the Transit trust root an operator has to restore + /// natively. + async fn vault_drill_bundle(dir: &Path, transit_key: &str, cluster_id: &str, kv_mount: &str, kv_prefix: &str) -> PathBuf { + const BACKUP_ID: &str = "drill-vault-0001"; + const SNAPSHOT_GENERATION: u64 = 11; + let key_id = "drill-vault-key"; + let bundle = dir.join("vault-bundle"); + fs::create_dir_all(bundle.join(VAULT_RECORD_ARTIFACT_DIR)) + .await + .expect("artifact dir"); + + let kek = test_kek(); + let path = format!("{VAULT_RECORD_ARTIFACT_DIR}/{key_id}{VAULT_RECORD_ARTIFACT_SUFFIX}"); + let plaintext = serde_json::to_vec(&serde_json::json!({ "key_state": "Enabled", "current_version": 1 })).expect("record"); + let nonce = [0x21u8; AEAD_NONCE_LEN]; + let ciphertext = kek + .cipher() + .encrypt( + &Nonce::from(nonce), + Payload { + msg: &plaintext, + aad: &artifact_aad(BACKUP_ID, SNAPSHOT_GENERATION, &path), + }, + ) + .expect("seal artifact"); + let mut payload = nonce.to_vec(); + payload.extend_from_slice(&ciphertext); + fs::write(bundle.join(&path), &payload).await.expect("write artifact"); + + let manifest = BackupManifest { + format_version: BackupManifest::FORMAT_VERSION, + backup_id: BACKUP_ID.to_string(), + created_at: "2026-08-01T00:00:00+00:00[UTC]".parse().expect("timestamp"), + rustfs_version: "1.0.0-test".to_string(), + deployment_identity: DEPLOYMENT.to_string(), + backend: BackupBackendKind::VaultTransit, + at_rest_protection: AtRestProtection::ExternalNonExportable, + responsibility: BackupResponsibility::MetadataPlusExternalRoot, + snapshot_generation: SNAPSHOT_GENERATION, + backup_kek: kek.descriptor(), + artifacts: vec![ArtifactDescriptor { + kind: ArtifactKind::KeyMetadata, + path, + len: payload.len() as u64, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + encrypted_digest: ContentDigest::sha256_of(&payload), + }], + local_kdf: None, + external_references: Some(VaultExternalReferences { + cluster_id: cluster_id.to_string(), + namespace: None, + kv_mount: kv_mount.to_string(), + kv_path_prefix: kv_prefix.to_string(), + transit: Some(VaultTransitReference { + mount_path: "transit".to_string(), + key_name: transit_key.to_string(), + required_min_version: 1, + current_version: 1, + native_snapshot_reference: None, + }), + kv_records: vec![VaultKvRecordReference { + key_id: key_id.to_string(), + kv_version: 1, + }], + }), + key_versions: None, + capability_discovery: None, + completeness: CompletenessState::InProgress, + manifest_digest: ContentDigest { + algorithm: DigestAlgorithm::Sha256, + hex: String::new(), + }, + }; + let sealed = manifest.seal().expect("seal manifest"); + fs::write(bundle.join(LOCAL_BUNDLE_MANIFEST_FILE), sealed.encode().expect("encode manifest")) + .await + .expect("write manifest"); + bundle + } + + /// Vault leg of the drill matrix, against a real server. + /// + /// Transit keys are non-exportable, so a Vault bundle never carries the + /// cryptographic root: restoring it is a Vault-native operator step, and + /// what RustFS owes is a refusal to proceed until that step has actually + /// happened. This drills exactly that refusal — the bundle names a Transit + /// key the recovered Vault does not have, and the preflight must report it + /// instead of publishing anything. + /// + /// Prerequisites: a Vault with the transit engine enabled + /// (`vault server -dev`, then `vault secrets enable transit`). Address and + /// token come from `RUSTFS_KMS_VAULT_ADDR` (default + /// `http://127.0.0.1:8200`) and `RUSTFS_KMS_VAULT_TOKEN` (default `root`). + #[tokio::test] + #[ignore = "needs a Vault with the transit engine enabled; see RUSTFS_KMS_VAULT_ADDR"] + async fn a_vault_bundle_is_refused_until_its_transit_trust_root_is_restored() { + let address = std::env::var("RUSTFS_KMS_VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()); + let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "root".to_string()); + let cluster_id = "vault-drill-cluster"; + let kv_mount = "secret"; + let kv_prefix = "rustfs/kms/drill"; + // A name no operator would have restored, so the trust root is + // provably absent whatever else the server holds. + let transit_key = format!("rustfs-kms-drill-absent-{}", uuid::Uuid::new_v4()); + + let work = TempDir::new().expect("work dir"); + let bundle = vault_drill_bundle(work.path(), &transit_key, cluster_id, kv_mount, kv_prefix).await; + + let target = VaultRestoreTarget { + address, + auth_method: VaultAuthMethod::Token { token }, + namespace: None, + cluster_id: cluster_id.to_string(), + kv_mount: kv_mount.to_string(), + kv_path_prefix: kv_prefix.to_string(), + transit_mount: Some("transit".to_string()), + }; + let client = VaultRestoreClient::connect(&target, &KmsConfig::default()) + .await + .expect("vault restore client"); + let report = dry_run_vault_restore( + &test_kek(), + &client, + &VaultRestoreRequest { + bundle_dir: bundle, + target, + target_deployment_identity: DEPLOYMENT.to_string(), + observed_generation: None, + conflict_policy: RestoreConflictPolicy::RestoreIntoEmptyTarget, + }, + ) + .await + .expect("preflight must produce a report, not an error"); + + assert!(!report.restore_permitted(), "got {report:?}"); + assert!( + report + .external_mismatches + .iter() + .any(|mismatch| mismatch.observed.contains(&transit_key) || mismatch.expected.contains(&transit_key)), + "the missing Transit trust root must be named in the report: {report:?}" + ); + } +} diff --git a/crates/kms/src/backup/mod.rs b/crates/kms/src/backup/mod.rs index 36ace38d7..4263686e0 100644 --- a/crates/kms/src/backup/mod.rs +++ b/crates/kms/src/backup/mod.rs @@ -21,7 +21,8 @@ //! [`vault_restore`] orchestrates the consumer side for the Vault backends, //! whose cryptographic root is restored by Vault's own disaster-recovery //! flow. All are crate-internal APIs; the admin API builds on these pieces in -//! follow-up changes. +//! follow-up changes. [`drill`] rehearses the whole loop end to end and turns +//! it into machine-readable evidence. //! //! # Bundle model //! @@ -51,6 +52,7 @@ //! format version. mod capability; +pub mod drill; mod dry_run; mod error; pub mod local_export; @@ -59,6 +61,10 @@ mod manifest; pub mod vault_restore; pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; +pub use drill::{ + DRILL_EVIDENCE_FORMAT_VERSION, DrillBundleEvidence, DrillDataset, DrillDisaster, DrillEvidence, DrillPhase, DrillPhaseTiming, + DrillRecoveryEvidence, DrillRequest, DrillRpoEvidence, DrillVerdict, EnvelopeProbe, run_local_drill, +}; pub use dry_run::{ ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, }; diff --git a/crates/kms/src/backup/vault_restore.rs b/crates/kms/src/backup/vault_restore.rs index 7cd3f4dbf..3986ad640 100644 --- a/crates/kms/src/backup/vault_restore.rs +++ b/crates/kms/src/backup/vault_restore.rs @@ -114,10 +114,13 @@ use vaultrs::api::transit::responses::ReadKeyData; use vaultrs::error::ClientError; use vaultrs::{kv2, transit::key}; +// The bundle layout names are pub(crate) so the drill harness +// (`crate::backup::drill`) addresses the exact paths a Vault bundle uses +// instead of copies that could drift. /// Bundle-relative directory holding one KV record artifact per key. -const VAULT_RECORD_ARTIFACT_DIR: &str = "vault/records"; +pub(crate) const VAULT_RECORD_ARTIFACT_DIR: &str = "vault/records"; /// Bundle-relative suffix of a KV record artifact. -const VAULT_RECORD_ARTIFACT_SUFFIX: &str = ".json.enc"; +pub(crate) const VAULT_RECORD_ARTIFACT_SUFFIX: &str = ".json.enc"; /// KV path segment (under the bundle's path prefix) of the restore commit /// marker. The leading dot keeps it out of the key id space the backends /// address; a bundle naming a key id equal to it is rejected. diff --git a/docs/operations/kms-disaster-recovery-drill.md b/docs/operations/kms-disaster-recovery-drill.md new file mode 100644 index 000000000..9dc3a77e3 --- /dev/null +++ b/docs/operations/kms-disaster-recovery-drill.md @@ -0,0 +1,113 @@ +# KMS disaster-recovery drill + +A KMS backup that has never been restored is a hypothesis. This runbook turns it into evidence: it rehearses the complete loop — back up, lose the persistence layer, preflight, restore, and read historical objects again — and files a machine-readable evidence bundle for each run. For what each backend's backup actually covers, see [KMS backend security properties](kms-backend-security.md); for the metrics and alerts around KMS operations, see the [KMS observability runbook](kms-observability-runbook.md). + +The acceptance criterion of a drill is not that files came back. It is that objects encrypted before the disaster decrypt after the restore. The harness keeps the ciphertext and encryption metadata of every object it sealed before the disaster and, once the restore is complete, decrypts each one through a freshly opened backend and compares against the pre-disaster digest. Anything less proves only that a bundle is well formed. + +## Scope + +The drill covers the **Local** backend, which is the only backend RustFS produces a full-material bundle for. The responsibility split is deliberate and is described in `crates/kms/src/backup/capability.rs`: + +| Backend | What a RustFS bundle carries | What restores it | +| --- | --- | --- | +| Local | Key records, all stored versions, the KDF salt, sanitized configuration | The RustFS restore in this runbook | +| Static | Non-sensitive references only | The operator re-supplies the secret out of band | +| Vault KV2 + Transit | KV metadata and Transit ciphertext references | Vault's native snapshot restore, then the RustFS orchestration | +| Vault Transit | Metadata, configuration references, verification data | Vault's native snapshot restore, then the RustFS orchestration | + +For the Vault backends there is no RustFS-side export, so there is no loop for a drill to close end to end: the cryptographic root is non-exportable and comes back through Vault's own disaster-recovery flow. What RustFS owns there is the refusal to proceed before that has happened, plus the ordering of everything after it. Rehearse it with the Vault section below. + +## What the drill measures + +**Recovery point (RPO).** The harness creates one key *after* the export fence closed and seals objects under it. After the restore that key is absent and its objects do not decrypt — as they must not. The recovery point of a KMS backup is therefore the snapshot generation of the last bundle, not the moment of the disaster, and `rpo.rpo_window_millis` in the evidence is the width of that window in the rehearsal. In production the same window is the age of your newest bundle, which is what your backup schedule sets. + +**Recovery time (RTO).** `rto_millis` is the sum of the phases an operator waits on: quarantine, preflight, restore, and verification. Seeding, sealing and the disaster itself are drill scaffolding and are excluded. Human decision time is excluded too and belongs to your incident process, not to this number. Per-phase costs are in `timings`. + +An RTO figure only transfers to production if the rehearsed deployment is comparable, so size `RUSTFS_KMS_DRILL_KEYS` to the number of master keys the real deployment holds. Both numbers are recorded in the evidence (`dataset`), so a stale measurement is visibly stale. + +## Running a drill + +```bash +export RUSTFS_KMS_DRILL_WORKSPACE=/var/lib/rustfs/dr-drill/$(date -u +%Y%m%dT%H%M%SZ) +export RUSTFS_KMS_DRILL_MASTER_KEY='' +export RUSTFS_KMS_BACKUP_KEK='' +export RUSTFS_KMS_BACKUP_KEK_ID='' +export RUSTFS_KMS_DRILL_KEYS=64 + +cargo run -q -p rustfs-kms --example kms_dr_drill +``` + +The workspace must be an absolute path that does not already hold a rehearsal; give each run its own directory so the evidence and the quarantined state stay side by side. The runner exits 0 only when every check held, so a scheduled drill fails its job on a bad result instead of quietly filing a bad report. + +`RUSTFS_KMS_BACKUP_KEK` and `RUSTFS_KMS_BACKUP_KEK_ID` are the same variables the admin backup API reads. Use the KEK real bundles are sealed under: retrieving it is the one part of a recovery no bundle can attest to, and a drill that invents its own KEK does not test it. The rehearsal deployment's at-rest master key is separate and throwaway — the drill creates and destroys that deployment itself and never touches the running KMS. + +Optional variables: `RUSTFS_KMS_DRILL_DISASTER` (see below), `RUSTFS_KMS_DRILL_ID`, `RUSTFS_KMS_DRILL_DEPLOYMENT`, `RUSTFS_KMS_DRILL_OBJECTS_PER_KEY`, `RUSTFS_KMS_DRILL_OBJECT_BYTES`, `RUSTFS_KMS_DRILL_FILE_PERMISSIONS`, `RUSTFS_KMS_BACKUP_KEK_VERSION`, and `RUSTFS_KMS_DRILL_EVIDENCE` (evidence path, default `/evidence.json`). + +## Disaster matrix + +Run all three; they exercise different failure surfaces and converge on the same procedure, which is the point — an operator does not have to diagnose the failure mode before acting. + +| `RUSTFS_KMS_DRILL_DISASTER` | Simulates | +| --- | --- | +| `key-directory-lost` (default) | The whole key directory is gone: lost volume, wiped host | +| `master-key-salt-lost` | Records survive but the KDF salt is gone, so no material unwraps | +| `key-record-corrupted` | One key record is truncated in place: torn write, partial media failure | + +Bundle-side faults — tampered, truncated, wrong-KEK, incomplete, unknown-version — are not drill scenarios. They are covered exhaustively and deterministically by the unit tests of `crates/kms/src/backup/local_export.rs` and `crates/kms/src/backup/local_restore.rs`, and re-running them as a drill would add fixtures without adding evidence. + +## Recovery procedure + +The drill executes exactly this sequence; running it by hand against a real deployment is the same procedure with your own paths. + +1. **Stop the KMS.** A restore must never race a running backend. +2. **Quarantine, do not delete.** Move the damaged key directory aside rather than clearing it. The drill records the quarantine path and its contents in the evidence, and nothing damaged is ever destroyed: a restore that turns out to be the wrong call has to stay reversible, and forensics need the original bytes. Restore refuses a non-empty target anyway, including one holding only an orphan salt. +3. **Preflight.** A dry-run decodes the whole bundle, checks the KDF descriptor against this build, verifies the operator-supplied master key against the bundle's one-way verifier, and enumerates conflicts. It writes nothing; the drill proves that by digesting the target before and after. Read every blocker before going further. +4. **Restore.** Artifacts are staged inside the target, decryption-probed, then published through a commit marker and an atomic cutover. +5. **Verify.** Open the restored directory and decrypt historical objects. This is the acceptance criterion, not step 4. +6. **Observation period.** Keep the quarantined directory and the bundle until you have observed the recovered deployment serving reads. Nothing in the restore path deletes old material for you. + +## Reading the evidence + +`evidence.json` is `DrillEvidence` (`format_version` 1). It carries identifiers, digests and durations only: no key material, no master key, no plaintext, no ciphertext. Archive it next to the incident or audit record. + +| Field | Why it matters | +| --- | --- | +| `verdict`, `findings` | The result and every check that did not hold | +| `envelope_probes[].verified` | Per-object proof that a historical data key still unwraps | +| `rpo.post_snapshot_objects_recovered` | Must be `0`; anything else means the bundle was not a point-in-time snapshot | +| `bundle.manifest_digest` vs `manifest_digest_after_recovery`, `bundle.source_unchanged` | The restore treated its bundle as strictly read-only | +| `recovery.dry_run_zero_write` | The preflight wrote nothing to the target | +| `recovery.commit_marker_cleared` | The cutover completed rather than leaving the target mid-restore | +| `recovery.repeat_restore_refused` and `..._left_target_unchanged` | Re-running the procedure cannot damage a healthy deployment | +| `dataset`, `timings`, `rto_millis`, `rpo.rpo_window_millis` | The measurement, and the deployment size it is valid for | + +## Interruptions and re-entry + +A restore has exactly one commit point: the durably published `.restore-commit.json` marker. Before it, the target's top level is untouched and re-running starts over. With it published, the backend refuses to start — a key directory mid-cutover must not serve requests — and you have two ways out: + +- **Roll forward**: re-run the restore with the same bundle. The marker is bound to the bundle by backup id and manifest digest, so a different bundle is refused rather than merged. +- **Roll back**: abort the restore, which takes back exactly the files the marker names, then the marker, then the staged state. The target returns to its pre-restore state and a fresh restore can start. + +Both paths are exercised by the drill's interrupted-cutover tests in `crates/kms/src/backup/drill.rs`, which crash a restore precisely at its commit point rather than simulating the resulting state. Every interruption converges on the complete old state or the complete new state; there is no half-activated outcome. + +## Vault backends + +There is no RustFS-side Vault export, so a Vault drill is not the same closed loop. Rehearse it as: + +1. Restore the Vault cluster from its own native snapshot, following your Vault runbook. Transit keys are non-exportable and RustFS never attempts to import them. +2. Run the RustFS preflight against the recovered cluster. It verifies cluster, namespace, mounts, the Transit key's identity and its version window, and the KV generation, and refuses on the first mismatch. A Transit key that was not restored, a `min_decryption_version` raised above what the bundle still needs, or a version history restored below the bundle's snapshot point are each reported as a distinct mismatch. +3. Only then let the restore publish KV records, in the order material and versions, then metadata, then configuration and cutover. + +`crates/kms/src/backup/drill.rs` carries an `#[ignore]`d leg for step 2 that drills the refusal against a real server. It needs a Vault with the transit engine enabled and reads `RUSTFS_KMS_VAULT_ADDR` and `RUSTFS_KMS_VAULT_TOKEN`: + +```bash +cargo test -p rustfs-kms --lib backup::drill -- --ignored --nocapture +``` + +## Running the drill matrix as tests + +```bash +cargo test -p rustfs-kms --lib backup::drill +``` + +This runs the offline matrix — all three disasters, the evidence contract, and both interrupted-cutover outcomes — with no external dependencies. Use it in CI to keep the harness itself honest; use the operator entry point above to produce evidence for a real deployment size.