// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Local backend restore: consume a sealed backup bundle into a key directory. //! //! The consumer side of [`crate::backup::local_export`]. The admin API is not //! wired here — callers construct the request and supply the backup KEK and //! the operator-re-supplied master key explicitly. The bundle source is //! strictly read-only throughout: restore never writes to or deletes from it. //! //! # Four-phase protocol //! //! 1. **Dry-run (zero writes)** — [`dry_run_local_restore`] decodes the full //! bundle chain in memory, checks the KDF descriptor against this build, //! enumerates target conflicts, and verifies the operator-supplied master //! key against the manifest's one-way verifier. It writes nothing. //! 2. **Staging** — decrypted artifacts are committed durably into the //! `.restore-staging/` subdirectory of the target. The leading dot and the //! directory shape keep it invisible to the backend's key scan and its //! orphan-temp matcher, so startup recovery can never delete staged state. //! Every record is decryption-probed with the derived master key before //! and after staging. //! 3. **Commit marker + cutover** — the durably published //! `.restore-commit.json` marker is the single commit point. Cutover //! publishes each staged file into the target top level via the no-clobber //! [`durable_file::link_durably`] primitive (salt first, keys after), then //! durably removes the marker and drops the staging directory. //! 4. **Crash re-entry** — before the marker, the target top level is //! untouched and a re-run starts over; with the marker published, backend //! startup fails closed and re-running the restore with the same bundle //! rolls the cutover forward, while [`abort_local_restore`] rolls it back. //! Every interruption converges to the complete old or complete new state. //! //! # Deliberately out of scope //! //! The `explicit-remap` conflict policy is not implemented: remapping stable //! key ids requires proving that object envelopes migrate in lockstep, and a //! bulk rekey is a non-goal of this change. Local has no rotation history //! (backlog#1565), so per-key version and deletion-state regression cannot //! become write operations under the empty-target policy; the snapshot //! generation check freezes the injected observe-and-reject-strictly-lower //! semantics for the admin layer. use crate::backends::local::{ LOCAL_KMS_MASTER_KEY_SALT_FILE, LOCAL_KMS_MASTER_KEY_SALT_LEN, LOCAL_RESTORE_COMMIT_MARKER_FILE, LocalKmsClient, StoredKeyProtection, durable_file, is_orphan_commit_temp_name, validate_key_id, }; use crate::backup::capability::AtRestProtection; use crate::backup::dry_run::{ ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, }; use crate::backup::error::BackupError; use crate::backup::local_export::{ BackupKek, MASTER_KEY_VERIFIER_ARGON2ID_PREFIX, MASTER_KEY_VERIFIER_LEGACY_PREFIX, compute_master_key_verifier, decrypt_bundle_artifact, fsync_dir, read_local_bundle_manifest, }; use crate::backup::manifest::{ArtifactKind, BackupManifest, ContentDigest, LocalKeyDerivation}; use crate::error::{KmsError, Result}; use aes_gcm::{ Aes256Gcm, Nonce, aead::{Aead, KeyInit}, }; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tokio::fs; use tracing::warn; use zeroize::Zeroizing; /// Name of the staging subdirectory inside the restore target. const RESTORE_STAGING_DIR: &str = ".restore-staging"; /// Format version of the cutover commit marker. const RESTORE_MARKER_FORMAT_VERSION: u32 = 1; /// Sentinel key id for conflicts that concern the whole target rather than /// one key (the snapshot generation). const WHOLE_TARGET_CONFLICT_ID: &str = "*"; /// How restore resolves conflicts with existing target state. /// /// Silent overwrite or merge is never an option; `explicit-remap` is /// deliberately not offered (see the module docs). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum RestoreConflictPolicy { /// Never write to the target. The default: any actual restore attempt is /// refused, so mutating a key directory always requires the explicit /// opt-in below. Dry-runs are unaffected. #[default] Fail, /// Restore only into an empty target. A target counts as non-empty when /// it holds any key file, a KDF salt (an orphan salt would poison every /// key created later), a marker of a different restore, or any other /// unexpected entry; only leftovers of an interrupted run of this same /// restore are tolerated. RestoreIntoEmptyTarget, } /// Parameters of one restore (or dry-run) attempt. /// /// `observed_generation` is injected by the caller, mirroring the export /// request's `snapshot_generation`: the admin layer owns where the target's /// highest observed generation is tracked; this module freezes only the /// strictly-lower-is-rejected semantics (equal generations stay allowed so /// drills can be repeated). pub struct LocalRestoreRequest { /// Bundle directory produced by the Local export. Read-only. pub bundle_dir: PathBuf, /// Key directory to restore into. pub target_key_dir: PathBuf, /// Deployment identity of the restore target; must match the manifest. pub target_deployment_identity: String, /// Highest snapshot generation the target has already observed, when the /// caller tracks one. pub observed_generation: Option, /// Operator-re-supplied master key string. The master key is outside the /// backup domain, so it must arrive out of band; required whenever the /// bundle carries encrypted-at-rest records. pub master_key: Option, /// Conflict policy; see [`RestoreConflictPolicy`]. pub conflict_policy: RestoreConflictPolicy, /// Unix permission bits for restored files, mirroring /// `LocalConfig::file_permissions`. pub file_permissions: Option, } impl std::fmt::Debug for LocalRestoreRequest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LocalRestoreRequest") .field("bundle_dir", &self.bundle_dir) .field("target_key_dir", &self.target_key_dir) .field("target_deployment_identity", &self.target_deployment_identity) .field("observed_generation", &self.observed_generation) .field("master_key", &self.master_key.as_ref().map(|_| "")) .field("conflict_policy", &self.conflict_policy) .field("file_permissions", &self.file_permissions) .finish() } } impl LocalRestoreRequest { fn validate(&self) -> Result<()> { if self.target_deployment_identity.is_empty() { return Err(KmsError::validation_error("restore target_deployment_identity must not be empty")); } Ok(()) } } /// Serializable outcome of a completed restore. Identifiers only; never key /// material or secrets. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct LocalRestoreReport { /// Identifier of the restored bundle. pub backup_id: String, /// Snapshot generation the target now holds. pub snapshot_generation: u64, /// Stable key ids restored into the target. pub restored_key_ids: Vec, /// Whether the master-key KDF salt was restored. pub salt_restored: bool, /// Whether this run rolled forward a previously interrupted cutover. pub resumed: bool, } /// The cutover commit marker: its durable publication is the single commit /// point of a restore. It names the exact bundle and the exact top-level /// files the cutover publishes, so re-entry can verify it is completing the /// same restore and an abort knows precisely what to take back. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct RestoreCommitMarker { format_version: u32, backup_id: String, manifest_digest: ContentDigest, /// Final top-level file names in link order (salt first, then keys). files: Vec, } impl RestoreCommitMarker { fn for_bundle(decoded: &DecodedBundle) -> Self { let mut files = Vec::with_capacity(decoded.records.len() + 1); if decoded.salt.is_some() { files.push(LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string()); } files.extend(decoded.records.iter().map(|record| record.file_name.clone())); Self { format_version: RESTORE_MARKER_FORMAT_VERSION, backup_id: decoded.manifest.backup_id.clone(), manifest_digest: decoded.manifest.manifest_digest.clone(), files, } } /// Strict decode: an unreadable or out-of-contract marker fails closed — /// abort and roll-forward both refuse to guess what a foreign or damaged /// marker was committing. fn decode(bytes: &[u8]) -> Result { let marker: Self = serde_json::from_slice(bytes) .map_err(|error| BackupError::corrupted(format!("restore commit marker does not decode: {error}")))?; if marker.format_version != RESTORE_MARKER_FORMAT_VERSION { return Err(BackupError::corrupted(format!( "restore commit marker has unknown format version {}", marker.format_version )) .into()); } if marker.files.is_empty() { return Err(BackupError::corrupted("restore commit marker lists no files").into()); } for name in &marker.files { if name == LOCAL_KMS_MASTER_KEY_SALT_FILE { continue; } match name.strip_suffix(".key") { Some(stem) if validate_key_id(stem).is_ok() => {} _ => { return Err(BackupError::corrupted(format!( "restore commit marker lists a file name outside the restore contract: {name:?}" )) .into()); } } } Ok(marker) } fn matches_bundle(&self, manifest: &BackupManifest) -> bool { self.backup_id == manifest.backup_id && self.manifest_digest == manifest.manifest_digest } } /// One key record decoded out of the bundle, fully re-verified. struct DecodedRecord { key_id: String, /// Final on-disk file name (`.key`). file_name: String, protection: StoredKeyProtection, /// Whether the record's material is AEAD-protected under the master key /// (explicit marker, or a legacy record with a nonce). effectively_encrypted: bool, /// Base64-decoded `encrypted_key_material` field. material: Zeroizing>, nonce: Vec, /// Raw record bytes exactly as exported; what staging writes. plaintext: Zeroizing>, } /// The fully decoded and cross-checked bundle content. struct DecodedBundle { manifest: BackupManifest, records: Vec, salt: Option>>, } impl DecodedBundle { fn any_record_encrypted(&self) -> bool { self.records.iter().any(|record| record.effectively_encrypted) } } /// Minimal projection of a stored key record, mirroring the export-side /// probe. Unknown fields are ignored on purpose: the record is restored /// byte-identical, so restore must not constrain its schema. #[derive(Deserialize)] struct RestoredRecordProbe { key_id: String, #[serde(default)] at_rest_protection: StoredKeyProtection, encrypted_key_material: String, #[serde(default)] nonce: Vec, } /// Zero-write restore preflight: evaluate the bundle against the target and /// report every blocker, conflict, and external dependency mismatch. /// /// The bundle is decoded fully in memory (including AEAD verification of /// every artifact); the target directory is only read. Environment failures /// (unreadable directories) surface as errors, verdicts about the bundle or /// the target surface inside the report. pub async fn dry_run_local_restore(kek: &BackupKek, request: &LocalRestoreRequest) -> Result { request.validate()?; let mut report = RestoreDryRunReport { backup_id: String::new(), target_deployment_identity: request.target_deployment_identity.clone(), blockers: Vec::new(), conflicts: Vec::new(), external_mismatches: Vec::new(), }; let decoded = match decode_bundle(&request.bundle_dir, kek).await { Ok(decoded) => decoded, Err(error) => match blocker_for(&error) { Some(blocker) => { report.blockers.push(blocker); return Ok(report); } None => return Err(error), }, }; report.backup_id = decoded.manifest.backup_id.clone(); if let Some(detail) = kdf_drift_detail(&decoded.manifest) { report.blockers.push(RestoreBlocker { code: RestoreBlockerCode::UnsupportedBackend, detail, }); } if decoded.manifest.deployment_identity != request.target_deployment_identity { report.blockers.push(RestoreBlocker { code: RestoreBlockerCode::DeploymentMismatch, detail: format!( "bundle was produced by deployment '{}', restore target is '{}'", decoded.manifest.deployment_identity, request.target_deployment_identity ), }); } if let Some(observed) = request.observed_generation && decoded.manifest.snapshot_generation < observed { report.conflicts.push(RestoreConflict { key_id: WHOLE_TARGET_CONFLICT_ID.to_string(), kind: RestoreConflictKind::GenerationRegression, detail: format!( "target observed snapshot generation {observed}, bundle carries {}; equal generations stay allowed for repeated drills", decoded.manifest.snapshot_generation ), }); } match check_master_key(&decoded, request.master_key.as_deref())? { MasterKeyCheck::Ok => {} MasterKeyCheck::MissingPassphrase(detail) => report.blockers.push(RestoreBlocker { code: RestoreBlockerCode::ExternalDependencyUnavailable, detail, }), MasterKeyCheck::UnknownVerifierScheme(detail) => report.blockers.push(RestoreBlocker { code: RestoreBlockerCode::BundleCorrupted, detail, }), MasterKeyCheck::Mismatch { expected, observed } => report.external_mismatches.push(ExternalDependencyMismatch { dependency: "local KMS master key (operator-supplied)".to_string(), expected, observed, }), } report .conflicts .extend(enumerate_target_conflicts(&request.target_key_dir, &decoded.manifest).await?); Ok(report) } /// Restore a sealed Local bundle into the target key directory. /// /// Fail-fast counterpart of the dry-run: the first violated precondition /// aborts with its typed error before the target top level is touched. Only /// [`RestoreConflictPolicy::RestoreIntoEmptyTarget`] ever writes. pub async fn restore_local_backup(kek: &BackupKek, request: &LocalRestoreRequest) -> Result { request.validate()?; if request.conflict_policy == RestoreConflictPolicy::Fail { return Err(KmsError::invalid_operation( "restore conflict policy 'fail' never writes to the target; review the dry-run report and \ re-run with the 'restore-into-empty-target' policy to proceed", )); } let decoded = decode_bundle(&request.bundle_dir, kek).await?; if let Some(detail) = kdf_drift_detail(&decoded.manifest) { return Err(KmsError::unsupported_capability("local-restore", detail)); } if decoded.manifest.deployment_identity != request.target_deployment_identity { return Err(KmsError::invalid_operation(format!( "bundle was produced by deployment '{}' but the restore target is '{}'", decoded.manifest.deployment_identity, request.target_deployment_identity ))); } if let Some(observed) = request.observed_generation && decoded.manifest.snapshot_generation < observed { return Err(KmsError::invalid_operation(format!( "restore would regress the snapshot generation: target observed {observed}, bundle carries {}", decoded.manifest.snapshot_generation ))); } match check_master_key(&decoded, request.master_key.as_deref())? { MasterKeyCheck::Ok => {} MasterKeyCheck::MissingPassphrase(detail) => return Err(KmsError::invalid_operation(detail)), MasterKeyCheck::UnknownVerifierScheme(detail) => return Err(BackupError::corrupted(detail).into()), MasterKeyCheck::Mismatch { .. } => { return Err(KmsError::validation_error( "operator-supplied master key does not match the bundle's master-key verifier", )); } } // Decryption-probe every record in memory before anything is written: a // wrong master key on a verifier-less legacy bundle must fail here, with // the target untouched. let ciphers = derive_ciphers(request.master_key.as_deref(), decoded.salt.as_deref().map(Vec::as_slice))?; for record in &decoded.records { probe_record_decryption(record, &ciphers)?; } let mode = inspect_target_for_restore(&request.target_key_dir, &decoded.manifest).await?; let expected_marker = RestoreCommitMarker::for_bundle(&decoded); let resumed = match &mode { TargetMode::Fresh { .. } => false, TargetMode::RollForward(published) => { if *published != expected_marker { return Err(BackupError::corrupted( "published restore marker names the same bundle but a different file set; \ abort the interrupted restore before retrying", ) .into()); } true } }; // Phase B: staging. let staging = request.target_key_dir.join(RESTORE_STAGING_DIR); if let TargetMode::Fresh { stale_entries } = &mode { // A fresh run owns no prior state: drop staging leftovers of an // earlier interrupted attempt and any orphan commit temps, then start // over from the bundle. if fs::try_exists(&staging).await? { remove_dir_all_durably(&request.target_key_dir, &staging).await?; } for stale in stale_entries { durable_file::remove_durably(request.target_key_dir.join(stale)).await?; } } fs::create_dir_all(&staging).await?; if let Some(salt) = &decoded.salt { stage_file(&staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, salt, request.file_permissions).await?; } for record in &decoded.records { stage_file(&staging, &record.file_name, &record.plaintext, request.file_permissions).await?; } // Make the staging directory entry itself durable before the commit // point: the marker must never survive a crash that its staged files did // not. fsync_dir(&request.target_key_dir).await?; probe_staged_bundle(&staging, &decoded, &ciphers).await?; // Phase C: commit point, then cutover. if !resumed { publish_marker(&request.target_key_dir, &expected_marker, request.file_permissions).await?; } cutover_and_finish(&request.target_key_dir, &staging, &expected_marker).await?; Ok(LocalRestoreReport { backup_id: decoded.manifest.backup_id.clone(), snapshot_generation: decoded.manifest.snapshot_generation, restored_key_ids: decoded.records.iter().map(|record| record.key_id.clone()).collect(), salt_restored: decoded.salt.is_some(), resumed, }) } /// Explicitly abort an interrupted restore, returning the target to its /// pre-restore (empty) state. /// /// With a published marker this takes back exactly the files the marker /// names, then the marker, then the staging directory. Without a marker only /// staging leftovers exist and are removed. Fails closed on a marker it /// cannot strictly decode. pub async fn abort_local_restore(target_key_dir: &Path) -> Result<()> { let marker_path = target_key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); let staging = target_key_dir.join(RESTORE_STAGING_DIR); match fs::read(&marker_path).await { Ok(bytes) => { let marker = RestoreCommitMarker::decode(&bytes)?; for file_name in &marker.files { let path = target_key_dir.join(file_name); if fs::try_exists(&path).await? { durable_file::remove_durably(path).await?; } } durable_file::remove_durably(marker_path).await?; } Err(error) if error.kind() == std::io::ErrorKind::NotFound => { if !fs::try_exists(&staging).await? { return Err(KmsError::invalid_operation(format!( "no restore is in progress under {}", target_key_dir.display() ))); } } Err(error) => return Err(error.into()), } if fs::try_exists(&staging).await? { remove_dir_all_durably(target_key_dir, &staging).await?; } Ok(()) } /// Decode and cross-verify the complete bundle in memory. /// /// Every artifact is read, digest-verified, and AEAD-authenticated; every key /// record is re-parsed, identity-checked against its artifact path, and /// checked for protection consistency with the manifest's KDF descriptor. async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result { let manifest = read_local_bundle_manifest(bundle_dir).await?; let descriptor = manifest .local_kdf .clone() .ok_or_else(|| BackupError::corrupted("local bundle manifest carries no KDF descriptor"))?; let mut records = Vec::new(); let mut salt: Option>> = None; for artifact in &manifest.artifacts { match artifact.kind { ArtifactKind::KeyMaterial => { let plaintext = decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?; records.push(decode_key_record(artifact.path.as_str(), plaintext, &descriptor.protection_modes)?); } ArtifactKind::MasterKeySalt => { if salt.is_some() { return Err(BackupError::corrupted("bundle carries more than one master-key salt artifact").into()); } let plaintext = decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?; if plaintext.len() != LOCAL_KMS_MASTER_KEY_SALT_LEN { return Err(BackupError::corrupted(format!( "bundled master-key salt is {} bytes, expected {}", plaintext.len(), LOCAL_KMS_MASTER_KEY_SALT_LEN )) .into()); } salt = Some(plaintext); } // No producer emits these for a Local bundle yet; restoring a // bundle that carries state this implementation cannot apply // would silently drop it, so fail closed instead. other => { return Err(KmsError::unsupported_capability( "local-restore", format!("bundle carries artifact kind {other:?}, which this restore implementation cannot apply"), )); } } } // The salt must be exactly as coherent with the KDF descriptor as the // export invariants promise; anything else is a broken or tampered // bundle. match (&descriptor.derivation, salt.is_some()) { (LocalKeyDerivation::Argon2id { .. }, false) => { return Err( BackupError::corrupted("KDF descriptor declares Argon2id but the bundle carries no salt artifact").into(), ); } (LocalKeyDerivation::LegacySha256, true) => { return Err( BackupError::corrupted("KDF descriptor declares the legacy derivation but the bundle carries a salt").into(), ); } _ => {} } Ok(DecodedBundle { manifest, records, salt }) } fn decode_key_record( artifact_path: &str, plaintext: Zeroizing>, allowed_modes: &[AtRestProtection], ) -> Result { let stem = Path::new(artifact_path) .file_name() .and_then(|name| name.to_str()) .and_then(|name| name.strip_suffix(".key.enc")) .ok_or_else(|| { BackupError::corrupted(format!( "key material artifact path {artifact_path:?} does not follow the '.key.enc' layout" )) })? .to_string(); validate_key_id(&stem)?; let probe: RestoredRecordProbe = serde_json::from_slice(&plaintext) .map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' does not deserialize: {error}")))?; if probe.key_id != stem { return Err(BackupError::corrupted(format!( "bundled key record identity mismatch: artifact names {stem:?}, record names {:?}", probe.key_id )) .into()); } if probe.encrypted_key_material.is_empty() { return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into()); } let material = Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| { BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}")) })?); if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) { return Err(BackupError::corrupted(format!( "bundled key record '{stem}' declares protection {:?}, absent from the manifest's KDF descriptor", probe.at_rest_protection )) .into()); } let effectively_encrypted = match probe.at_rest_protection { StoredKeyProtection::EncryptedMasterKey => true, StoredKeyProtection::PlaintextDevOnly => false, StoredKeyProtection::LegacyUnspecified => !probe.nonce.is_empty(), }; if effectively_encrypted && probe.nonce.len() != 12 { return Err(BackupError::corrupted(format!( "bundled key record '{stem}' has an invalid nonce length ({} bytes, expected 12)", probe.nonce.len() )) .into()); } Ok(DecodedRecord { file_name: format!("{stem}.key"), key_id: stem, protection: probe.at_rest_protection, effectively_encrypted, material, nonce: probe.nonce, plaintext, }) } fn protection_mode(protection: StoredKeyProtection) -> AtRestProtection { match protection { StoredKeyProtection::EncryptedMasterKey => AtRestProtection::EncryptedMasterKey, StoredKeyProtection::PlaintextDevOnly => AtRestProtection::PlaintextDevOnly, StoredKeyProtection::LegacyUnspecified => AtRestProtection::LegacyUnspecified, } } /// Detail message when the bundle's recorded KDF differs from what this build /// compiles in; restoring such a bundle would produce keys this build cannot /// decrypt. fn kdf_drift_detail(manifest: &BackupManifest) -> Option { match &manifest.local_kdf.as_ref()?.derivation { derivation @ LocalKeyDerivation::Argon2id { .. } => { let current = LocalKeyDerivation::current_argon2id(); (*derivation != current).then(|| { format!("bundle KDF descriptor {derivation:?} does not match this build's compiled-in derivation {current:?}") }) } LocalKeyDerivation::LegacySha256 => None, } } enum MasterKeyCheck { Ok, MissingPassphrase(String), UnknownVerifierScheme(String), Mismatch { expected: String, observed: String }, } /// Verify the operator-supplied master key against the manifest's one-way /// verifier, and require one whenever the bundle carries encrypted records. /// /// Bundles without a verifier (produced before the verifier landed) fall /// back to the staging-phase decryption probe; unknown verifier schemes fail /// closed. fn check_master_key(decoded: &DecodedBundle, master_key: Option<&str>) -> Result { let Some(master_key) = master_key else { if decoded.any_record_encrypted() { return Ok(MasterKeyCheck::MissingPassphrase( "bundle carries encrypted-at-rest key records; the operator must re-supply the master key before restore" .to_string(), )); } return Ok(MasterKeyCheck::Ok); }; let verifier = decoded .manifest .local_kdf .as_ref() .and_then(|descriptor| descriptor.master_key_verifier.as_deref()); let Some(verifier) = verifier else { return Ok(MasterKeyCheck::Ok); }; let observed = if verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX) { let Some(salt) = decoded.salt.as_deref() else { return Err( BackupError::corrupted("manifest carries an Argon2id master-key verifier but the bundle has no salt").into(), ); }; compute_master_key_verifier(master_key, Some(salt), &decoded.manifest.backup_id)? } else if verifier.starts_with(MASTER_KEY_VERIFIER_LEGACY_PREFIX) { compute_master_key_verifier(master_key, None, &decoded.manifest.backup_id)? } else { return Ok(MasterKeyCheck::UnknownVerifierScheme( "manifest carries a master-key verifier with an unknown scheme prefix; failing closed".to_string(), )); }; if observed != verifier { return Ok(MasterKeyCheck::Mismatch { expected: verifier.to_string(), observed, }); } Ok(MasterKeyCheck::Ok) } /// AEAD ciphers derived from the operator-supplied master key, mirroring the /// backend's current-plus-legacy pair. struct DerivedCiphers { current: Option, legacy: Option, } fn derive_ciphers(master_key: Option<&str>, salt: Option<&[u8]>) -> Result { let Some(master_key) = master_key else { return Ok(DerivedCiphers { current: None, legacy: None, }); }; let legacy = Aes256Gcm::new(&LocalKmsClient::derive_legacy_master_key(master_key)?); let current = match salt { Some(salt) => Aes256Gcm::new(&LocalKmsClient::derive_master_key(master_key, salt)?), None => legacy.clone(), }; Ok(DerivedCiphers { current: Some(current), legacy: Some(legacy), }) } /// Decryption probe for one record, mirroring the backend's read path: the /// current cipher first, with the legacy fallback only for legacy-unmarked /// records. Local has no rotation history, so probing every record is the /// full-coverage probe. fn probe_record_decryption(record: &DecodedRecord, ciphers: &DerivedCiphers) -> Result<()> { if !record.effectively_encrypted { return Ok(()); } let Some(current) = ciphers.current.as_ref() else { return Err(KmsError::invalid_operation(format!( "bundled key record '{}' is encrypted at rest and requires the operator-supplied master key", record.key_id ))); }; let mut nonce = [0u8; 12]; nonce.copy_from_slice(&record.nonce); let nonce = Nonce::from(nonce); match current.decrypt(&nonce, record.material.as_slice()) { Ok(material) => { drop(Zeroizing::new(material)); Ok(()) } Err(_) if record.protection == StoredKeyProtection::LegacyUnspecified => { let legacy = ciphers .legacy .as_ref() .expect("legacy cipher exists whenever the current cipher does"); let material = legacy .decrypt(&nonce, record.material.as_slice()) .map_err(|_| KmsError::material_authentication_failed(&record.key_id))?; drop(Zeroizing::new(material)); Ok(()) } Err(_) => Err(KmsError::material_authentication_failed(&record.key_id)), } } /// Re-run the decryption probe against the bytes that actually landed in /// staging, so cutover only ever publishes state proven readable from disk. async fn probe_staged_bundle(staging: &Path, decoded: &DecodedBundle, ciphers: &DerivedCiphers) -> Result<()> { if let Some(salt) = &decoded.salt { let staged_salt = fs::read(staging.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)).await?; if staged_salt.as_slice() != salt.as_slice() { return Err(BackupError::corrupted("staged master-key salt does not match the bundle").into()); } } for record in &decoded.records { let staged = Zeroizing::new(fs::read(staging.join(&record.file_name)).await?); if staged.as_slice() != record.plaintext.as_slice() { return Err( BackupError::corrupted(format!("staged key record '{}' does not match the bundle", record.key_id)).into(), ); } let staged_record = decode_key_record( &format!("staged/{}.enc", record.file_name), staged, &decoded .manifest .local_kdf .as_ref() .expect("decoded bundle always carries a KDF descriptor") .protection_modes, )?; probe_record_decryption(&staged_record, ciphers)?; } Ok(()) } enum TargetMode { /// The target is empty apart from removable leftovers of an interrupted /// earlier attempt (staging, orphan commit temps). Fresh { stale_entries: Vec }, /// A published marker of this same bundle: complete its cutover. RollForward(RestoreCommitMarker), } /// Fail-fast target inspection for an actual restore. async fn inspect_target_for_restore(target: &Path, manifest: &BackupManifest) -> Result { if !fs::try_exists(target).await? { return Ok(TargetMode::Fresh { stale_entries: Vec::new(), }); } let marker = read_marker(target).await?; if let Some(marker) = marker { if !marker.matches_bundle(manifest) { return Err(KmsError::invalid_operation(format!( "target key directory holds a restore marker for a different bundle ('{}'); \ roll that restore forward with its own bundle or abort it explicitly", marker.backup_id ))); } // Everything at the top level must belong to this restore: the // marker's files (partially linked) and our own working entries. let allowed: Vec<&str> = marker.files.iter().map(String::as_str).collect(); for entry in list_target_entries(target).await? { if !allowed.contains(&entry.as_str()) && !is_orphan_commit_temp_name(&entry) { return Err(KmsError::invalid_operation(format!( "target key directory holds an entry outside the interrupted restore: {entry:?}; \ clean it up manually before resuming" ))); } } return Ok(TargetMode::RollForward(marker)); } let mut stale_entries = Vec::new(); for entry in list_target_entries(target).await? { if is_orphan_commit_temp_name(&entry) { // Unpublished commit temps are never authoritative; a re-run may // remove them exactly as startup recovery would. stale_entries.push(entry); continue; } let detail = if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE { "an orphan KDF salt would poison every key created after the restore; clean it up manually" } else { "restore only ever writes into an empty target" }; return Err(KmsError::invalid_operation(format!( "target key directory is not empty (found {entry:?}); {detail}" ))); } Ok(TargetMode::Fresh { stale_entries }) } /// Top-level entries of the target, excluding the staging directory and the /// commit marker (both handled separately). async fn list_target_entries(target: &Path) -> Result> { let mut names = Vec::new(); let mut entries = fs::read_dir(target).await?; while let Some(entry) = entries.next_entry().await? { let name = entry .file_name() .to_str() .ok_or_else(|| KmsError::invalid_operation("target key directory holds a non-UTF-8 entry; clean it up manually"))? .to_string(); if name == RESTORE_STAGING_DIR || name == LOCAL_RESTORE_COMMIT_MARKER_FILE { continue; } names.push(name); } names.sort(); Ok(names) } async fn read_marker(target: &Path) -> Result> { match fs::read(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)).await { Ok(bytes) => Ok(Some(RestoreCommitMarker::decode(&bytes)?)), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(error.into()), } } /// Conflict enumeration for the dry-run: every target entry that would block /// the restore, without failing on the first one. async fn enumerate_target_conflicts(target: &Path, manifest: &BackupManifest) -> Result> { if !fs::try_exists(target).await? { return Ok(Vec::new()); } let mut conflicts = Vec::new(); let marker = match read_marker(target).await { Ok(marker) => marker, // A marker that does not strictly decode blocks exactly like a // foreign one; the dry-run reports it instead of failing. Err(error) => { conflicts.push(RestoreConflict { key_id: LOCAL_RESTORE_COMMIT_MARKER_FILE.to_string(), kind: RestoreConflictKind::KeyAlreadyExists, detail: format!("target holds an undecodable restore marker: {error}"), }); None } }; let own_files: Vec = match &marker { Some(marker) if marker.matches_bundle(manifest) => marker.files.clone(), Some(marker) => { conflicts.push(RestoreConflict { key_id: LOCAL_RESTORE_COMMIT_MARKER_FILE.to_string(), kind: RestoreConflictKind::KeyAlreadyExists, detail: format!("target holds a restore marker for a different bundle ('{}')", marker.backup_id), }); Vec::new() } None => Vec::new(), }; for entry in list_target_entries(target).await? { if own_files.contains(&entry) || is_orphan_commit_temp_name(&entry) { continue; } let (key_id, detail) = if let Some(stem) = entry.strip_suffix(".key") { (stem.to_string(), format!("target already has key file {entry:?}")) } else if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE { ( entry.clone(), "target holds an orphan master-key KDF salt; it would poison every key created after the restore".to_string(), ) } else { (entry.clone(), format!("target holds an unexpected entry {entry:?}")) }; conflicts.push(RestoreConflict { key_id, kind: RestoreConflictKind::KeyAlreadyExists, detail, }); } Ok(conflicts) } /// Durably commit one staged file. Idempotent across roll-forward re-entry: /// an already staged file is accepted only byte-identical. async fn stage_file(staging: &Path, file_name: &str, content: &[u8], permissions: Option) -> Result<()> { let final_path = staging.join(file_name); match fs::read(&final_path).await { Ok(existing) => { let existing = Zeroizing::new(existing); if existing.as_slice() == content { return Ok(()); } return Err(BackupError::corrupted(format!( "staging already holds '{file_name}' with different content; abort the interrupted restore before retrying" )) .into()); } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(error.into()), } let temp_path = staging.join(format!("{file_name}.tmp-{}", uuid::Uuid::new_v4())); durable_file::commit(temp_path, final_path, content.to_vec(), permissions, durable_file::Publish::NoClobber) .await .map_err(KmsError::from) } /// Durably publish the commit marker. This is the restore's single commit /// point: before it, the target top level is untouched; after it, the only /// ways out are roll-forward or explicit abort. async fn publish_marker(target: &Path, marker: &RestoreCommitMarker, permissions: Option) -> Result<()> { let content = serde_json::to_vec_pretty(marker) .map_err(|error| KmsError::internal_error(format!("restore commit marker serialization failed: {error}")))?; let final_path = target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); let temp_path = target.join(format!("{LOCAL_RESTORE_COMMIT_MARKER_FILE}.tmp-{}", uuid::Uuid::new_v4())); durable_file::commit(temp_path, final_path, content, permissions, durable_file::Publish::NoClobber) .await .map_err(KmsError::from) } /// The atomic cutover: publish every staged file under its final name (salt /// first, keys after — the marker froze that order), durably remove the /// marker, then drop the staging directory. async fn cutover_and_finish(target: &Path, staging: &Path, marker: &RestoreCommitMarker) -> Result<()> { for file_name in &marker.files { durable_file::link_durably(staging.join(file_name), target.join(file_name)) .await .map_err(|error| KmsError::io_error(format!("restore cutover failed publishing '{file_name}': {error}")))?; } durable_file::remove_durably(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)) .await .map_err(|error| KmsError::io_error(format!("restore cutover failed removing the commit marker: {error}")))?; // The restore is complete once the marker is gone; a leftover staging // directory is inert (its entries are hard links of the published files), // so its removal is cleanup, not protocol. if let Err(error) = remove_dir_all_durably(target, staging).await { warn!(%error, "restored Local KMS key directory kept an inert staging directory"); } Ok(()) } async fn remove_dir_all_durably(parent: &Path, dir: &Path) -> Result<()> { let dir = dir.to_path_buf(); tokio::task::spawn_blocking(move || std::fs::remove_dir_all(&dir)) .await .map_err(|error| KmsError::io_error(error.to_string()))??; fsync_dir(parent).await } fn blocker_for(error: &KmsError) -> Option { match error { KmsError::Backup(inner) => Some(RestoreBlocker::from(inner)), KmsError::UnsupportedCapability { .. } => Some(RestoreBlocker { code: RestoreBlockerCode::UnsupportedBackend, detail: error.to_string(), }), _ => None, } } #[cfg(test)] mod tests { use super::*; use crate::backup::local_export::{LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, export_local_backup}; use crate::config::LocalConfig; use sha2::{Digest, Sha256}; use std::time::SystemTime; use tempfile::TempDir; const TEST_MASTER_KEY: &str = "test-master-key"; const DEPLOYMENT: &str = "deployment-test"; const BACKUP_ID: &str = "backup-0001"; const SNAPSHOT_GENERATION: u64 = 7; fn local_config(dir: &Path, master_key: Option<&str>) -> LocalConfig { LocalConfig { key_dir: dir.to_path_buf(), master_key: master_key.map(str::to_string), file_permissions: Some(0o600), } } async fn source_with_keys(master_key: Option<&str>, keys: &[&str]) -> (LocalKmsClient, TempDir) { let temp = TempDir::new().expect("temp dir"); let client = LocalKmsClient::new(local_config(temp.path(), master_key)) .await .expect("client should initialize"); for key in keys { client.create_key(key, "AES_256", None).await.expect("create key"); } (client, temp) } fn test_kek() -> BackupKek { BackupKek::new("backup-kek-test", 1, [0x42; 32]).expect("kek") } async fn export_bundle(client: &LocalKmsClient, dir: &Path) -> (PathBuf, BackupManifest) { let destination = dir.join("bundle"); let manifest = export_local_backup( client, &test_kek(), &LocalBackupExportRequest { backup_id: BACKUP_ID.to_string(), deployment_identity: DEPLOYMENT.to_string(), rustfs_version: "1.0.0-test".to_string(), snapshot_generation: SNAPSHOT_GENERATION, destination: destination.clone(), }, ) .await .expect("export should succeed"); (destination, manifest) } fn restore_request(bundle: &Path, target: &Path) -> LocalRestoreRequest { LocalRestoreRequest { bundle_dir: bundle.to_path_buf(), target_key_dir: target.to_path_buf(), target_deployment_identity: DEPLOYMENT.to_string(), observed_generation: None, master_key: Some(TEST_MASTER_KEY.to_string()), conflict_policy: RestoreConflictPolicy::RestoreIntoEmptyTarget, file_permissions: Some(0o600), } } /// Recursive (path, content SHA-256, mtime) snapshot; equality across an /// operation proves it performed no persistent write at all. fn snapshot_tree(dir: &Path) -> Vec<(String, Vec, SystemTime)> { fn walk(root: &Path, dir: &Path, out: &mut Vec<(String, Vec, SystemTime)>) { for entry in std::fs::read_dir(dir).expect("read dir") { let entry = entry.expect("dir entry"); let path = entry.path(); if path.is_dir() { walk(root, &path, out); continue; } let relative = path.strip_prefix(root).expect("under root").to_string_lossy().into_owned(); let content = std::fs::read(&path).expect("read file"); let mtime = entry.metadata().expect("metadata").modified().expect("mtime"); out.push((relative, Sha256::digest(&content).to_vec(), mtime)); } } let mut out = Vec::new(); walk(dir, dir, &mut out); out.sort(); out } async fn top_level_names(dir: &Path) -> Vec { let mut names = Vec::new(); let mut entries = fs::read_dir(dir).await.expect("read dir"); while let Some(entry) = entries.next_entry().await.expect("dir entry") { names.push(entry.file_name().to_string_lossy().into_owned()); } names.sort(); names } /// Re-seal the bundle manifest after mutating it, keeping the digest /// contract intact so only the mutated aspect is under test. async fn reseal_manifest(bundle_dir: &Path, mutate: impl FnOnce(&mut BackupManifest)) { let path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE); let mut manifest = BackupManifest::decode(&fs::read(&path).await.expect("manifest bytes")).expect("manifest decodes"); mutate(&mut manifest); let sealed = manifest.seal().expect("re-seal"); fs::write(&path, sealed.encode().expect("encode")) .await .expect("write manifest"); } /// The full acceptance check for a restored directory: raw files /// byte-identical to the source, and every record decodable through the /// read-only export constructor with byte-identical material. async fn assert_restored_matches_source(source: &LocalKmsClient, target: &Path, keys: &[&str]) { for key in keys { let source_bytes = fs::read(source.key_directory().join(format!("{key}.key"))) .await .expect("source record"); let restored_bytes = fs::read(target.join(format!("{key}.key"))).await.expect("restored record"); assert_eq!(source_bytes, restored_bytes, "record {key} must restore byte-for-byte"); } let source_salt = fs::read(source.master_key_salt_file()).await.expect("source salt"); let restored_salt = fs::read(target.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)) .await .expect("restored salt"); assert_eq!(source_salt, restored_salt, "salt must restore byte-for-byte"); let restored = LocalKmsClient::new_for_key_export(local_config(target, Some(TEST_MASTER_KEY))) .await .expect("restored directory must open read-only"); for key in keys { let source_material = source.decrypt_key_material_for_export(key).await.expect("source material"); let restored_material = restored .decrypt_key_material_for_export(key) .await .expect("restored material"); assert_eq!( source_material.as_ref(), restored_material.as_ref(), "key material for {key} must survive the restore byte-for-byte" ); } } #[tokio::test] async fn dry_run_is_zero_write_and_permits_clean_restore() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); fs::create_dir_all(&target).await.expect("create target"); let bundle_before = snapshot_tree(&bundle); let target_before = snapshot_tree(&target); let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, &target)) .await .expect("dry run should succeed"); assert!(report.restore_permitted(), "clean bundle and empty target must permit: {report:?}"); assert_eq!(report.backup_id, BACKUP_ID); assert_eq!(report.target_deployment_identity, DEPLOYMENT); assert_eq!(snapshot_tree(&bundle), bundle_before, "dry run must not touch the bundle"); assert_eq!(snapshot_tree(&target), target_before, "dry run must not touch the target"); // A missing target directory stays missing. let missing = work.path().join("missing-target"); let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, &missing)) .await .expect("dry run should succeed"); assert!(report.restore_permitted()); assert!(!missing.exists(), "dry run must not create the target directory"); } #[tokio::test] async fn dry_run_flags_deployment_mismatch_and_generation_regression() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let mut request = restore_request(&bundle, &target); request.target_deployment_identity = "deployment-other".to_string(); request.observed_generation = Some(SNAPSHOT_GENERATION + 1); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!( report .blockers .iter() .any(|blocker| blocker.code == RestoreBlockerCode::DeploymentMismatch), "expected a deployment mismatch blocker: {report:?}" ); assert!( report .conflicts .iter() .any(|conflict| conflict.kind == RestoreConflictKind::GenerationRegression), "expected a generation regression conflict: {report:?}" ); // Equal generations are allowed so drills can repeat. let mut request = restore_request(&bundle, &target); request.observed_generation = Some(SNAPSHOT_GENERATION); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!(report.restore_permitted(), "equal generation must not conflict: {report:?}"); } #[tokio::test] async fn dry_run_checks_the_operator_master_key() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let verifier = manifest .local_kdf .as_ref() .and_then(|kdf| kdf.master_key_verifier.as_deref()) .expect("encrypted bundle must carry a verifier"); assert!(verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX), "got {verifier:?}"); // No passphrase: encrypted records make the master key a required // external dependency. let mut request = restore_request(&bundle, &target); request.master_key = None; let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!( report .blockers .iter() .any(|blocker| blocker.code == RestoreBlockerCode::ExternalDependencyUnavailable), "expected a missing-master-key blocker: {report:?}" ); // Wrong passphrase: reported as an external dependency mismatch. let mut request = restore_request(&bundle, &target); request.master_key = Some("wrong-master-key".to_string()); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert_eq!(report.external_mismatches.len(), 1, "{report:?}"); assert_eq!(report.external_mismatches[0].expected, verifier); assert!(!report.restore_permitted()); } #[tokio::test] async fn dry_run_enumerates_existing_target_state() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; // A populated target: a key file, the salt, and a stray file. let (_occupant, target_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["occupied"]).await; fs::write(target_dir.path().join("operator-notes.txt"), b"leftover") .await .expect("stray file"); let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, target_dir.path())) .await .expect("dry run"); let conflicting_ids: Vec<&str> = report.conflicts.iter().map(|conflict| conflict.key_id.as_str()).collect(); assert!(conflicting_ids.contains(&"occupied"), "{report:?}"); assert!(conflicting_ids.contains(&LOCAL_KMS_MASTER_KEY_SALT_FILE), "{report:?}"); assert!(conflicting_ids.contains(&"operator-notes.txt"), "{report:?}"); assert!(!report.restore_permitted()); } #[tokio::test] async fn restore_into_empty_target_round_trips_byte_for_byte() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let bundle_before = snapshot_tree(&bundle); let report = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) .await .expect("restore should succeed"); assert_eq!( report, LocalRestoreReport { backup_id: BACKUP_ID.to_string(), snapshot_generation: SNAPSHOT_GENERATION, restored_key_ids: vec!["alpha".to_string(), "beta".to_string()], salt_restored: true, resumed: false, } ); assert_eq!(snapshot_tree(&bundle), bundle_before, "the bundle source must stay byte-identical"); assert_eq!( top_level_names(&target).await, vec![ LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string(), "alpha.key".to_string(), "beta.key".to_string() ], "cutover must leave no marker, staging, or temp files behind" ); assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; // The restored directory boots as a normal backend directory. let restored = LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))) .await .expect("restored directory must pass startup validation"); restored.describe_key("alpha", None).await.expect("restored key describes"); } #[tokio::test] async fn default_fail_policy_never_writes() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let mut request = restore_request(&bundle, &target); request.conflict_policy = RestoreConflictPolicy::Fail; let error = restore_local_backup(&test_kek(), &request) .await .expect_err("the default policy must refuse to write"); assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); assert!(!target.exists(), "the fail policy must not create the target"); } #[tokio::test] async fn non_empty_targets_are_rejected() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, manifest) = export_bundle(&client, work.path()).await; // Existing key material. let (_occupant, occupied) = source_with_keys(Some(TEST_MASTER_KEY), &["occupied"]).await; let before = snapshot_tree(occupied.path()); let error = restore_local_backup(&test_kek(), &restore_request(&bundle, occupied.path())) .await .expect_err("occupied target must be rejected"); assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); assert_eq!(snapshot_tree(occupied.path()), before, "the rejected target must stay untouched"); // An orphan salt alone makes the target non-empty: a foreign salt // would poison every key created after the restore. let orphan = TempDir::new().expect("orphan dir"); fs::write(orphan.path().join(LOCAL_KMS_MASTER_KEY_SALT_FILE), [7u8; 16]) .await .expect("seed orphan salt"); let before = snapshot_tree(orphan.path()); let error = restore_local_backup(&test_kek(), &restore_request(&bundle, orphan.path())) .await .expect_err("orphan salt must be rejected"); assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); assert!(error.to_string().contains("salt"), "got {error}"); assert_eq!(snapshot_tree(orphan.path()), before); // A marker of a different restore. let foreign = TempDir::new().expect("foreign dir"); let mut digest = manifest.manifest_digest.clone(); digest.hex = digest.hex.chars().rev().collect(); let foreign_marker = RestoreCommitMarker { format_version: RESTORE_MARKER_FORMAT_VERSION, backup_id: "backup-foreign".to_string(), manifest_digest: digest, files: vec!["foreign.key".to_string()], }; fs::write( foreign.path().join(LOCAL_RESTORE_COMMIT_MARKER_FILE), serde_json::to_vec_pretty(&foreign_marker).expect("marker json"), ) .await .expect("seed foreign marker"); let before = snapshot_tree(foreign.path()); let error = restore_local_backup(&test_kek(), &restore_request(&bundle, foreign.path())) .await .expect_err("foreign marker must be rejected"); assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); assert!(error.to_string().contains("different bundle"), "got {error}"); assert_eq!(snapshot_tree(foreign.path()), before); } #[tokio::test] async fn wrong_passphrase_fails_via_verifier_before_any_target_write() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let mut request = restore_request(&bundle, &target); request.master_key = Some("wrong-master-key".to_string()); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("wrong master key must be rejected by the verifier"); assert!(matches!(error, KmsError::ValidationError { .. }), "got {error:?}"); assert!(!target.exists(), "the verifier must reject before any target write"); } #[tokio::test] async fn wrong_passphrase_fails_via_probe_when_bundle_has_no_verifier() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); // Simulate a bundle produced before the verifier landed. reseal_manifest(&bundle, |manifest| { manifest .local_kdf .as_mut() .expect("local bundle has a KDF descriptor") .master_key_verifier = None; }) .await; let mut request = restore_request(&bundle, &target); request.master_key = Some("wrong-master-key".to_string()); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("wrong master key must be caught by the decryption probe"); assert!(matches!(error, KmsError::MaterialAuthenticationFailed { .. }), "got {error:?}"); assert!(!target.exists(), "the in-memory probe must reject before any target write"); // The right passphrase still restores the verifier-less bundle. restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) .await .expect("verifier-less bundle must restore with the right master key"); assert_restored_matches_source(&client, &target, &["alpha"]).await; } #[tokio::test] async fn bundle_defects_fail_closed_on_restore() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["victim"]).await; let work = TempDir::new().expect("work dir"); let (bundle, manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let request = restore_request(&bundle, &target); let artifact_file = bundle.join(&manifest.artifacts[0].path); let original_artifact = std::fs::read(&artifact_file).expect("artifact bytes"); // Tampered artifact byte. let mut tampered = original_artifact.clone(); let last = tampered.len() - 1; tampered[last] ^= 0x01; std::fs::write(&artifact_file, &tampered).expect("write tampered"); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("tampered artifact must be rejected"); assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!( report .blockers .iter() .any(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted), "{report:?}" ); // Truncated artifact. std::fs::write(&artifact_file, &original_artifact[..original_artifact.len() - 4]).expect("truncate"); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("truncated artifact must be rejected"); assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); std::fs::write(&artifact_file, &original_artifact).expect("restore artifact"); // Wrong KEK identity and wrong KEK material. let wrong_id = BackupKek::new("other-kek", 1, [0x42; 32]).expect("kek"); let error = restore_local_backup(&wrong_id, &request) .await .expect_err("wrong KEK id must be rejected"); assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); let wrong_material = BackupKek::new("backup-kek-test", 1, [0x24; 32]).expect("kek"); let error = restore_local_backup(&wrong_material, &request) .await .expect_err("wrong KEK material must be rejected"); assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); // Missing manifest: the bundle never sealed. std::fs::remove_file(bundle.join(LOCAL_BUNDLE_MANIFEST_FILE)).expect("remove manifest"); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("unsealed bundle must be rejected"); assert!(matches!(error, KmsError::Backup(BackupError::IncompleteBundle { .. })), "got {error:?}"); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!( report .blockers .iter() .any(|blocker| blocker.code == RestoreBlockerCode::IncompleteBundle), "{report:?}" ); assert!(!target.exists(), "no rejected bundle may touch the target"); } #[tokio::test] async fn kdf_parameter_drift_is_rejected() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); reseal_manifest(&bundle, |manifest| { let descriptor = manifest.local_kdf.as_mut().expect("local bundle has a KDF descriptor"); match &mut descriptor.derivation { LocalKeyDerivation::Argon2id { memory_kib, .. } => *memory_kib *= 2, other => panic!("expected an Argon2id descriptor, got {other:?}"), } }) .await; let request = restore_request(&bundle, &target); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("a KDF parameter drift must be rejected"); assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "got {error:?}"); assert!(!target.exists()); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!( report .blockers .iter() .any(|blocker| blocker.code == RestoreBlockerCode::UnsupportedBackend), "{report:?}" ); } #[tokio::test] async fn unknown_verifier_scheme_fails_closed() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); reseal_manifest(&bundle, |manifest| { manifest .local_kdf .as_mut() .expect("local bundle has a KDF descriptor") .master_key_verifier = Some("post-quantum-v2:0123abcd".to_string()); }) .await; let request = restore_request(&bundle, &target); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("an unknown verifier scheme must fail closed"); assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); assert!(!target.exists()); let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); assert!( report .blockers .iter() .any(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted), "{report:?}" ); } #[tokio::test] async fn generation_regression_is_rejected_and_equal_is_allowed() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let mut request = restore_request(&bundle, &target); request.observed_generation = Some(SNAPSHOT_GENERATION + 1); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("a strictly lower bundle generation must be rejected"); assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); assert!(!target.exists()); request.observed_generation = Some(SNAPSHOT_GENERATION); restore_local_backup(&test_kek(), &request) .await .expect("an equal generation must stay restorable for repeated drills"); assert_restored_matches_source(&client, &target, &["alpha"]).await; } #[tokio::test] async fn dev_mode_bundle_round_trips_without_passphrase() { let (client, _source_dir) = source_with_keys(None, &["dev-key"]).await; let work = TempDir::new().expect("work dir"); let (bundle, manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); assert_eq!(kdf.master_key_verifier, None, "dev-mode bundles have no master key to verify"); let mut request = restore_request(&bundle, &target); request.master_key = None; let report = restore_local_backup(&test_kek(), &request) .await .expect("dev-mode restore must succeed without a passphrase"); assert!(!report.salt_restored); assert_eq!(report.restored_key_ids, vec!["dev-key".to_string()]); let source_bytes = fs::read(client.key_directory().join("dev-key.key")).await.expect("source"); let restored_bytes = fs::read(target.join("dev-key.key")).await.expect("restored"); assert_eq!(source_bytes, restored_bytes); let restored = LocalKmsClient::new(local_config(&target, None)) .await .expect("restored dev directory must boot"); restored.describe_key("dev-key", None).await.expect("restored key describes"); } #[tokio::test] async fn legacy_sha256_bundle_round_trips() { // A pre-beta.9 directory: legacy SHA-256 encrypted record, no salt, // no protection marker (same fixture as the backend's beta.5 test; // the material decrypts to 32 x 0x42 under this master key). let source_dir = TempDir::new().expect("legacy dir"); let master_key = "beta5-test-master-key"; let record = serde_json::json!({ "key_id": "beta5-key", "version": 1u32, "algorithm": "AES_256", "usage": "EncryptDecrypt", "status": "Active", "description": serde_json::Value::Null, "metadata": std::collections::HashMap::::new(), "created_at": "2024-01-01T00:00:00+00:00", "rotated_at": serde_json::Value::Null, "created_by": "beta5-fixture", "encrypted_key_material": "xjwGa4Lj4qzKg6XQl8s2btyFkPHPChMAkjqs268TFGyvFUv8WjDD5HQCUDLViZmt", "nonce": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] }); fs::write( source_dir.path().join("beta5-key.key"), serde_json::to_vec_pretty(&record).expect("record json"), ) .await .expect("write legacy record"); // Export through the read-only constructor so no Argon2 salt gets // created: the bundle must record the legacy derivation. let client = LocalKmsClient::new_for_key_export(local_config(source_dir.path(), Some(master_key))) .await .expect("open legacy dir read-only"); let work = TempDir::new().expect("work dir"); let (bundle, manifest) = export_bundle(&client, work.path()).await; let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); assert_eq!(kdf.derivation, LocalKeyDerivation::LegacySha256); let verifier = kdf.master_key_verifier.as_deref().expect("legacy verifier"); assert!(verifier.starts_with(MASTER_KEY_VERIFIER_LEGACY_PREFIX), "got {verifier:?}"); let target = work.path().join("target"); let mut request = restore_request(&bundle, &target); request.master_key = Some("wrong-master-key".to_string()); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("the legacy verifier must reject a wrong master key"); assert!(matches!(error, KmsError::ValidationError { .. }), "got {error:?}"); assert!(!target.exists()); request.master_key = Some(master_key.to_string()); let report = restore_local_backup(&test_kek(), &request).await.expect("legacy restore"); assert!(!report.salt_restored, "a legacy bundle carries no salt"); // The restored directory boots (creating a fresh Argon2 salt is // legitimate for legacy records) and the material still decrypts via // the legacy fallback. let restored = LocalKmsClient::new(local_config(&target, Some(master_key))) .await .expect("restored legacy directory must boot"); let material = restored .decrypt_key_material_for_export("beta5-key") .await .expect("legacy material must decrypt after restore"); assert_eq!(material.as_ref(), &[0x42; 32]); } #[tokio::test] async fn crash_during_staging_keeps_top_level_untouched_and_rerun_converges() { use durable_file::{CommitStep, failpoint}; for step in [ CommitStep::TempWritten, CommitStep::FileSynced, CommitStep::Published, CommitStep::DirSynced, ] { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let request = restore_request(&bundle, &target); failpoint::arm(&target, step); let error = restore_local_backup(&test_kek(), &request) .await .expect_err("armed staging commit must simulate a crash"); failpoint::disarm(&target); assert!(error.to_string().contains("injected crash"), "step {step:?}: {error}"); // Top level: only the staging directory, never a marker or a key. assert_eq!( top_level_names(&target).await, vec![RESTORE_STAGING_DIR.to_string()], "step {step:?}: a staging crash must leave the top level untouched" ); restore_local_backup(&test_kek(), &request) .await .unwrap_or_else(|error| panic!("step {step:?}: re-run must converge: {error}")); assert_restored_matches_source(&client, &target, &["alpha"]).await; } } /// Drive the phases manually up to the marker commit, crash the commit at /// every step, and prove both re-entry outcomes: an unpublished marker /// temp is removable leftover state, a published marker rolls forward. #[tokio::test] async fn crash_around_marker_publish_recovers() { use durable_file::{CommitStep, failpoint}; for step in [ CommitStep::TempWritten, CommitStep::FileSynced, CommitStep::Published, CommitStep::DirSynced, ] { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let request = restore_request(&bundle, &target); // Stage everything through the real phase functions. let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); let staging = target.join(RESTORE_STAGING_DIR); fs::create_dir_all(&staging).await.expect("create staging"); let salt = decoded.salt.as_ref().expect("bundle salt"); stage_file(&staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, salt, Some(0o600)) .await .expect("stage salt"); for record in &decoded.records { stage_file(&staging, &record.file_name, &record.plaintext, Some(0o600)) .await .expect("stage record"); } let marker = RestoreCommitMarker::for_bundle(&decoded); failpoint::arm(&target, step); let error = publish_marker(&target, &marker, Some(0o600)) .await .expect_err("armed marker commit must simulate a crash"); failpoint::disarm(&target); assert!(error.to_string().contains("injected crash"), "step {step:?}: {error}"); let marker_path = target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); let published = matches!(step, CommitStep::Published | CommitStep::DirSynced); assert_eq!(marker_path.exists(), published, "step {step:?}"); if !published { // The unpublished marker temp is exactly the shape startup // recovery and a restore re-run both classify as removable. let temp = top_level_names(&target) .await .into_iter() .find(|name| is_orphan_commit_temp_name(name)); assert!(temp.is_some(), "step {step:?}: expected a leftover marker temp"); } else { // A published marker makes backend startup fail closed. let error = match LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))).await { Ok(_) => panic!("step {step:?}: startup must fail closed on a published marker"), Err(error) => error, }; assert!(matches!(error, KmsError::ConfigurationError { .. }), "step {step:?}: {error:?}"); assert!(error.to_string().contains("unfinished restore"), "step {step:?}: {error}"); } let report = restore_local_backup(&test_kek(), &request) .await .unwrap_or_else(|error| panic!("step {step:?}: re-run must converge: {error}")); assert_eq!(report.resumed, published, "step {step:?}"); assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; } } #[tokio::test] async fn crash_mid_cutover_fails_startup_then_rolls_forward_or_aborts() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; // Reconstruct the exact state of a cutover killed after k of n links. let build_partial_cutover = |target: PathBuf| { let bundle = bundle.clone(); async move { let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); let staging = target.join(RESTORE_STAGING_DIR); fs::create_dir_all(&staging).await.expect("create staging"); stage_file( &staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, decoded.salt.as_ref().expect("bundle salt"), Some(0o600), ) .await .expect("stage salt"); for record in &decoded.records { stage_file(&staging, &record.file_name, &record.plaintext, Some(0o600)) .await .expect("stage record"); } let marker = RestoreCommitMarker::for_bundle(&decoded); publish_marker(&target, &marker, Some(0o600)).await.expect("publish marker"); // First link only (the salt), then "power loss". durable_file::link_durably( staging.join(LOCAL_KMS_MASTER_KEY_SALT_FILE), target.join(LOCAL_KMS_MASTER_KEY_SALT_FILE), ) .await .expect("link salt"); } }; // Outcome 1: roll forward with the same bundle. let target = work.path().join("target-forward"); build_partial_cutover(target.clone()).await; let error = match LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))).await { Ok(_) => panic!("startup must fail closed mid-cutover"), Err(error) => error, }; assert!(error.to_string().contains("unfinished restore"), "got {error}"); assert!(matches!( LocalKmsClient::new_for_key_export(local_config(&target, Some(TEST_MASTER_KEY))).await, Err(KmsError::ConfigurationError { .. }) )); let report = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) .await .expect("roll-forward must converge"); assert!(report.resumed); assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; // Outcome 2: explicit abort returns the complete old (empty) state. let target = work.path().join("target-abort"); build_partial_cutover(target.clone()).await; abort_local_restore(&target).await.expect("abort must succeed"); assert_eq!( top_level_names(&target).await, Vec::::new(), "abort must take back every published file, the marker, and staging" ); // The aborted target is a healthy empty directory again: both a fresh // backend start and a fresh restore work. restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) .await .expect("a fresh restore after abort must succeed"); assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; } #[tokio::test] async fn roll_forward_requires_identical_file_set() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); fs::create_dir_all(target.join(RESTORE_STAGING_DIR)) .await .expect("create staging"); let mut marker = RestoreCommitMarker::for_bundle(&decoded); marker.files.pop(); publish_marker(&target, &marker, Some(0o600)).await.expect("publish marker"); let error = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) .await .expect_err("a marker with a diverging file set must fail closed"); assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); assert!(error.to_string().contains("different file set"), "got {error}"); } #[tokio::test] async fn completed_restore_tolerates_inert_staging_leftover() { let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; let work = TempDir::new().expect("work dir"); let (bundle, _manifest) = export_bundle(&client, work.path()).await; let target = work.path().join("target"); restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) .await .expect("restore"); // A crash between marker removal and staging cleanup leaves an inert // staging directory; startup must ignore it entirely. let staging = target.join(RESTORE_STAGING_DIR); fs::create_dir_all(&staging).await.expect("recreate staging"); fs::write(staging.join("leftover"), b"inert").await.expect("seed leftover"); let restored = LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))) .await .expect("startup must ignore a leftover staging directory"); restored.describe_key("alpha", None).await.expect("key describes"); assert!(staging.exists(), "startup must not touch the staging leftover"); } #[tokio::test] async fn startup_removes_unpublished_marker_temp() { let (client, source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; drop(client); let temp_name = format!("{LOCAL_RESTORE_COMMIT_MARKER_FILE}.tmp-{}", uuid::Uuid::new_v4()); let temp_path = source_dir.path().join(&temp_name); fs::write(&temp_path, b"unpublished marker").await.expect("seed marker temp"); let client = LocalKmsClient::new(local_config(source_dir.path(), Some(TEST_MASTER_KEY))) .await .expect("an unpublished marker temp must not block startup"); assert!(!temp_path.exists(), "startup recovery must remove the unpublished marker temp"); client.describe_key("alpha", None).await.expect("key describes"); } #[tokio::test] async fn abort_without_restore_in_progress_errors_and_staging_only_is_cleaned() { let empty = TempDir::new().expect("empty dir"); let error = abort_local_restore(empty.path()) .await .expect_err("nothing to abort must be an error"); assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); let staging_only = TempDir::new().expect("staging dir"); let staging = staging_only.path().join(RESTORE_STAGING_DIR); fs::create_dir_all(&staging).await.expect("create staging"); fs::write(staging.join("partial.key"), b"partial") .await .expect("seed partial"); abort_local_restore(staging_only.path()) .await .expect("a pre-marker abort only removes staging"); assert_eq!(top_level_names(staging_only.path()).await, Vec::::new()); } }