gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-04 11:15:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(kms): enforce shared key state machine across backends (#5489)

Browse Source 
* feat(kms): enforce shared key state machine across backends

Unify the key state x operation matrix behind a single gate in
backends/mod.rs and wire it into the Local, Vault KV2 and Vault Transit
backends: Disabled keys reject encryption, data key generation and
rotation while still allowing decryption and lifecycle recovery;
PendingDeletion keys reject everything except decryption and
cancellation (including repeated deletion scheduling); cancellation now
requires an actual pending deletion everywhere. This closes the missing
gates on KV2 encrypt/generate and Local generate_data_key, and stops
enable_key from silently reverting a pending deletion.

Decryption is deliberately left ungated in Disabled/PendingDeletion — an
explicit, documented and tested deviation from AWS KMS, since gating it
would break reads of existing objects the moment a key is disabled.

Add shared contract tests driving the full matrix offline for Local (and
via ignored tests against a live Vault for KV2/Transit), a stateless
contract for Static, an SSE-shaped regression proving existing envelopes
stay decryptable after disable, and a pin on the known-risk Enabled
default of Transit's synthesized metadata fallback.

Refs rustfs/backlog#1571 (part of rustfs/backlog#1562)

* feat(kms): persist deletion deadlines and run a restartable deletion worker (#5491)
This commit is contained in:
Zhengchao An
2026-07-31 07:24:39 +08:00
committed by GitHub
parent 3921336b23
commit 2e29c330a9
10 changed files with 1394 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/kms/src/backends/vault_transit.rs
+105 -13
View File
@@ -18,7 +18,9 @@ use crate::backends::vault_credentials::{
CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider,
token_source_for,
};
use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient};
use crate::backends::{
BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits,
};
use crate::config::{KmsConfig, VaultTransitConfig};
use crate::encryption::{DataKeyEnvelope, generate_key_material};
use crate::error::{KmsError, Result};
@@ -84,6 +86,12 @@ impl TransitKeyMetadata {
}
}
// KNOWN RISK (rustfs/backlog#1571, residual of rustfs/backlog#808): this
// fallback defaults to Enabled, so a key whose KV metadata read fails is
// treated as usable — a disabled or pending-deletion key can transiently
// "revive" on that path. State gates therefore only hold as strongly as
// metadata reads do. Changing the fallback is out of scope here; the
// synthesized_metadata_defaults_to_enabled test pins the current behavior.
fn synthesized() -> Self {
Self {
key_usage: KeyUsage::EncryptDecrypt,
@@ -370,14 +378,9 @@ impl VaultTransitKmsClient {
})
}
async fn ensure_key_active(&self, key_id: &str) -> Result<TransitKeyMetadata> {
async fn ensure_key_state_allows(&self, key_id: &str, operation: StateGatedOperation) -> Result<TransitKeyMetadata> {
let metadata = self.get_key_metadata(key_id).await?;
if metadata.key_state != KeyState::Enabled {
return Err(KmsError::invalid_operation(format!(
"Key {key_id} is not active (state: {:?})",
metadata.key_state
)));
}
ensure_key_state_permits(key_id, &metadata.key_state, operation)?;
Ok(metadata)
}
}
@@ -385,7 +388,8 @@ impl VaultTransitKmsClient {
#[async_trait]
impl KmsClient for VaultTransitKmsClient {
async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> {
self.ensure_key_active(&request.master_key_id).await?;
self.ensure_key_state_allows(&request.master_key_id, StateGatedOperation::GenerateDataKey)
.await?;
let plaintext_key = generate_key_material(&request.key_spec)?;
let encrypted_key = self
@@ -416,7 +420,9 @@ impl KmsClient for VaultTransitKmsClient {
}
async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> {
let metadata = self.ensure_key_active(&request.key_id).await?;
let metadata = self
.ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt)
.await?;
let ciphertext = self
.transit_encrypt(&request.key_id, &request.plaintext, &request.encryption_context)
.await?;
@@ -482,6 +488,7 @@ impl KmsClient for VaultTransitKmsClient {
created_at: metadata.created_at,
rotated_at: None,
created_by: metadata.created_by,
deletion_date: None,
})
}
@@ -528,14 +535,16 @@ impl KmsClient for VaultTransitKmsClient {
}
async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> {
let mut metadata = self.get_key_metadata(key_id).await?;
// A pending deletion must be reverted through cancel_key_deletion, not
// silently by enabling, so the gate rejects PendingDeletion here.
let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Enable).await?;
metadata.key_state = KeyState::Enabled;
metadata.deletion_date = None;
self.store_key_metadata(key_id, &metadata).await
}
async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> {
let mut metadata = self.get_key_metadata(key_id).await?;
let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Disable).await?;
metadata.key_state = KeyState::Disabled;
self.store_key_metadata(key_id, &metadata).await
}
@@ -546,7 +555,9 @@ impl KmsClient for VaultTransitKmsClient {
pending_window_days: u32,
_context: Option<&OperationContext>,
) -> Result<()> {
let mut metadata = self.get_key_metadata(key_id).await?;
let mut metadata = self
.ensure_key_state_allows(key_id, StateGatedOperation::ScheduleDeletion)
.await?;
metadata.key_state = KeyState::PendingDeletion;
metadata.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400));
self.store_key_metadata(key_id, &metadata).await
@@ -554,12 +565,17 @@ impl KmsClient for VaultTransitKmsClient {
async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> {
let mut metadata = self.get_key_metadata(key_id).await?;
if metadata.key_state != KeyState::PendingDeletion {
return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
}
metadata.key_state = KeyState::Enabled;
metadata.deletion_date = None;
self.store_key_metadata(key_id, &metadata).await
}
async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
self.ensure_key_state_allows(key_id, StateGatedOperation::Rotate).await?;
key::rotate(&self.vault()?.client, &self.config.mount_path, key_id)
.await
.map_err(|e| KmsError::backend_error(format!("Failed to rotate Vault Transit key {key_id}: {e}")))?;
@@ -579,6 +595,7 @@ impl KmsClient for VaultTransitKmsClient {
created_at: metadata.created_at,
rotated_at: Some(Zoned::now()),
created_by: metadata.created_by,
deletion_date: None,
})
}
@@ -600,6 +617,14 @@ pub struct VaultTransitKmsBackend {
}
impl VaultTransitKmsBackend {
/// Lifecycle driver for the shared state-machine contract tests. Using the
/// backend's own client keeps its in-process metadata cache coherent with
/// the transitions the tests perform.
#[cfg(test)]
pub(crate) fn lifecycle_client(&self) -> &VaultTransitKmsClient {
&self.client
}
pub async fn new(config: KmsConfig) -> Result<Self> {
config.validate()?;
@@ -736,6 +761,8 @@ impl KmsBackend for VaultTransitKmsBackend {
None
}
} else {
ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"));
@@ -788,6 +815,60 @@ impl KmsBackend for VaultTransitKmsBackend {
.with_versioning(true)
.with_physical_delete(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
// The transit key's existence anchors "already removed": once it is
// gone only stale scheduling metadata can remain, so clean that up.
match self.client.read_transit_key(key_id).await {
Ok(_) => {}
Err(KmsError::KeyNotFound { .. }) => {
self.client.delete_key_metadata(key_id).await?;
return Ok(ExpiredKeyRemoval::Removed);
}
Err(error) => return Err(error),
}
// A metadata read failure synthesizes an Enabled record (see
// TransitKeyMetadata::synthesized), which lands in StateChanged below:
// the worker never destroys material based on synthesized state.
let mut metadata = self.client.get_key_metadata(key_id).await?;
match metadata.key_state {
// Tombstone left by a crashed removal: complete it.
KeyState::Unavailable => {}
KeyState::PendingDeletion => {
match &metadata.deletion_date {
Some(deadline) if deadline <= now => {}
// Not yet due, or no persisted deadline — never auto-remove.
_ => return Ok(ExpiredKeyRemoval::NotExpired),
}
// Tombstone first: an Unavailable record is rejected by every
// state gate, and a crashed removal can simply be re-run.
metadata.key_state = KeyState::Unavailable;
self.client.store_key_metadata(key_id, &metadata).await?;
}
KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => {
return Ok(ExpiredKeyRemoval::StateChanged);
}
}
if !self.client.read_transit_key(key_id).await?.deletion_allowed {
let mut update_builder = UpdateKeyConfigurationRequestBuilder::default();
update_builder.deletion_allowed(true);
key::update(
&self.client.vault()?.client,
&self.client.config.mount_path,
key_id,
Some(&mut update_builder),
)
.await
.map_err(|e| KmsError::backend_error(format!("Failed to allow deletion of Vault Transit key {key_id}: {e}")))?;
}
key::delete(&self.client.vault()?.client, &self.client.config.mount_path, key_id)
.await
.map_err(|e| KmsError::backend_error(format!("Failed to delete Vault Transit key {key_id}: {e}")))?;
self.client.delete_key_metadata(key_id).await?;
Ok(ExpiredKeyRemoval::Removed)
}
}
#[cfg(test)]
@@ -999,4 +1080,15 @@ mod tests {
// Cleanup so repeated runs against the same Vault do not accumulate keys.
let _ = client.schedule_key_deletion(&key_id, 7, None).await;
}
/// Pins the known-risk fallback documented on `TransitKeyMetadata::synthesized`:
/// when KV metadata cannot be read, the synthesized record defaults to Enabled,
/// which weakens every state gate on that path. If this test turns red the
/// fallback semantics changed on purpose — update the comment there as well.
#[test]
fn synthesized_metadata_defaults_to_enabled() {
let metadata = TransitKeyMetadata::synthesized();
assert_eq!(metadata.key_state, KeyState::Enabled);
assert!(metadata.deletion_date.is_none());
}
}