mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 09:18:28 +00:00
test(kms): keep concurrent rotation regression fail-closed
This commit is contained in:
@@ -217,11 +217,15 @@ struct Kv2State {
|
|||||||
|
|
||||||
impl Kv2State {
|
impl Kv2State {
|
||||||
fn new(current_data: serde_json::Value) -> Self {
|
fn new(current_data: serde_json::Value) -> Self {
|
||||||
|
let mut version_records = BTreeMap::new();
|
||||||
|
if current_data["baseline_version"].as_u64() == Some(1) {
|
||||||
|
version_records.insert(1, current_data.clone());
|
||||||
|
}
|
||||||
Self {
|
Self {
|
||||||
history: BTreeMap::from([(1, current_data.clone())]),
|
history: BTreeMap::from([(1, current_data.clone())]),
|
||||||
current_data,
|
current_data,
|
||||||
current_secret_version: 1,
|
current_secret_version: 1,
|
||||||
version_records: BTreeMap::new(),
|
version_records,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -696,15 +696,8 @@ impl VaultKmsClient {
|
|||||||
async fn ensure_version_history_consistent(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> {
|
async fn ensure_version_history_consistent(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> {
|
||||||
let recorded = self.recorded_version_numbers(key_id).await?;
|
let recorded = self.recorded_version_numbers(key_id).await?;
|
||||||
|
|
||||||
// A first-rotation record set (the current v1 record and, at most, the
|
|
||||||
// next v2 record) can be in flight while another caller is between its
|
|
||||||
// create-only writes and the baseline CAS. The material check below
|
|
||||||
// validates the v1 record before adopting it; a later key with a
|
|
||||||
// missing persisted baseline still indicates a lost baseline and fails
|
|
||||||
// closed.
|
|
||||||
if key_data.baseline_version.is_none()
|
if key_data.baseline_version.is_none()
|
||||||
&& let Some(oldest_recorded) = recorded.iter().min().copied()
|
&& let Some(oldest_recorded) = recorded.iter().min().copied()
|
||||||
&& !(key_data.version == 1 && recorded.iter().all(|version| *version <= 2))
|
|
||||||
{
|
{
|
||||||
warn!(
|
warn!(
|
||||||
key_id,
|
key_id,
|
||||||
@@ -3253,7 +3246,9 @@ mod tests {
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
const ATTEMPTS: usize = 8;
|
const ATTEMPTS: usize = 8;
|
||||||
let (vault, client) = scripted_kv2_client(&healthy_key_data()).await;
|
let mut key_data = healthy_key_data();
|
||||||
|
key_data.baseline_version = Some(1);
|
||||||
|
let (vault, client) = scripted_kv2_client(&key_data).await;
|
||||||
let client = Arc::new(client);
|
let client = Arc::new(client);
|
||||||
let barrier = Arc::new(tokio::sync::Barrier::new(ATTEMPTS));
|
let barrier = Arc::new(tokio::sync::Barrier::new(ATTEMPTS));
|
||||||
let tasks: Vec<_> = (0..ATTEMPTS)
|
let tasks: Vec<_> = (0..ATTEMPTS)
|
||||||
@@ -3579,50 +3574,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A concurrent first rotation may leave both immutable records visible
|
|
||||||
/// before this caller observes the baseline CAS. That transient state must
|
|
||||||
/// continue through the create-only/adopt path rather than being reported
|
|
||||||
/// as a permanently lost baseline.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn wired_rotate_recovers_concurrent_first_rotation_without_baseline() {
|
|
||||||
let key_data = healthy_key_data();
|
|
||||||
let record_v2 = VaultKeyVersionRecord {
|
|
||||||
version: 2,
|
|
||||||
encrypted_key_material: rotated_material(),
|
|
||||||
created_at: Zoned::now(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let (vault, client) = scripted_client(vec![
|
|
||||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
|
||||||
ScriptedResponse::ok(kv2_read_data(&key_data)),
|
|
||||||
ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })),
|
|
||||||
// Another rotation already created the baseline record.
|
|
||||||
ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE),
|
|
||||||
ScriptedResponse::ok(kv2_read_version_record_data(&VaultKeyVersionRecord {
|
|
||||||
version: 1,
|
|
||||||
encrypted_key_material: key_data.encrypted_key_material.clone(),
|
|
||||||
created_at: key_data.created_at.clone(),
|
|
||||||
})),
|
|
||||||
// Persist the baseline pointer, then adopt the already-created v2.
|
|
||||||
ScriptedResponse::ok(kv2_write_ack()),
|
|
||||||
ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE),
|
|
||||||
ScriptedResponse::ok(kv2_read_version_record_data(&record_v2)),
|
|
||||||
ScriptedResponse::ok(kv2_write_ack()),
|
|
||||||
])
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let rotated = client
|
|
||||||
.rotate_key("wired-key", None)
|
|
||||||
.await
|
|
||||||
.expect("an in-flight first rotation must be recoverable");
|
|
||||||
assert_eq!(rotated.version, 2);
|
|
||||||
assert_eq!(
|
|
||||||
parse_write_body(&vault.request_bodies()[8])["data"]["encrypted_key_material"],
|
|
||||||
serde_json::json!(record_v2.encrypted_key_material),
|
|
||||||
"the pointer must adopt the immutable v2 material"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The mixed-version corruption path: a node older than versioned rotation
|
/// The mixed-version corruption path: a node older than versioned rotation
|
||||||
/// performed a lifecycle write, which rewrites the whole key record and
|
/// performed a lifecycle write, which rewrites the whole key record and
|
||||||
/// silently drops the `baseline_version` it does not know. Version records
|
/// silently drops the `baseline_version` it does not know. Version records
|
||||||
|
|||||||
Reference in New Issue
Block a user