Merge remote-tracking branch 'origin/main' into reatang/fix-replication-target-health-tls

# Conflicts:
#	crates/ecstore/src/bucket/bucket_target_sys.rs
This commit is contained in:
唐小鸭
2026-08-02 14:52:14 +08:00
103 changed files with 12817 additions and 1043 deletions
@@ -126,6 +126,51 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
assert_eq!(cancelled["success"], true);
assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled");
// A window outside 7-30 days is refused at the endpoint, whatever the
// backend: the bound is enforced once in the service, so no backend can
// stretch or skip it (rustfs/backlog#1585).
for days in [6, 31] {
let refused = kms_admin_request(
base_url,
http::Method::DELETE,
"/rustfs/admin/v3/kms/keys/delete",
Some(
&serde_json::json!({
"key_id": key_id,
"pending_window_in_days": days
})
.to_string(),
),
access_key,
secret_key,
)
.await
.err()
.ok_or_else(|| format!("a {days}-day deletion window must be refused"))?;
assert!(
refused.to_string().contains("400 Bad Request"),
"a {days}-day deletion window must report a client error: {refused}"
);
}
// Immediate deletion is no longer reachable through the query string, so it
// fails before the service gate is even consulted.
let refused = kms_admin_request(
base_url,
http::Method::DELETE,
&format!("/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
None,
access_key,
secret_key,
)
.await
.err()
.ok_or("immediate KMS key deletion must not be reachable through the query string")?;
assert!(
refused.to_string().contains("400 Bad Request"),
"a query-string immediate deletion must report a client error: {refused}"
);
// A default server refuses to skip the waiting window (rustfs/backlog#1585):
// immediate deletion is unrecoverable and takes every object encrypted under
// the key with it, so the endpoint must reject it rather than honour it.
@@ -151,7 +196,7 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
"refused immediate deletion must report a client error: {refused}"
);
// The refused request left the key alone, so the window-bounded path still
// The refused requests left the key alone, so the window-bounded path still
// has something to schedule.
let described = kms_admin_request(
base_url,
@@ -0,0 +1,483 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Negative authorization matrix for per-key KMS access control.
//!
//! Every case here is an end-to-end denial that the pre-`kms` resource server
//! allowed, so a regression that reopens one of them fails this file rather than
//! only a unit test. The matrix varies one dimension at a time:
//!
//! - **wrong identity**: a caller holding S3 rights but no `kms` grant at all
//! - **wrong key**: a caller scoped to key A naming key B
//! - **wrong action**: a caller holding `kms:GenerateDataKey` but not `kms:Decrypt`
//! (and, on the admin plane, `kms:DisableKey` but not `kms:RotateKey`)
//! - **wrong context**: an explicit `Deny` beating a wildcard `Allow`, and SSE-S3
//! staying exempt from `kms` authorization
//!
//! Each matrix opens with a positive control. Without it a denial proves nothing:
//! an identity whose policy has not propagated yet is denied everything.
use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
use crate::common::{admin_ok, admin_request, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Config, Credentials, Region};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use serial_test::serial;
use std::time::Duration;
use tracing::info;
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
const OTHER_KEY: &str = "kms-matrix-other-key";
const BUCKET: &str = "kms-authz-matrix";
const SECRET: &str = "kms-matrix-secret";
const PAYLOAD: &[u8] = b"kms authorization matrix payload";
/// How long an identity change may take to reach the request path.
const IAM_PROPAGATION: Duration = Duration::from_secs(20);
fn s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
let config = Config::builder()
.credentials_provider(Credentials::new(access_key, secret_key, None, None, "kms-authz-matrix"))
.region(Region::new("us-east-1"))
.endpoint_url(url)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
}
/// Start a server whose SSE-KMS data path authorizes against the named key.
///
/// The enforcement switch defaults to off for compatibility, so it has to be set
/// explicitly; without it every negative case below would silently pass as an allow.
async fn start_enforcing_server(env: &mut LocalKMSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
create_key_with_specific_id(&env.kms_keys_dir, ALLOWED_KEY).await?;
create_key_with_specific_id(&env.kms_keys_dir, OTHER_KEY).await?;
let key_dir = env.kms_keys_dir.clone();
let args = vec![
"--kms-enable",
"--kms-backend",
"local",
"--kms-key-dir",
key_dir.as_str(),
"--kms-default-key-id",
ALLOWED_KEY,
];
let mut envs = vec![("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")];
envs.extend_from_slice(extra_env);
env.base_env.start_rustfs_server_with_env(args, &envs).await?;
Ok(())
}
/// Create `user` with `policy_document` attached under a canned policy of the same name.
async fn provision_user(env: &LocalKMSTestEnvironment, user: &str, policy_document: &str) -> TestResult {
admin_ok(
&env.base_env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-canned-policy?name={user}"),
Some(policy_document.to_string()),
)
.await?;
provision_user_with_policy(env, user, user).await
}
/// Create `user` and attach an existing policy (built-in or canned) by name.
async fn provision_user_with_policy(env: &LocalKMSTestEnvironment, user: &str, policy_name: &str) -> TestResult {
admin_ok(
&env.base_env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
Some(serde_json::json!({ "secretKey": SECRET, "status": "enabled" }).to_string()),
)
.await?;
admin_ok(
&env.base_env,
http::Method::PUT,
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={user}&isGroup=false"),
None,
)
.await?;
Ok(())
}
/// The S3 half of every data-path policy below: full object access, no KMS grant.
fn s3_full_access_statement() -> serde_json::Value {
serde_json::json!({
"Effect": "Allow",
"Action": ["s3:*"],
"Resource": ["arn:aws:s3:::*"]
})
}
fn policy_document(statements: Vec<serde_json::Value>) -> String {
serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
}
async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
client
.put_object()
.bucket(BUCKET)
.key(key)
.body(ByteStream::from_static(PAYLOAD))
.server_side_encryption(ServerSideEncryption::AwsKms)
.ssekms_key_id(kms_key_id)
.send()
.await
.map(|_| ())
.map_err(aws_sdk_s3::Error::from)
}
/// Assert the operation failed with `AccessDenied` rather than any other error.
///
/// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
/// would hide both a leak of key state and an outage masquerading as a denial.
fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
let error = result.expect_err(&format!("{what} must be denied"));
assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
}
/// Retry an SSE-KMS write until the identity's policy has reached the request path.
async fn wait_for_sse_kms_write(client: &Client, key: &str, kms_key_id: &str) -> TestResult {
let deadline = tokio::time::Instant::now() + IAM_PROPAGATION;
loop {
match put_sse_kms(client, key, kms_key_id).await {
Ok(()) => return Ok(()),
Err(error) if tokio::time::Instant::now() >= deadline => {
return Err(format!("positive control never became authorized: {error:?}").into());
}
Err(_) => tokio::time::sleep(Duration::from_millis(500)).await,
}
}
}
/// Retry an admin call until it stops returning 403, i.e. the policy is live.
async fn wait_for_admin_success(
env: &LocalKMSTestEnvironment,
user: &str,
method: http::Method,
path: &str,
body: Option<String>,
) -> TestResult {
let deadline = tokio::time::Instant::now() + IAM_PROPAGATION;
loop {
let (status, response) = admin_request(&env.base_env.url, method.clone(), path, body.clone(), user, SECRET).await?;
if status.is_success() {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("positive control never became authorized: {method} {path} -> {status} {response}").into());
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
async fn assert_admin_denied(
env: &LocalKMSTestEnvironment,
user: &str,
method: http::Method,
path: &str,
body: Option<String>,
what: &str,
) -> TestResult {
let (status, response) = admin_request(&env.base_env.url, method, path, body, user, SECRET).await?;
assert_eq!(status.as_u16(), 403, "{what} must be denied, got {status}: {response}");
assert!(response.contains("AccessDenied"), "{what} must carry AccessDenied: {response}");
Ok(())
}
fn disable_body(key_id: &str) -> String {
serde_json::json!({ "key_id": key_id }).to_string()
}
/// Data-path matrix: SSE-KMS writes and reads are authorized against the resolved key.
#[tokio::test]
#[serial]
async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
init_logging();
let mut env = LocalKMSTestEnvironment::new().await?;
start_enforcing_server(&mut env, &[("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true")]).await?;
env.base_env.create_test_bucket(BUCKET).await?;
// Scoped to ALLOWED_KEY only.
provision_user(
&env,
"kmsmatrixscoped",
&policy_document(vec![
s3_full_access_statement(),
serde_json::json!({
"Effect": "Allow",
"Action": ["kms:GenerateDataKey", "kms:Decrypt"],
"Resource": [format!("arn:aws:kms:::key/{ALLOWED_KEY}")]
}),
]),
)
.await?;
// S3 rights only: the identity shape that existed before per-key authorization.
provision_user(&env, "kmsmatrixs3only", &policy_document(vec![s3_full_access_statement()])).await?;
// May wrap a data key but may never unwrap one.
provision_user(
&env,
"kmsmatrixwriter",
&policy_document(vec![
s3_full_access_statement(),
serde_json::json!({
"Effect": "Allow",
"Action": ["kms:GenerateDataKey"],
"Resource": ["arn:aws:kms:::*"]
}),
]),
)
.await?;
// Wildcard allow, explicit deny on one key.
provision_user(
&env,
"kmsmatrixdenied",
&policy_document(vec![
s3_full_access_statement(),
serde_json::json!({
"Effect": "Allow",
"Action": ["kms:*"],
"Resource": ["arn:aws:kms:::*"]
}),
serde_json::json!({
"Effect": "Deny",
"Action": ["kms:*"],
"Resource": [format!("arn:aws:kms:::key/{OTHER_KEY}")]
}),
]),
)
.await?;
let scoped = s3_client(&env.base_env.url, "kmsmatrixscoped", SECRET);
let s3_only = s3_client(&env.base_env.url, "kmsmatrixs3only", SECRET);
let writer = s3_client(&env.base_env.url, "kmsmatrixwriter", SECRET);
let denied = s3_client(&env.base_env.url, "kmsmatrixdenied", SECRET);
// --- positive control -----------------------------------------------------
wait_for_sse_kms_write(&scoped, "scoped/allowed", ALLOWED_KEY).await?;
let read = scoped.get_object().bucket(BUCKET).key("scoped/allowed").send().await?;
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), PAYLOAD);
info!("positive control: scoped identity may write and read under its own key");
// --- wrong key ------------------------------------------------------------
assert_access_denied(
put_sse_kms(&scoped, "scoped/other", OTHER_KEY).await,
"SSE-KMS write under a key outside the identity's scope",
);
// --- wrong identity -------------------------------------------------------
assert_access_denied(
put_sse_kms(&s3_only, "s3only/allowed", ALLOWED_KEY).await,
"SSE-KMS write by an identity holding no kms grant",
);
// The object the scoped identity wrote is readable by its owner only.
assert_access_denied(
s3_only
.get_object()
.bucket(BUCKET)
.key("scoped/allowed")
.send()
.await
.map(|_| ())
.map_err(aws_sdk_s3::Error::from),
"SSE-KMS read by an identity holding no kms grant",
);
// --- wrong action ---------------------------------------------------------
wait_for_sse_kms_write(&writer, "writer/allowed", ALLOWED_KEY).await?;
assert_access_denied(
writer
.get_object()
.bucket(BUCKET)
.key("writer/allowed")
.send()
.await
.map(|_| ())
.map_err(aws_sdk_s3::Error::from),
"SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
);
// --- wrong context: explicit Deny beats a wildcard Allow -------------------
wait_for_sse_kms_write(&denied, "denied/allowed", ALLOWED_KEY).await?;
assert_access_denied(
put_sse_kms(&denied, "denied/other", OTHER_KEY).await,
"SSE-KMS write under a key covered by an explicit Deny",
);
// --- wrong context: SSE-S3 is out of scope --------------------------------
// SSE-S3 wraps its data key with a server-owned key the caller never names, so
// it must stay reachable for an identity with no kms grant at all.
s3_only
.put_object()
.bucket(BUCKET)
.key("s3only/sse-s3")
.body(ByteStream::from_static(PAYLOAD))
.server_side_encryption(ServerSideEncryption::Aes256)
.send()
.await?;
let sse_s3_read = s3_only.get_object().bucket(BUCKET).key("s3only/sse-s3").send().await?;
assert_eq!(sse_s3_read.body.collect().await?.into_bytes().as_ref(), PAYLOAD);
// ... and so must an unencrypted object.
s3_only
.put_object()
.bucket(BUCKET)
.key("s3only/plain")
.body(ByteStream::from_static(PAYLOAD))
.send()
.await?;
s3_only.get_object().bucket(BUCKET).key("s3only/plain").send().await?;
Ok(())
}
/// Admin-plane matrix: KMS key endpoints are authorized against the key they name.
///
/// Runs without the SSE enforcement switch: admin scoping is unconditional, and
/// leaving the switch off proves the two planes are independent.
#[tokio::test]
#[serial]
async fn kms_admin_per_key_authorization_negative_matrix() -> TestResult {
init_logging();
let mut env = LocalKMSTestEnvironment::new().await?;
start_enforcing_server(&mut env, &[]).await?;
// Built-in role templates, attached by name.
provision_user_with_policy(&env, "kmsmatrixkeyadmin", "KMSKeyAdministrator").await?;
provision_user_with_policy(&env, "kmsmatrixauditor", "KMSAuditor").await?;
// A narrowed copy of the administrator template, scoped to one key.
provision_user(
&env,
"kmsmatrixscopedadmin",
&policy_document(vec![serde_json::json!({
"Effect": "Allow",
"Action": ["kms:DisableKey", "kms:EnableKey"],
"Resource": [format!("arn:aws:kms:::key/{ALLOWED_KEY}")]
})]),
)
.await?;
// --- positive control -----------------------------------------------------
wait_for_admin_success(
&env,
"kmsmatrixkeyadmin",
http::Method::POST,
"/rustfs/admin/v3/kms/keys/disable",
Some(disable_body(OTHER_KEY)),
)
.await?;
admin_request(
&env.base_env.url,
http::Method::POST,
"/rustfs/admin/v3/kms/keys/enable",
Some(disable_body(OTHER_KEY)),
"kmsmatrixkeyadmin",
SECRET,
)
.await?;
// --- wrong action: the administrator template withholds service-wide powers -
assert_admin_denied(
&env,
"kmsmatrixkeyadmin",
http::Method::GET,
"/rustfs/admin/v3/kms/config",
None,
"KMSKeyAdministrator reading the KMS backend configuration (kms:Configure)",
)
.await?;
assert_admin_denied(
&env,
"kmsmatrixkeyadmin",
http::Method::GET,
"/rustfs/admin/v3/kms/backup",
None,
"KMSKeyAdministrator exporting a backup bundle (kms:Backup)",
)
.await?;
// Separation of duties: managing a key never implies using it.
assert_admin_denied(
&env,
"kmsmatrixkeyadmin",
http::Method::POST,
"/rustfs/admin/v3/kms/generate-data-key",
Some(serde_json::json!({ "key_id": ALLOWED_KEY }).to_string()),
"KMSKeyAdministrator generating a data key (kms:GenerateDataKey)",
)
.await?;
// --- wrong action: the auditor template is read-only ----------------------
wait_for_admin_success(&env, "kmsmatrixauditor", http::Method::GET, "/rustfs/admin/v3/kms/keys", None).await?;
assert_admin_denied(
&env,
"kmsmatrixauditor",
http::Method::POST,
"/rustfs/admin/v3/kms/keys/disable",
Some(disable_body(ALLOWED_KEY)),
"KMSAuditor disabling a key (kms:DisableKey)",
)
.await?;
// --- wrong key ------------------------------------------------------------
wait_for_admin_success(
&env,
"kmsmatrixscopedadmin",
http::Method::POST,
"/rustfs/admin/v3/kms/keys/disable",
Some(disable_body(ALLOWED_KEY)),
)
.await?;
assert_admin_denied(
&env,
"kmsmatrixscopedadmin",
http::Method::POST,
"/rustfs/admin/v3/kms/keys/disable",
Some(disable_body(OTHER_KEY)),
"key-scoped administrator disabling a key outside its scope",
)
.await?;
// --- wrong action, same key ----------------------------------------------
assert_admin_denied(
&env,
"kmsmatrixscopedadmin",
http::Method::POST,
"/rustfs/admin/v3/kms/keys/rotate",
Some(disable_body(ALLOWED_KEY)),
"key-scoped administrator rotating a key it may only enable and disable",
)
.await?;
// --- wrong identity -------------------------------------------------------
provision_user(&env, "kmsmatrixnokms", &policy_document(vec![s3_full_access_statement()])).await?;
assert_admin_denied(
&env,
"kmsmatrixnokms",
http::Method::POST,
"/rustfs/admin/v3/kms/keys/disable",
Some(disable_body(ALLOWED_KEY)),
"identity holding no kms grant disabling a key",
)
.await?;
Ok(())
}
+20 -3
View File
@@ -417,6 +417,22 @@ async fn test_vault_kms_key_crud(
info!("✅ Read: Successfully listed keys, found test key");
// A waiting window outside 7-30 days is refused at the endpoint for this
// backend too: the bound is enforced once in the service (rustfs/backlog#1585).
for days in [6, 31] {
let window_error = crate::common::execute_awscurl(
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&pending_window_in_days={days}"),
"DELETE",
None,
access_key,
secret_key,
)
.await
.err()
.ok_or_else(|| format!("A {days}-day deletion window must be refused"))?;
info!("✅ Delete window {} correctly refused: {}", days, window_error);
}
// Delete
let delete_response = crate::common::execute_awscurl(
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}"),
@@ -449,9 +465,10 @@ async fn test_vault_kms_key_crud(
info!("✅ Delete verification: Key state correctly changed to: {}", key_state);
// Force Delete - a default server refuses to skip the waiting window
// (rustfs/backlog#1585): destroying the key material immediately would take
// every object encrypted under the key with it.
// Force Delete - the query string can no longer ask for immediate deletion,
// and a default server refuses it in any case (rustfs/backlog#1585):
// destroying the key material immediately would take every object encrypted
// under the key with it.
let force_delete_error = crate::common::execute_awscurl(
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
"DELETE",
+3
View File
@@ -53,3 +53,6 @@ mod copy_object_version_restore_sse_test;
#[cfg(test)]
mod configured_roundtrip_test;
#[cfg(test)]
mod kms_authorization_negative_matrix_test;
@@ -1507,6 +1507,10 @@ async fn build_sse_replication_pair(
("RUSTFS_KMS_KEY_DIR", source_kms_key_dir.as_str()),
("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID),
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
// Per-key KMS authorization is on so this contract is pinned in the
// configuration replication will eventually ship with: the replication
// worker carries no request identity and must stay exempt.
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
]);
}
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
@@ -1519,6 +1523,7 @@ async fn build_sse_replication_pair(
("RUSTFS_KMS_KEY_DIR", target_kms_key_dir.as_str()),
("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID),
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
]);
}
target_env
+17 -15
View File
@@ -173,23 +173,25 @@ pub mod bucket {
pub mod replication {
pub use crate::bucket::replication::replication_pool::{
DurableMrfBacklogSummary, DurableMrfBucketBacklog, MrfBacklogObservabilitySummary, MrfBucketBacklogObservability,
durable_mrf_backlog_summary_snapshot, mrf_backlog_observability_snapshot,
DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
MrfBucketBacklogObservability, durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot,
mrf_backlog_observability_snapshot,
};
pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
ReplicateObjectInfo, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
ReplicationType, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus, VersionPurgeStatusType,
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
get_global_replication_stats, init_background_replication, read_durable_mrf_backlog, replication_state_to_filemeta,
replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
should_use_existing_delete_replication_source, unsupported_replication_config_field,
validate_replication_config_target_arns, version_purge_status_to_filemeta,
BucketReplicationResyncStatus, BucketStats, DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo,
DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts,
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationConfig, ReplicationConfigurationExt,
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
TargetReplicationResyncStatus, VersionPurgeStatusType, delete_replication_state_from_config,
delete_replication_version_id, get_global_replication_pool, get_global_replication_stats,
init_background_replication, invalid_replication_config_status_field, read_durable_mrf_backlog,
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
unsupported_replication_config_field, validate_replication_config_target_arns, version_purge_status_to_filemeta,
};
}
@@ -2136,6 +2136,49 @@ mod tests {
use super::*;
use rcgen::generate_simple_self_signed;
#[derive(Clone, Debug)]
struct RecordingHttpConnector {
request_uris: Arc<std::sync::Mutex<Vec<String>>>,
}
impl SmithyHttpConnector for RecordingHttpConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
self.request_uris
.lock()
.expect("recorded request lock should not be poisoned")
.push(request.uri().to_string());
HttpConnectorFuture::ready(Ok(HttpResponse::new(
aws_smithy_runtime_api::http::StatusCode::try_from(204_u16).expect("204 should be a valid response status"),
SdkBody::empty(),
)))
}
}
fn recording_target_client() -> (TargetClient, Arc<std::sync::Mutex<Vec<String>>>) {
let request_uris = Arc::new(std::sync::Mutex::new(Vec::new()));
let connector = SharedHttpConnector::new(RecordingHttpConnector {
request_uris: Arc::clone(&request_uris),
});
let http_client = http_client_fn(move |_settings, _components| connector.clone());
let client = s3_client_for_test(443, Some(http_client));
(
TargetClient {
endpoint: "https://localhost:443".to_string(),
credentials: None,
bucket: "target-bucket".to_string(),
storage_class: String::new(),
disable_proxy: false,
arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
reset_id: String::new(),
secure: true,
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
client: Arc::new(client),
},
request_uris,
)
}
fn spawn_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>, requests: usize) -> (u16, std::thread::JoinHandle<()>) {
use std::io::{Read, Write};
@@ -2606,6 +2649,32 @@ mod tests {
assert_eq!(got.as_deref(), Some(vid.as_str()));
}
#[tokio::test]
async fn remove_object_writes_null_purge_and_omits_marker_creation_version_queries() {
let (client, request_uris) = recording_target_client();
client
.remove_object("target-bucket", "object", Some("null".to_string()), remove_opts(true, false))
.await
.expect("explicit null version purge should reach the target client");
client
.remove_object("target-bucket", "object", Some(Uuid::new_v4().to_string()), remove_opts(true, true))
.await
.expect("delete marker creation should reach the target client");
let request_uris = request_uris.lock().expect("recorded request lock should not be poisoned");
assert_eq!(request_uris.len(), 2);
assert!(
request_uris[0].contains("versionId=null"),
"an explicit null purge must be emitted as a target versionId query: {}",
request_uris[0]
);
assert!(
!request_uris[1].contains("versionId="),
"delete marker creation must omit the target versionId query: {}",
request_uris[1]
);
}
#[test]
fn put_object_headers_include_non_empty_source_etag_only() {
let mut opts = PutObjectOptions::default();
+88 -4
View File
@@ -16,6 +16,7 @@ use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use super::object_lock::ObjectLockApi;
use super::versioning::VersioningApi;
use super::{quota::BucketQuota, target::BucketTargets};
use crate::bucket::replication::invalid_replication_config_status_field;
use crate::bucket::utils::deserialize;
use crate::config::com::{read_config, save_config};
use crate::disk::BUCKET_META_PREFIX;
@@ -25,9 +26,9 @@ use crate::store::ECStore;
use byteorder::{BigEndian, ByteOrder, LittleEndian};
use rustfs_policy::policy::BucketPolicy;
use s3s::dto::{
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration, RequestPaymentConfiguration,
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
};
use serde::Serializer;
use sha2::{Digest, Sha256};
@@ -751,11 +752,33 @@ impl BucketMetadata {
self.object_lock_config_updated_at = updated;
}
BUCKET_VERSIONING_CONFIG => {
let config = if data.is_empty() {
None
} else {
let config = deserialize::<VersioningConfiguration>(&data)?;
if config.status.as_ref().is_some_and(|status| {
!matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
}) {
return Err(Error::other("bucket versioning configuration has an invalid status"));
}
Some(config)
};
self.versioning_config_xml = data;
self.versioning_config = config;
self.versioning_config_updated_at = updated;
}
BUCKET_REPLICATION_CONFIG => {
let config = if data.is_empty() {
None
} else {
let config = deserialize::<ReplicationConfiguration>(&data)?;
if let Some(field) = invalid_replication_config_status_field(&config) {
return Err(Error::other(format!("replication field {field} has an invalid status")));
}
Some(config)
};
self.replication_config_xml = data;
self.replication_config = config;
self.replication_config_updated_at = updated;
}
BUCKET_TARGETS_FILE => {
@@ -825,7 +848,6 @@ impl BucketMetadata {
/// ambient (first) one. [`BucketMetadata::save`] keeps the ambient default.
pub async fn save_with_store(&mut self, store: std::sync::Arc<crate::store::ECStore>) -> Result<()> {
self.parse_all_configs()?;
let mut buf: Vec<u8> = vec![0; 4];
LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT);
@@ -906,6 +928,7 @@ impl BucketMetadata {
"Failed to parse bucket metadata config"
);
}
self.versioning_config = None;
if !self.versioning_config_xml.is_empty()
&& let Err(e) =
deserialize::<VersioningConfiguration>(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c))
@@ -960,6 +983,7 @@ impl BucketMetadata {
"Failed to parse bucket metadata config"
);
}
self.replication_config = None;
if !self.replication_config_xml.is_empty()
&& let Err(e) =
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
@@ -1345,6 +1369,66 @@ mod test {
assert!(bm.tagging_config.is_none());
}
#[test]
fn delete_admission_configs_update_parsed_state_atomically() {
let mut bm = BucketMetadata::new("test-bucket");
let versioning_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
let replication_xml = b"<ReplicationConfiguration><Role>arn:aws:s3:::target-bucket</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix></Prefix><Destination><Bucket>arn:aws:s3:::target-bucket</Bucket></Destination></Rule></ReplicationConfiguration>";
bm.update_config(BUCKET_VERSIONING_CONFIG, versioning_xml.to_vec())
.expect("valid versioning config should update parsed state");
bm.update_config(BUCKET_REPLICATION_CONFIG, replication_xml.to_vec())
.expect("valid replication config should update parsed state");
assert!(bm.versioning_config.as_ref().is_some_and(VersioningConfiguration::enabled));
assert_eq!(
bm.replication_config.as_ref().map(|config| config.role.as_str()),
Some("arn:aws:s3:::target-bucket")
);
assert!(
bm.update_config(BUCKET_VERSIONING_CONFIG, b"<VersioningConfiguration>".to_vec())
.is_err()
);
assert!(
bm.update_config(BUCKET_REPLICATION_CONFIG, b"<ReplicationConfiguration>".to_vec())
.is_err()
);
assert_eq!(bm.versioning_config_xml, versioning_xml);
assert_eq!(bm.replication_config_xml, replication_xml);
assert!(bm.versioning_config.as_ref().is_some_and(VersioningConfiguration::enabled));
assert_eq!(
bm.replication_config.as_ref().map(|config| config.role.as_str()),
Some("arn:aws:s3:::target-bucket")
);
assert!(
bm.update_config(
BUCKET_VERSIONING_CONFIG,
b"<VersioningConfiguration><Status>Enabld</Status></VersioningConfiguration>".to_vec(),
)
.is_err()
);
assert!(
bm.update_config(
BUCKET_REPLICATION_CONFIG,
b"<ReplicationConfiguration><Role>arn:aws:s3:::target-bucket</Role><Rule><ID>rule1</ID><Status>Enabld</Status><Prefix></Prefix><Destination><Bucket>arn:aws:s3:::target-bucket</Bucket></Destination></Rule></ReplicationConfiguration>".to_vec(),
)
.is_err()
);
assert_eq!(bm.versioning_config_xml, versioning_xml);
assert_eq!(bm.replication_config_xml, replication_xml);
bm.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
bm.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
bm.parse_all_configs()
.expect("bulk config parsing reports malformed fields through cleared typed state");
assert!(bm.versioning_config.is_none());
assert!(bm.replication_config.is_none());
}
#[tokio::test]
async fn marshal_msg_complete_example() {
// Create a complete BucketMetadata with various configurations
+28 -3
View File
@@ -1027,7 +1027,6 @@ impl BucketMetadataSys {
/// server's metadata never leaks into the ambient (first) instance.
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
let mut bm = bm;
bm.save_with_store(self.api.clone()).await?;
self.set(bm.name.clone(), Arc::new(bm)).await;
@@ -1221,7 +1220,9 @@ impl BucketMetadataSys {
}
};
if let Some(config) = &bm.versioning_config {
if !bm.versioning_config_xml.is_empty() && bm.versioning_config.is_none() {
Err(Error::other("persisted bucket versioning configuration is invalid"))
} else if let Some(config) = &bm.versioning_config {
Ok((config.clone(), bm.versioning_config_updated_at))
} else {
Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at))
@@ -1407,7 +1408,9 @@ impl BucketMetadataSys {
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.replication_config {
if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
Err(Error::other("persisted bucket replication configuration is invalid"))
} else if let Some(config) = &bm.replication_config {
Ok((config.clone(), bm.replication_config_updated_at))
} else {
Err(Error::ConfigNotFound)
@@ -1481,6 +1484,28 @@ mod tests {
use serial_test::serial;
use tokio::time::timeout;
#[tokio::test]
async fn malformed_delete_configs_are_not_treated_as_absent() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let bucket = "malformed-delete-config";
let mut metadata = BucketMetadata::new(bucket);
metadata.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
metadata.versioning_config = None;
metadata.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
metadata.replication_config = None;
sys.set(bucket.to_string(), Arc::new(metadata)).await;
assert!(
sys.get_versioning_config(bucket).await.is_err(),
"malformed versioning metadata must block destructive requests"
);
assert!(
sys.get_replication_config(bucket).await.is_err(),
"malformed replication metadata must not be reported as ConfigNotFound"
);
}
/// Concurrent cache misses for one bucket must collapse into a single disk
/// load.
///
+5 -4
View File
@@ -45,8 +45,9 @@ mod runtime_boundary;
pub use datatypes::ResyncStatusType;
pub use replication_config_boundary::{
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
validate_replication_config_target_arns,
};
#[cfg(test)]
pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
@@ -62,7 +63,7 @@ pub(crate) use replication_filemeta_boundary::{
pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
pub use replication_object_bridge::ReplicationObjectBridge;
pub use replication_object_config::ReplicationConfig;
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
pub use replication_object_decision_boundary::{
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, delete_replication_state_from_config,
delete_replication_version_id, should_schedule_delete_replication, should_use_existing_delete_replication_info,
@@ -78,7 +79,7 @@ pub use replication_queue_boundary::{
};
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
pub use replication_scanner_bridge::ReplicationScannerBridge;
pub use replication_state::ReplicationStats;
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
pub use replication_stats_boundary::BucketStats;
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
@@ -13,6 +13,7 @@
// limitations under the License.
pub use rustfs_replication::{
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
ObjectOpts, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_target_arns,
};
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_filemeta::NULL_VERSION_ID;
pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
pub(crate) use rustfs_replication::{
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
@@ -12,14 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::bucket::metadata_sys;
use std::sync::Arc;
use crate::bucket::{metadata::BucketMetadata, metadata_sys};
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::runtime::instance::InstanceContext;
use rustfs_utils::path::path_join_buf;
use s3s::dto::ReplicationConfiguration;
use time::OffsetDateTime;
use super::replication_error_boundary::{Error, Result};
pub(crate) type ReplicationInstanceContext = InstanceContext;
const REPLICATION_DIR: &str = ".replication";
const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -45,6 +50,20 @@ impl ReplicationMetadataStore {
Ok(config)
}
pub(crate) async fn delete_metadata(bucket: &str) -> Result<Arc<BucketMetadata>> {
let sys = metadata_sys::get_bucket_metadata_sys()?;
let sys = sys.read().await;
Ok(sys.get_config(bucket).await?.0)
}
pub(crate) async fn delete_metadata_in(ctx: &ReplicationInstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
let sys = ctx
.bucket_metadata_sys()
.ok_or_else(|| Error::other("request instance bucket metadata system is not initialized"))?;
let sys = sys.read().await;
Ok(sys.get_config(bucket).await?.0)
}
pub(crate) fn rustfs_meta_bucket() -> &'static str {
RUSTFS_META_BUCKET
}
@@ -16,14 +16,17 @@ use std::{collections::HashMap, sync::Arc};
use super::replication_error_boundary::Result;
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicationStatusType, ReplicationType};
use super::replication_metadata_boundary::ReplicationInstanceContext;
use super::replication_object_config::{
check_replicate_delete, check_replicate_delete_strict, get_must_replicate_options, must_replicate,
DeleteReplicationConfigSnapshot, check_replicate_delete, check_replicate_delete_strict, check_replicate_delete_with_snapshot,
get_must_replicate_options, load_delete_replication_config_in, load_delete_request_config_in, must_replicate,
};
use super::replication_object_decision_boundary::MustReplicateOptions;
use super::replication_pool::{schedule_replication, schedule_replication_delete};
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
use super::replication_storage_boundary::{
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationStorage, deleted_object_for_replication,
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationObjectStore, ReplicationStorage,
deleted_object_for_replication,
};
pub struct ReplicationObjectBridge;
@@ -63,6 +66,39 @@ impl ReplicationObjectBridge {
check_replicate_delete_strict(bucket, object, source, opts, get_error).await
}
pub async fn delete_request_config(api: &ReplicationObjectStore, bucket: &str) -> Result<DeleteReplicationConfigSnapshot> {
load_delete_request_config_in(&api.ctx, bucket).await
}
pub(crate) async fn delete_request_config_in(
ctx: &ReplicationInstanceContext,
bucket: &str,
) -> Result<DeleteReplicationConfigSnapshot> {
load_delete_request_config_in(ctx, bucket).await
}
pub(crate) async fn delete_config_snapshot_in(
ctx: &ReplicationInstanceContext,
bucket: &str,
opts: &ObjectOptions,
) -> Result<DeleteReplicationConfigSnapshot> {
load_delete_replication_config_in(ctx, bucket, opts).await
}
pub fn has_active_delete_rule(snapshot: &DeleteReplicationConfigSnapshot, object: &str) -> bool {
snapshot.has_active_rule(object)
}
pub fn check_delete_with_snapshot(
object: &ObjectToDelete,
source: &ObjectInfo,
opts: &ObjectOptions,
source_error: bool,
snapshot: &DeleteReplicationConfigSnapshot,
) -> ReplicateDecision {
check_replicate_delete_with_snapshot(object, source, opts, source_error, snapshot)
}
pub async fn schedule_object<S: ReplicationStorage>(
object: ObjectInfo,
storage: Arc<S>,
@@ -12,23 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::{collections::HashMap, fmt, sync::Arc};
use crate::bucket::metadata::BucketMetadata;
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
use s3s::dto::ReplicationConfiguration;
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
use serde::{Deserialize, Serialize};
use tracing::error;
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt as _};
use super::replication_config_boundary::{
ObjectOpts, ReplicationConfigurationExt as _, ReplicationRuleExt as _, invalid_replication_config_status_field,
};
use super::replication_error_boundary::Result;
use super::replication_filemeta_boundary::{
ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, ReplicationType, ResyncDecision,
};
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
use super::replication_metadata_boundary::ReplicationMetadataStore;
use super::replication_metadata_boundary::{ReplicationInstanceContext, ReplicationMetadataStore};
use super::replication_object_decision_boundary::{
MustReplicateOptions, ReplicationDeleteSource, ReplicationResyncTargetObject, delete_replication_missing_source_decision,
delete_replication_object_opts, resync_target_for_object,
delete_replication_object_opts, heal_uses_delete_replication_path, resync_target_for_object,
};
use super::replication_storage_boundary::{ObjectInfo, ObjectOptions, ObjectToDelete, object_to_delete_for_replication};
use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
@@ -36,7 +39,197 @@ use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources;
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
ReplicationMetadataStore::optional_replication_config(bucket).await
let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
Ok(config)
}
#[derive(Default)]
pub struct DeleteReplicationConfigSnapshot {
metadata: Option<Arc<BucketMetadata>>,
versioning: VersioningConfiguration,
}
impl fmt::Debug for DeleteReplicationConfigSnapshot {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("DeleteReplicationConfigSnapshot")
.field("has_replication_config", &self.replication_config().is_some())
.field("versioning_status", &self.versioning.status)
.finish()
}
}
impl DeleteReplicationConfigSnapshot {
#[cfg(test)]
pub(crate) fn from_configs_for_test(
versioning: VersioningConfiguration,
replication: Option<ReplicationConfiguration>,
) -> Self {
let metadata = replication.map(|config| {
let mut metadata = BucketMetadata::new("test-bucket");
metadata.replication_config = Some(config);
Arc::new(metadata)
});
Self { metadata, versioning }
}
pub fn versioning_config(&self) -> &VersioningConfiguration {
&self.versioning
}
pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
self.metadata
.as_ref()
.and_then(|metadata| metadata.replication_config.as_ref())
}
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
self.replication_config()
.is_some_and(|config| config.has_active_rules(object, true))
}
pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
self.replication_config().is_some_and(|config| {
config.rules.iter().any(|rule| {
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
return false;
}
if !object.starts_with(rule.prefix()) {
return false;
}
rule.filter.as_ref().is_some_and(|filter| {
filter.tag.is_some()
|| filter
.and
.as_ref()
.and_then(|and| and.tags.as_ref())
.is_some_and(|tags| !tags.is_empty())
})
})
})
}
}
fn validate_delete_replication_config(
versioning: &VersioningConfiguration,
config: Option<&ReplicationConfiguration>,
) -> Result<()> {
if versioning
.status
.as_ref()
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
{
return Err(super::replication_error_boundary::Error::other(
"bucket versioning configuration has an invalid status",
));
}
if let Some(config) = config {
if let Some(field) = invalid_replication_config_status_field(config) {
return Err(super::replication_error_boundary::Error::other(format!(
"replication field {field} has an invalid status"
)));
}
let role = config.role.trim();
let mut role_destination = None;
for rule in &config.rules {
if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
let destination = rule.destination.bucket.trim();
if role.is_empty() && destination.is_empty() {
return Err(super::replication_error_boundary::Error::other(
"enabled replication rule has no destination ARN",
));
}
if !role.is_empty() && !destination.is_empty() {
match role_destination {
Some(existing) if existing != destination => {
return Err(super::replication_error_boundary::Error::other(
"replication role cannot address multiple active destinations",
));
}
None => role_destination = Some(destination),
_ => {}
}
}
}
}
}
Ok(())
}
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
return Err(super::replication_error_boundary::Error::other(
"persisted bucket replication configuration is invalid",
));
}
Ok(metadata.replication_config.as_ref())
}
fn delete_request_snapshot_from_metadata(metadata: Arc<BucketMetadata>) -> Result<DeleteReplicationConfigSnapshot> {
if !metadata.versioning_config_xml.is_empty() && metadata.versioning_config.is_none() {
return Err(super::replication_error_boundary::Error::other(
"persisted bucket versioning configuration is invalid",
));
}
let versioning = metadata.versioning_config.clone().unwrap_or_default();
let has_config = {
let config = replication_config_from_metadata(&metadata)?;
if versioning.status.is_none() && config.is_some() {
return Err(super::replication_error_boundary::Error::other(
"bucket replication configuration requires versioning",
));
}
validate_delete_replication_config(&versioning, config)?;
config.is_some()
};
Ok(DeleteReplicationConfigSnapshot {
metadata: has_config.then_some(metadata),
versioning,
})
}
fn delete_snapshot_from_metadata(metadata: Arc<BucketMetadata>) -> Result<DeleteReplicationConfigSnapshot> {
let has_config = {
let config = replication_config_from_metadata(&metadata)?;
validate_delete_replication_config(&VersioningConfiguration::default(), config)?;
config.is_some()
};
Ok(DeleteReplicationConfigSnapshot {
metadata: has_config.then_some(metadata),
versioning: VersioningConfiguration::default(),
})
}
pub(crate) async fn load_delete_request_config_in(
ctx: &ReplicationInstanceContext,
bucket: &str,
) -> Result<DeleteReplicationConfigSnapshot> {
delete_request_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata_in(ctx, bucket).await?)
}
pub(crate) async fn load_delete_replication_config(
bucket: &str,
opts: &ObjectOptions,
) -> Result<DeleteReplicationConfigSnapshot> {
if opts.replication_request || (!opts.versioned && !opts.version_suspended) {
return Ok(DeleteReplicationConfigSnapshot::default());
}
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
}
pub(crate) async fn load_delete_replication_config_in(
ctx: &ReplicationInstanceContext,
bucket: &str,
opts: &ObjectOptions,
) -> Result<DeleteReplicationConfigSnapshot> {
if opts.replication_request || (!opts.versioned && !opts.version_suspended) {
return Ok(DeleteReplicationConfigSnapshot::default());
}
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata_in(ctx, bucket).await?)
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
@@ -54,10 +247,31 @@ impl ReplicationConfig {
self.config.is_none()
}
pub(crate) fn validate(&self) -> Result<()> {
validate_delete_replication_config(&VersioningConfiguration::default(), self.config.as_ref())
}
pub fn replicate(&self, obj: &ObjectOpts) -> bool {
self.config.as_ref().is_some_and(|config| config.replicate(obj))
}
pub(crate) fn check_delete_for_heal(
&self,
object: &ObjectToDelete,
source: &ObjectInfo,
opts: &ObjectOptions,
) -> ReplicateDecision {
check_replicate_delete_with_config(
object,
source,
opts,
false,
self.config.as_ref(),
source.delete_marker && source.version_purge_status.is_empty(),
true,
)
}
pub async fn resync(
&self,
oi: ObjectInfo,
@@ -70,30 +284,34 @@ impl ReplicationConfig {
let mut dsc = dsc;
if oi.delete_marker {
if heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status) {
if !dsc.targets_map.is_empty() {
return self.resync_internal(oi, dsc, status);
}
let opts = ObjectOpts {
name: oi.name.clone(),
version_id: oi.version_id,
delete_marker: true,
version_id: if oi.version_purge_status.is_empty() {
None
} else {
oi.version_id
},
delete_marker: oi.delete_marker,
op_type: ReplicationType::Delete,
existing_object: true,
..Default::default()
};
let arns = self
let targets = self
.config
.as_ref()
.map(|config| config.filter_target_arns(&opts))
.map(|config| config.filter_target_replication_decisions(&opts))
.unwrap_or_default();
if arns.is_empty() {
if targets.is_empty() {
return ResyncDecision::default();
}
for arn in arns {
let mut opts = opts.clone();
opts.target_arn = arn;
dsc.set(ReplicateTargetDecision::new(opts.target_arn.clone(), self.replicate(&opts), false));
for (arn, replicate) in targets {
dsc.set(ReplicateTargetDecision::new(arn, replicate, false));
}
return self.resync_internal(oi, dsc, status);
@@ -169,8 +387,10 @@ pub(crate) async fn check_replicate_delete(
del_opts: &ObjectOptions,
gerr: Option<String>,
) -> ReplicateDecision {
match check_replicate_delete_strict(bucket, dobj, oi, del_opts, gerr).await {
Ok(decision) => decision,
match load_delete_replication_config(bucket, del_opts).await {
Ok(snapshot) => {
check_replicate_delete_with_config(dobj, oi, del_opts, gerr.is_some(), snapshot.replication_config(), false, false)
}
Err(err) => {
error!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
@@ -193,66 +413,101 @@ pub(crate) async fn check_replicate_delete_strict(
del_opts: &ObjectOptions,
gerr: Option<String>,
) -> Result<ReplicateDecision> {
let rcfg = match get_replication_config(bucket).await {
Ok(Some(config)) => config,
Ok(None) => return Ok(ReplicateDecision::default()),
Err(err) => return Err(err),
};
if del_opts.replication_request {
let Some(config) = get_replication_config(bucket).await? else {
return Ok(ReplicateDecision::default());
};
let mut decision = check_replicate_delete_with_config(dobj, oi, del_opts, gerr.is_some(), Some(&config), false, false);
if gerr.is_some() {
return Ok(decision);
}
for target in decision.targets_map.values_mut() {
if let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &target.arn).await {
target.synchronous = client.replicate_sync;
} else {
target.replicate = false;
target.synchronous = false;
}
}
Ok(decision)
}
pub(crate) fn check_replicate_delete_with_snapshot(
dobj: &ObjectToDelete,
oi: &ObjectInfo,
del_opts: &ObjectOptions,
source_error: bool,
snapshot: &DeleteReplicationConfigSnapshot,
) -> ReplicateDecision {
check_replicate_delete_with_config(dobj, oi, del_opts, source_error, snapshot.replication_config(), false, false)
}
fn check_replicate_delete_with_config(
dobj: &ObjectToDelete,
oi: &ObjectInfo,
del_opts: &ObjectOptions,
source_error: bool,
config: Option<&ReplicationConfiguration>,
existing_delete_marker: bool,
trust_persisted_replica_status: bool,
) -> ReplicateDecision {
if del_opts.replication_request {
return ReplicateDecision::default();
}
if !del_opts.versioned && !del_opts.version_suspended {
return Ok(ReplicateDecision::default());
return ReplicateDecision::default();
}
let Some(rcfg) = config else {
return ReplicateDecision::default();
};
let replication_delete = object_to_delete_for_replication(dobj);
let opts = delete_replication_object_opts(
let missing_source_marker = source_error && dobj.version_id.is_none();
let mut opts = delete_replication_object_opts(
&replication_delete,
&ReplicationDeleteSource {
user_defined: oi.user_defined.as_ref(),
user_tags: oi.user_tags.as_str(),
delete_marker: oi.delete_marker,
replication_status: oi.replication_status.clone(),
delete_marker: oi.delete_marker || missing_source_marker,
replication_status: if trust_persisted_replica_status {
oi.replication_status.clone()
} else {
ReplicationStatusType::Empty
},
},
);
let tgt_arns = rcfg.filter_target_arns(&opts);
let mut dsc = ReplicateDecision::new();
if tgt_arns.is_empty() {
return Ok(dsc);
if existing_delete_marker {
opts.version_id = None;
}
for tgt_arn in tgt_arns {
let mut opts = opts.clone();
opts.target_arn = tgt_arn.clone();
let replicate = rcfg.replicate(&opts);
let sync = false;
let target_decisions = rcfg.filter_target_replication_decisions(&opts);
let mut dsc = ReplicateDecision::new();
if gerr.is_some() {
if let Some(replicate) = delete_replication_missing_source_decision(
oi.delete_marker,
if target_decisions.is_empty() {
return dsc;
}
for (tgt_arn, replicate) in target_decisions {
let effective_replicate = if source_error {
delete_replication_missing_source_decision(
oi.delete_marker || missing_source_marker,
oi.target_replication_status(&tgt_arn),
replicate,
&oi.version_purge_status,
) {
dsc.set(ReplicateTargetDecision::new(tgt_arn, replicate, sync));
}
continue;
}
let tgt = ReplicationTargetStore::remote_target_client(bucket, &tgt_arn).await;
let tgt_dsc = if let Some(tgt) = tgt {
ReplicateTargetDecision::new(tgt_arn, replicate, tgt.replicate_sync)
)
} else {
ReplicateTargetDecision::new(tgt_arn, false, false)
Some(replicate)
};
dsc.set(tgt_dsc);
let Some(effective_replicate) = effective_replicate else {
continue;
};
dsc.set(ReplicateTargetDecision::new(tgt_arn, effective_replicate, false));
}
Ok(dsc)
dsc
}
pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision {
@@ -312,8 +567,13 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
#[cfg(test)]
mod tests {
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
use s3s::dto::{
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
};
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
use super::super::replication_target_boundary::BucketTarget;
use super::*;
fn replication_rule() -> ReplicationRule {
@@ -373,4 +633,379 @@ mod tests {
assert!(options.is_replication_request());
assert_eq!(options.user_tags(), "env=prod");
}
#[test]
fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
let mut rule = replication_rule();
rule.destination.bucket.clear();
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
let err = validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config))
.expect_err("an enabled rule without a destination must fail closed");
assert!(err.to_string().contains("destination ARN"));
}
#[test]
fn delete_snapshot_rejects_role_with_multiple_destinations() {
let first = replication_rule();
let mut second = replication_rule();
second.destination.bucket = "arn:aws:s3:::other-target".to_string();
let config = ReplicationConfiguration {
role: "arn:aws:s3:::role-target".to_string(),
rules: vec![first, second],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
}
#[test]
fn delete_snapshot_rejects_unknown_status_values() {
let invalid_versioning = VersioningConfiguration {
status: Some("Enabld".to_string().into()),
..Default::default()
};
assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
let mut invalid_rule = replication_rule();
invalid_rule.status = "Enabld".to_string().into();
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_rule],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
let mut invalid_delete = replication_rule();
invalid_delete.delete_replication = Some(DeleteReplication {
status: "Enabld".to_string().into(),
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_delete],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
let mut invalid_delete_marker = replication_rule();
invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some("Enabld".to_string().into()),
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_delete_marker],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
let mut invalid_replica_modifications = replication_rule();
invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
replica_modifications: Some(ReplicaModifications {
status: "Enabld".to_string().into(),
}),
sse_kms_encrypted_objects: None,
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_replica_modifications],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
}
#[test]
fn request_snapshot_borrows_cached_replication_config() {
let mut metadata = BucketMetadata::new("bucket");
metadata.versioning_config_xml = b"configured".to_vec();
metadata.versioning_config = Some(VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
});
metadata.replication_config_xml = b"configured".to_vec();
metadata.replication_config = Some(ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule()],
});
let metadata = Arc::new(metadata);
let cached_config = metadata.replication_config.as_ref().expect("cached config") as *const _;
let snapshot = delete_request_snapshot_from_metadata(Arc::clone(&metadata)).expect("valid snapshot");
assert_eq!(snapshot.replication_config().expect("snapshot config") as *const _, cached_config);
}
#[test]
fn delete_snapshot_debug_redacts_bucket_target_credentials() {
let secret = "snapshot-secret-must-not-be-formatted";
let mut metadata = BucketMetadata::new("bucket");
metadata.bucket_targets_config_json = format!(r#"{{"secretKey":"{secret}"}}"#).into_bytes();
let opts = ObjectOptions {
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot {
metadata: Some(Arc::new(metadata)),
versioning: VersioningConfiguration::default(),
})),
..Default::default()
};
let debug = format!("{opts:?}");
assert!(!debug.contains(secret), "delete tracing must not expose replication target credentials");
assert!(debug.contains("has_replication_config"));
}
#[test]
fn request_snapshot_rejects_replication_without_versioning_status() {
let mut malformed = BucketMetadata::new("bucket");
malformed.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
assert!(
delete_request_snapshot_from_metadata(Arc::new(malformed)).is_err(),
"malformed replication metadata must fail closed even when versioning has no status"
);
let mut inconsistent = BucketMetadata::new("bucket");
inconsistent.replication_config = Some(ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule()],
});
assert!(
delete_request_snapshot_from_metadata(Arc::new(inconsistent)).is_err(),
"replication metadata without an enabled or suspended versioning state must fail closed"
);
}
#[test]
fn missing_source_marker_creation_is_still_admitted() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
});
let mut metadata = BucketMetadata::new("bucket");
metadata.replication_config = Some(ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
});
let snapshot = DeleteReplicationConfigSnapshot {
metadata: Some(Arc::new(metadata)),
..Default::default()
};
let decision = check_replicate_delete_with_snapshot(
&ObjectToDelete {
object_name: "object".to_string(),
..Default::default()
},
&ObjectInfo::default(),
&ObjectOptions {
versioned: true,
..Default::default()
},
true,
&snapshot,
);
assert!(decision.replicate_any());
assert!(decision.targets_map.get(arn).is_some_and(|target| target.replicate));
}
#[test]
fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
let mut prefix_rule = replication_rule();
prefix_rule.prefix = Some("logs/".to_string());
prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
});
let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![prefix_rule],
}),
);
let mut tag_rule = replication_rule();
tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
});
tag_rule.filter = Some(ReplicationRuleFilter {
tag: Some(Tag {
key: Some("class".to_string()),
value: Some("audit".to_string()),
}),
..Default::default()
});
let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![tag_rule],
}),
);
assert!(!prefix_snapshot.active_delete_marker_rules_require_tags("logs/2026/app.log"));
assert!(tag_snapshot.active_delete_marker_rules_require_tags("logs/2026/app.log"));
}
#[test]
fn heal_uses_delete_switch_for_pending_purges_and_marker_switch_for_stored_markers() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
});
let config = ReplicationConfig::new(
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
}),
None,
);
let object = ObjectToDelete {
object_name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
..Default::default()
};
let opts = ObjectOptions {
versioned: true,
..Default::default()
};
let purge = ObjectInfo {
version_id: object.version_id,
version_purge_status: VersionPurgeStatusType::Pending,
..Default::default()
};
assert!(config.check_delete_for_heal(&object, &purge, &opts).replicate_any());
let marker = ObjectInfo {
delete_marker: true,
version_id: object.version_id,
..Default::default()
};
assert!(!config.check_delete_for_heal(&object, &marker, &opts).replicate_any());
}
#[test]
fn live_delete_does_not_trust_persisted_replica_status() {
let mut rule = replication_rule();
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.source_selection_criteria = Some(SourceSelectionCriteria {
replica_modifications: Some(ReplicaModifications {
status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
}),
sse_kms_encrypted_objects: None,
});
let replication = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(replication.clone()),
);
let object = ObjectToDelete {
object_name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
..Default::default()
};
let source = ObjectInfo {
replication_status: ReplicationStatusType::Replica,
..Default::default()
};
let opts = ObjectOptions {
versioned: true,
..Default::default()
};
assert!(
check_replicate_delete_with_snapshot(&object, &source, &opts, false, &snapshot).replicate_any(),
"an ordinary authenticated delete must not inherit replica identity from object metadata"
);
assert!(
!ReplicationConfig::new(Some(replication), None)
.check_delete_for_heal(&object, &source, &opts)
.replicate_any(),
"heal must still honor the persisted replica identity"
);
}
#[tokio::test]
async fn resync_keeps_marker_version_purges_separate_from_marker_creation() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
});
let config = ReplicationConfig::new(
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
}),
Some(BucketTargets {
targets: vec![BucketTarget {
arn: arn.to_string(),
..Default::default()
}],
}),
);
let marker = ObjectInfo {
name: "object".to_string(),
delete_marker: true,
version_id: Some(uuid::Uuid::new_v4()),
..Default::default()
};
let purge = config
.resync(
ObjectInfo {
version_purge_status: VersionPurgeStatusType::Pending,
..marker.clone()
},
ReplicateDecision::default(),
&HashMap::new(),
)
.await;
assert!(purge.targets.get(arn).is_some_and(|target| target.replicate));
let mut object_purge_decision = ReplicateDecision::default();
object_purge_decision.set(ReplicateTargetDecision::new(arn.to_string(), true, false));
let object_purge = config
.resync(
ObjectInfo {
name: "object".to_string(),
version_id: Some(uuid::Uuid::new_v4()),
version_purge_status: VersionPurgeStatusType::Pending,
..Default::default()
},
object_purge_decision,
&HashMap::new(),
)
.await;
assert!(
object_purge.targets.get(arn).is_some_and(|target| target.replicate),
"non-marker PENDING purges must stay on the delete resync path"
);
let stored_marker = config.resync(marker, ReplicateDecision::default(), &HashMap::new()).await;
assert!(!stored_marker.targets.contains_key(arn));
}
}
@@ -86,12 +86,26 @@ pub struct DurableMrfBucketBacklog {
pub bytes: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DurableMrfTargetBacklog {
pub bucket: String,
pub target_arn: String,
pub count: u64,
pub bytes: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DurableMrfBacklogSummary {
pub available: bool,
pub buckets: Vec<DurableMrfBucketBacklog>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct DurableMrfBacklogSnapshot {
summary: DurableMrfBacklogSummary,
targets: Vec<DurableMrfTargetBacklog>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MrfBucketBacklogObservability {
pub bucket: String,
@@ -110,6 +124,8 @@ pub struct MrfBacklogObservabilitySummary {
static DURABLE_MRF_BACKLOG_SUMMARY: LazyLock<StdRwLock<DurableMrfBacklogSummary>> =
LazyLock::new(|| StdRwLock::new(DurableMrfBacklogSummary::default()));
static DURABLE_MRF_TARGET_BACKLOG: LazyLock<StdRwLock<Vec<DurableMrfTargetBacklog>>> =
LazyLock::new(|| StdRwLock::new(Vec::new()));
static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTracker>> =
LazyLock::new(|| StdRwLock::new(MrfBacklogObservabilityTracker::default()));
@@ -117,13 +133,15 @@ static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTrac
struct DurableMrfBacklogTracker {
available: bool,
buckets: HashMap<String, DurableMrfBucketBacklog>,
targets: HashMap<(String, String), DurableMrfTargetBacklog>,
}
impl DurableMrfBacklogTracker {
fn add_entry(&mut self, bucket_name: String, entry_size: i64) {
let Ok(size) = u64::try_from(entry_size) else {
fn add_entry(&mut self, entry: &MrfReplicateEntry) {
let Ok(size) = u64::try_from(entry.size) else {
self.available = false;
self.buckets.clear();
self.targets.clear();
return;
};
@@ -131,6 +149,7 @@ impl DurableMrfBacklogTracker {
return;
}
let bucket_name = entry.bucket.clone();
let bucket = match self.buckets.entry(bucket_name) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
@@ -143,16 +162,39 @@ impl DurableMrfBacklogTracker {
};
bucket.count = bucket.count.saturating_add(1);
bucket.bytes = bucket.bytes.saturating_add(size);
for target_arn in &entry.target_arns {
if target_arn.is_empty() {
continue;
}
let key = (entry.bucket.clone(), target_arn.clone());
let target = match self.targets.entry(key) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
let (bucket, target_arn) = entry.key().clone();
entry.insert(DurableMrfTargetBacklog {
bucket,
target_arn,
..Default::default()
})
}
};
target.count = target.count.saturating_add(1);
target.bytes = target.bytes.saturating_add(size);
}
}
fn into_summary(self) -> DurableMrfBacklogSummary {
fn into_snapshot(self) -> DurableMrfBacklogSnapshot {
if !self.available {
return DurableMrfBacklogSummary::default();
return DurableMrfBacklogSnapshot::default();
}
DurableMrfBacklogSummary {
available: true,
buckets: self.buckets.into_values().collect(),
DurableMrfBacklogSnapshot {
summary: DurableMrfBacklogSummary {
available: true,
buckets: self.buckets.into_values().collect(),
},
targets: self.targets.into_values().collect(),
}
}
}
@@ -218,7 +260,21 @@ impl MrfBacklogObservabilityTracker {
}
}
fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSummary
fn durable_mrf_backlog_summary_from_entries<'a>(
entries: impl IntoIterator<Item = &'a MrfReplicateEntry>,
) -> DurableMrfBacklogSnapshot {
let mut tracker = DurableMrfBacklogTracker {
available: true,
..Default::default()
};
for entry in entries {
tracker.add_entry(entry);
}
tracker.into_snapshot()
}
#[cfg(test)]
fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSnapshot
where
I: IntoIterator<Item = (String, i64)>,
{
@@ -227,16 +283,38 @@ where
..Default::default()
};
for (bucket_name, entry_size) in entries {
tracker.add_entry(bucket_name, entry_size);
tracker.add_entry(&MrfReplicateEntry {
bucket: bucket_name,
object: String::new(),
version_id: None,
retry_count: 0,
size: entry_size,
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
});
}
tracker.into_snapshot()
}
fn set_durable_mrf_backlog_snapshot(snapshot: DurableMrfBacklogSnapshot) {
match DURABLE_MRF_BACKLOG_SUMMARY.write() {
Ok(mut guard) => *guard = snapshot.summary,
Err(poisoned) => *poisoned.into_inner() = snapshot.summary,
}
match DURABLE_MRF_TARGET_BACKLOG.write() {
Ok(mut guard) => *guard = snapshot.targets,
Err(poisoned) => *poisoned.into_inner() = snapshot.targets,
}
tracker.into_summary()
}
fn set_durable_mrf_backlog_summary(summary: DurableMrfBacklogSummary) {
match DURABLE_MRF_BACKLOG_SUMMARY.write() {
Ok(mut guard) => *guard = summary,
Err(poisoned) => *poisoned.into_inner() = summary,
}
set_durable_mrf_backlog_snapshot(DurableMrfBacklogSnapshot {
summary,
targets: Vec::new(),
});
}
pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
@@ -246,6 +324,13 @@ pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
}
}
pub fn durable_mrf_target_backlog_snapshot() -> Vec<DurableMrfTargetBacklog> {
match DURABLE_MRF_TARGET_BACKLOG.read() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
}
}
pub fn mrf_backlog_observability_snapshot() -> MrfBacklogObservabilitySummary {
match MRF_BACKLOG_OBSERVABILITY.read() {
Ok(guard) => guard.snapshot(),
@@ -675,6 +760,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
let target_arns = ri.dsc.replicate_target_arns();
// If object is large, queue it to a static set of large workers
if should_queue_large_object(ri.size) {
use std::collections::hash_map::DefaultHasher;
@@ -691,10 +777,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
if let Some(worker) = lrg_workers.get(index) {
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
self.stats.inc_target_q(&ri.bucket, &target_arns, ri.size);
if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
self.stats.dec_target_q(&ri.bucket, &target_arns, ri.size);
// Try to add more workers if possible
let max_l_workers = *self.max_l_workers.read().await;
@@ -723,10 +811,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
};
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
self.stats.inc_target_q(&ri.bucket, &target_arns, ri.size);
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
self.stats.dec_target_q(&ri.bucket, &target_arns, ri.size);
// Queue to MRF if all workers are busy.
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
@@ -740,6 +830,11 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues a replica delete task
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission {
let target_arns = if doi.target_arn.is_empty() {
Vec::new()
} else {
vec![doi.target_arn.clone()]
};
let ch = self
.worker_queue_channel(&doi.op_type, &doi.bucket, &doi.delete_object.object_name, 0)
.await;
@@ -749,10 +844,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
};
self.stats.inc_q(&doi.bucket, 0, true, doi.op_type);
self.stats.inc_target_q(&doi.bucket, &target_arns, 0);
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
self.stats.dec_q(&doi.bucket, 0, true, doi.op_type);
self.stats.dec_target_q(&doi.bucket, &target_arns, 0);
let admission = self.queue_mrf_save_admission(doi.to_mrf_entry(), "delete").await;
@@ -771,9 +868,11 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let bucket = entry.bucket.clone();
let size = entry.size;
let is_delete = matches!(entry.op, MrfOpKind::Delete);
let target_arns = entry.target_arns.clone();
let admission = queue_mrf_save_entry(&self.mrf_save_tx, entry, queue_type).await;
if admission == ReplicationQueueAdmission::Queued {
self.stats.inc_q(&bucket, size, is_delete, ReplicationType::Heal);
self.stats.inc_target_q(&bucket, &target_arns, size);
}
admission
}
@@ -827,9 +926,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
return;
}
};
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
entries.iter().map(|entry| (entry.bucket.clone(), entry.size)),
));
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
let total = entries.len();
let mut queued_count = 0usize;
@@ -1018,7 +1115,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
continue;
}
durable_tracker.add_entry(e.bucket.clone(), e.size);
durable_tracker.add_entry(&e);
observe_mrf_pending(&e);
pending.push(e);
dirty = true;
@@ -1029,7 +1126,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
if pending.len() - flushed_len >= 1000
&& let Some(duration_millis) = flush_mrf_to_disk(&pending, &storage).await
{
set_durable_mrf_backlog_summary(durable_tracker.clone().into_summary());
set_durable_mrf_backlog_snapshot(durable_tracker.clone().into_snapshot());
observe_mrf_pending_flushed(&pending[flushed_len..], duration_millis);
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
flushed_len = pending.len();
@@ -1039,7 +1136,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
None => {
// Channel closed (pool shutting down) — final flush.
if dirty && let Some(duration_millis) = flush_mrf_to_disk(&pending, &storage).await {
set_durable_mrf_backlog_summary(durable_tracker.clone().into_summary());
set_durable_mrf_backlog_snapshot(durable_tracker.clone().into_snapshot());
observe_mrf_pending_flushed(&pending[flushed_len..], duration_millis);
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
}
@@ -1048,7 +1145,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
},
_ = interval.tick() => {
if dirty && let Some(duration_millis) = flush_mrf_to_disk(&pending, &storage).await {
set_durable_mrf_backlog_summary(durable_tracker.clone().into_summary());
set_durable_mrf_backlog_snapshot(durable_tracker.clone().into_snapshot());
observe_mrf_pending_flushed(&pending[flushed_len..], duration_millis);
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
flushed_len = pending.len();
@@ -1451,6 +1548,7 @@ struct ReplicationBacklogGuard {
size: i64,
is_delete_marker: bool,
op_type: ReplicationType,
target_arns: Vec<String>,
}
impl ReplicationBacklogGuard {
@@ -1461,16 +1559,23 @@ impl ReplicationBacklogGuard {
size: object.size,
is_delete_marker: object.delete_marker,
op_type: object.op_type,
target_arns: object.dsc.replicate_target_arns(),
}
}
fn for_delete(stats: Arc<ReplicationStats>, delete: &DeletedObjectReplicationInfo) -> Self {
let target_arns = if delete.target_arn.is_empty() {
Vec::new()
} else {
vec![delete.target_arn.clone()]
};
Self {
stats,
bucket: delete.bucket.clone(),
size: 0,
is_delete_marker: true,
op_type: delete.op_type,
target_arns,
}
}
}
@@ -1478,6 +1583,7 @@ impl ReplicationBacklogGuard {
impl Drop for ReplicationBacklogGuard {
fn drop(&mut self) {
self.stats.dec_q(&self.bucket, self.size, self.is_delete_marker, self.op_type);
self.stats.dec_target_q(&self.bucket, &self.target_arns, self.size);
}
}
@@ -1524,6 +1630,7 @@ async fn queue_mrf_save_entry(
fn dec_mrf_entries(stats: &ReplicationStats, entries: &[MrfReplicateEntry]) {
for entry in entries {
stats.dec_q(&entry.bucket, entry.size, matches!(entry.op, MrfOpKind::Delete), ReplicationType::Heal);
stats.dec_target_q(&entry.bucket, &entry.target_arns, entry.size);
}
}
@@ -1860,7 +1967,24 @@ pub(crate) async fn queue_replication_heal_internal(
};
}
roi = get_heal_replicate_object_info(&oi, &rcfg).await;
roi = match get_heal_replicate_object_info(&oi, &rcfg).await {
Ok(roi) => roi,
Err(err) => {
warn!(
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %oi.bucket,
object = %oi.name,
error = %err,
"Failed to classify object for replication heal"
);
return ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Missed,
};
}
};
roi.retry_count = retry_count;
match replication_heal_queue_action(&mut roi) {
@@ -1915,6 +2039,7 @@ async fn queue_replicate_deletes(batch: ReplicationHealResyncDeletes) -> Replica
#[cfg(test)]
mod tests {
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
use super::super::replication_resync_boundary::{decode_mrf_file, encode_mrf_file, encode_resync_file};
use super::super::replication_storage_boundary::{
DeletedObject, FileInfo, GetObjectReader, HTTPRangeSpec, ListOperations, ObjectIO, ObjectOperations, PutObjReader,
@@ -2260,6 +2385,22 @@ mod tests {
(stats.replication_stats.q_stat.curr.count, stats.replication_stats.q_stat.curr.bytes)
}
fn current_target_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, target_arn: &str) -> Option<(u64, u64)> {
pool.stats
.runtime_target_backlog_snapshot()
.into_iter()
.find(|target| target.bucket == bucket && target.target_arn == target_arn)
.map(|target| (target.count, target.bytes))
}
fn test_replicate_decision(target_arns: &[&str]) -> ReplicateDecision {
let mut decision = ReplicateDecision::default();
for target_arn in target_arns {
decision.set(ReplicateTargetDecision::new((*target_arn).to_string(), true, false));
}
decision
}
async fn wait_for_current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, expected: (i64, i64)) {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
@@ -2293,6 +2434,35 @@ mod tests {
assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
}
#[tokio::test]
async fn regular_worker_admission_counts_target_backlog_before_receive() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, _rx) = mpsc::channel(1);
pool.workers.write().await.push(tx);
let admission = pool
.queue_replica_task(ReplicateObjectInfo {
bucket: "target-admission-bucket".to_string(),
name: "object".to_string(),
size: 4096,
op_type: ReplicationType::Object,
dsc: test_replicate_decision(&["arn:rustfs:replication:target-b", "arn:rustfs:replication:target-a"]),
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
assert_eq!(current_queue(&pool, "target-admission-bucket").await, (1, 4096));
assert_eq!(
current_target_queue(&pool, "target-admission-bucket", "arn:rustfs:replication:target-a"),
Some((1, 4096))
);
assert_eq!(
current_target_queue(&pool, "target-admission-bucket", "arn:rustfs:replication:target-b"),
Some((1, 4096))
);
}
#[tokio::test]
async fn large_worker_admission_counts_channel_backlog_before_receive() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -2336,6 +2506,33 @@ mod tests {
assert_eq!(current_queue(&pool, "delete-admission-bucket").await, (1, 0));
}
#[tokio::test]
async fn delete_admission_counts_target_backlog_before_receive() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, _rx) = mpsc::channel(1);
pool.workers.write().await.push(tx);
let admission = pool
.queue_replica_delete_task(DeletedObjectReplicationInfo {
bucket: "delete-target-admission-bucket".to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "deleted-object".to_string(),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
assert_eq!(current_queue(&pool, "delete-target-admission-bucket").await, (1, 0));
assert_eq!(
current_target_queue(&pool, "delete-target-admission-bucket", "arn:rustfs:replication:target-a"),
Some((1, 0))
);
}
#[tokio::test]
async fn regular_worker_drains_current_backlog_after_processing() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -2681,6 +2878,55 @@ mod tests {
assert_eq!(admission, ReplicationQueueAdmission::Missed);
}
#[tokio::test]
async fn heal_queue_marks_missing_versioning_state_as_missed() {
use super::super::replication_target_boundary::BucketTargets;
use s3s::dto::{
DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
ReplicationRuleStatus,
};
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let result = queue_replication_heal_internal(
"missing-versioning-state",
ObjectInfo {
bucket: "missing-versioning-state".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
version_purge_status: super::super::replication_filemeta_boundary::VersionPurgeStatusType::Pending,
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
},
ReplicationConfig::new(
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: None,
id: Some("delete".to_string()),
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
Some(BucketTargets::default()),
),
0,
)
.await;
assert_eq!(result.admission, ReplicationQueueAdmission::Missed);
}
#[tokio::test]
async fn queue_replica_task_counts_mrf_pending_backlog_when_worker_queue_is_full() {
let shared = empty_resync_shared_state();
@@ -2702,6 +2948,7 @@ mod tests {
name: "fallback-object".to_string(),
size: 2048,
op_type: ReplicationType::Object,
dsc: test_replicate_decision(&["arn:rustfs:replication:target-a"]),
..Default::default()
})
.await;
@@ -2710,6 +2957,10 @@ mod tests {
let queued = pool.stats.get_latest_replication_stats("runtime-backlog").await;
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 2048);
assert_eq!(
current_target_queue(&pool, "runtime-backlog", "arn:rustfs:replication:target-a"),
Some((1, 2048))
);
}
#[test]
@@ -2745,6 +2996,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
let second = MrfReplicateEntry {
object: "second".to_string(),
@@ -2794,6 +3046,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
},
"test",
)
@@ -2824,6 +3077,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
observe_mrf_pending(&entry);
@@ -2845,12 +3099,14 @@ mod tests {
async fn replication_backlog_guard_decrements_on_drop() {
let stats = Arc::new(ReplicationStats::new());
stats.inc_q("guard-bucket", 256, false, ReplicationType::Object);
stats.inc_target_q("guard-bucket", &["arn:rustfs:replication:target-a".to_string()], 256);
{
let object = ReplicateObjectInfo {
bucket: "guard-bucket".to_string(),
size: 256,
op_type: ReplicationType::Object,
dsc: test_replicate_decision(&["arn:rustfs:replication:target-a"]),
..Default::default()
};
let _guard = ReplicationBacklogGuard::for_object(stats.clone(), &object);
@@ -2859,6 +3115,30 @@ mod tests {
let queued = stats.get_latest_replication_stats("guard-bucket").await;
assert_eq!(queued.replication_stats.q_stat.curr.count, 0);
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 0);
assert!(stats.runtime_target_backlog_snapshot().is_empty());
}
#[test]
fn dec_mrf_entries_decrements_target_backlog() {
let stats = ReplicationStats::new();
let entry = MrfReplicateEntry {
bucket: "mrf-target-drain-bucket".to_string(),
object: "object".to_string(),
version_id: None,
retry_count: 1,
size: 1024,
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
};
stats.inc_q(&entry.bucket, entry.size, false, ReplicationType::Heal);
stats.inc_target_q(&entry.bucket, &entry.target_arns, entry.size);
dec_mrf_entries(&stats, std::slice::from_ref(&entry));
assert!(stats.runtime_target_backlog_snapshot().is_empty());
}
#[test]
@@ -2873,6 +3153,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
let second = MrfReplicateEntry {
object: "second".to_string(),
@@ -2984,6 +3265,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
@@ -3017,6 +3299,7 @@ mod tests {
delete_marker_version_id: Some(dm_vid),
delete_marker: true,
delete_marker_mtime: Some(mtime_nanos),
target_arns: Vec::new(),
};
let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
@@ -3050,6 +3333,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
let encoded = encode_mrf_file(&[entry]).expect("encode");
@@ -3078,6 +3362,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
},
MrfReplicateEntry {
bucket: "b".to_string(),
@@ -3089,6 +3374,7 @@ mod tests {
delete_marker_version_id: Some(del_dm_vid),
delete_marker: true,
delete_marker_mtime: None,
target_arns: Vec::new(),
},
];
@@ -3118,6 +3404,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
assert_eq!(obj_entry.op, MrfOpKind::Object);
@@ -3132,6 +3419,7 @@ mod tests {
delete_marker_version_id: Some(Uuid::new_v4()),
delete_marker: true,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
assert_eq!(del_entry.op, MrfOpKind::Delete);
@@ -3147,6 +3435,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
};
assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
}
@@ -3196,6 +3485,7 @@ mod tests {
// The "deleteMarkerMtime" key was absent in old files — #[serde(default)] must fill in
// None so replay falls back to the current time (backlog#867 backward compatibility).
assert_eq!(entry.delete_marker_mtime, None, "missing deleteMarkerMtime key must default to None");
assert!(entry.target_arns.is_empty(), "old MRF entries must not be attributed to a target");
}
#[test]
@@ -3210,6 +3500,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
}];
let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
@@ -3226,9 +3517,10 @@ mod tests {
#[test]
fn durable_mrf_summary_aggregates_entries_by_bucket_for_obs() {
let summary =
let snapshot =
durable_mrf_backlog_summary_from_sizes([("b1".to_string(), 1024), ("b1".to_string(), 512), ("b2".to_string(), 0)]);
let summary = snapshot.summary;
assert!(summary.available);
let buckets = summary
.buckets
@@ -3239,13 +3531,82 @@ mod tests {
assert_eq!(buckets["b1"].bytes, 1536);
assert_eq!(buckets["b2"].count, 1);
assert_eq!(buckets["b2"].bytes, 0);
assert!(snapshot.targets.is_empty());
}
#[test]
fn durable_mrf_summary_aggregates_target_backlog_without_attributing_legacy_entries() {
let entries = vec![
MrfReplicateEntry {
bucket: "b1".to_string(),
object: "object-a".to_string(),
version_id: None,
retry_count: 0,
size: 1024,
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
},
MrfReplicateEntry {
bucket: "b1".to_string(),
object: "object-b".to_string(),
version_id: None,
retry_count: 0,
size: 512,
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: vec!["arn:target-a".to_string()],
},
MrfReplicateEntry {
bucket: "b1".to_string(),
object: "legacy-object".to_string(),
version_id: None,
retry_count: 0,
size: 256,
op: MrfOpKind::Object,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
},
];
let snapshot = durable_mrf_backlog_summary_from_entries(&entries);
let summary = snapshot.summary;
assert!(summary.available);
let buckets = summary
.buckets
.into_iter()
.map(|bucket| (bucket.bucket.clone(), bucket))
.collect::<HashMap<_, _>>();
assert_eq!(buckets["b1"].count, 3);
assert_eq!(buckets["b1"].bytes, 1792);
let targets = snapshot
.targets
.into_iter()
.map(|target| ((target.bucket.clone(), target.target_arn.clone()), target))
.collect::<HashMap<_, _>>();
let target_a = &targets[&("b1".to_string(), "arn:target-a".to_string())];
assert_eq!(target_a.count, 2);
assert_eq!(target_a.bytes, 1536);
let target_b = &targets[&("b1".to_string(), "arn:target-b".to_string())];
assert_eq!(target_b.count, 1);
assert_eq!(target_b.bytes, 1024);
}
#[test]
fn durable_mrf_summary_marks_invalid_sizes_unavailable() {
let invalid = durable_mrf_backlog_summary_from_sizes([("bucket".to_string(), -1)]);
assert!(!invalid.available);
assert!(invalid.buckets.is_empty());
let summary = invalid.summary;
assert!(!summary.available);
assert!(summary.buckets.is_empty());
assert!(invalid.targets.is_empty());
}
#[test]
@@ -3264,6 +3625,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: Vec::new(),
}])
.expect("invalid persisted entry should still encode for boundary testing");
let invalid = durable_mrf_backlog_from_read(Ok(negative));
@@ -18,16 +18,16 @@ use super::replication_config_store::ReplicationConfigStore;
use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found};
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
use super::replication_filemeta_boundary::{
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo,
ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state,
parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
ReplicatedTargetInfo, ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use super::replication_lock_boundary::ReplicationLockTiming;
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
use super::replication_metadata_boundary::ReplicationMetadataStore;
#[cfg(test)]
use super::replication_msgp_boundary::ReplicationMsgpCodec;
use super::replication_object_config::{ReplicationConfig, check_replicate_delete, get_replication_config, must_replicate};
use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
use super::replication_object_decision_boundary::{
MustReplicateOptions, ReplicationMultipartPartInput, heal_uses_delete_replication_path,
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
@@ -642,6 +642,21 @@ impl ReplicationResyncer {
};
let rcfg = ReplicationConfig::new(cfg.clone(), Some(targets));
if let Err(err) = rcfg.validate() {
error!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
error = %err,
reason = "replication_config_invalid",
"Replication resync config is invalid"
);
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
}
let target_arns = if let Some(cfg) = cfg {
cfg.filter_target_arns(&ObjectOpts {
@@ -982,7 +997,36 @@ impl ReplicationResyncer {
}
last_checkpoint = None;
let roi = get_heal_replicate_object_info(&object, &rcfg).await;
let roi = match get_heal_replicate_object_info(&object, &rcfg).await {
Ok(roi) => roi,
Err(err) => {
error!(
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
object = %object.name,
error = %err,
"Failed to classify object for replication resync"
);
let worker_failed = finish_resync_workers(worker_txs, results_tx, futures, false).await;
if worker_failed {
error!(
event = EVENT_RESYNC_TASK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %opts.bucket,
arn = %opts.arn,
reason = "worker_join_failed_after_classification_error",
"Replication resync worker cleanup observed task failure"
);
}
self.resync_bucket_mark_status(ResyncStatusType::ResyncFailed, opts.clone(), storage.clone())
.await;
return;
}
};
if !roi.existing_obj_resync.must_resync() {
continue;
}
@@ -1037,18 +1081,25 @@ impl ReplicationResyncer {
}
}
pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationConfig) -> ReplicateObjectInfo {
pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationConfig) -> Result<ReplicateObjectInfo> {
let mut oi = oi.clone();
let mut user_defined = (*oi.user_defined).clone();
let delete_path = heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status);
let stored_delete_decision = if delete_path && !oi.replication_decision.is_empty() {
Some(parse_replicate_decision(&oi.bucket, &oi.replication_decision)?)
} else {
None
};
let has_stored_delete_decision = stored_delete_decision.is_some();
if let Some(rc) = rcfg.config.as_ref()
&& !rc.role.is_empty()
{
if !oi.version_purge_status.is_empty() {
if oi.version_purge_status_internal.is_none() && !oi.version_purge_status.is_empty() {
oi.version_purge_status_internal = Some(format!("{}={};", rc.role, oi.version_purge_status.as_str()));
}
if !oi.replication_status.is_empty() {
if oi.replication_status_internal.is_none() && !oi.replication_status.is_empty() {
oi.replication_status_internal = Some(format!("{}={};", rc.role, oi.replication_status.as_str()));
}
@@ -1064,23 +1115,31 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
}
}
let dsc = if heal_uses_delete_replication_path(oi.delete_marker, &oi.version_purge_status) {
check_replicate_delete(
oi.bucket.as_str(),
&ObjectToDelete {
object_name: oi.name.clone(),
version_id: oi.version_id,
..Default::default()
},
&oi,
&ObjectOptions {
versioned: ReplicationVersioningStore::prefix_enabled(&oi.bucket, &oi.name).await,
version_suspended: ReplicationVersioningStore::prefix_suspended(&oi.bucket, &oi.name).await,
..Default::default()
},
None,
)
.await
let delete_state = if delete_path && !has_stored_delete_decision {
ReplicationVersioningStore::prefix_state(&oi.bucket, &oi.name).await?
} else {
(false, false)
};
let dsc = if let Some(decision) = stored_delete_decision {
decision
} else if delete_path {
if !delete_state.0 && !delete_state.1 {
ReplicateDecision::default()
} else {
rcfg.check_delete_for_heal(
&ObjectToDelete {
object_name: oi.name.clone(),
version_id: oi.version_id,
..Default::default()
},
&oi,
&ObjectOptions {
versioned: delete_state.0,
version_suspended: delete_state.1,
..Default::default()
},
)
}
} else {
must_replicate(
oi.bucket.as_str(),
@@ -1092,12 +1151,16 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
let target_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
let target_purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
let existing_obj_resync = rcfg.resync(oi.clone(), dsc.clone(), &target_statuses).await;
let existing_obj_resync = if delete_path && !has_stored_delete_decision && !delete_state.0 && !delete_state.1 {
Default::default()
} else {
rcfg.resync(oi.clone(), dsc.clone(), &target_statuses).await
};
let mut replication_state = oi.replication_state();
replication_state.replicate_decision_str = dsc.to_string();
let actual_size = oi.get_actual_size().unwrap_or_default();
ReplicateObjectInfo {
Ok(ReplicateObjectInfo {
name: oi.name.clone(),
size: oi.size,
actual_size,
@@ -1122,7 +1185,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
user_tags: (*oi.user_tags).clone(),
checksum: oi.checksum.clone(),
retry_count: 0,
}
})
}
pub(crate) async fn save_resync_status<S: ReplicationObjectIO>(
@@ -1566,7 +1629,7 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
.remove_object(
&tgt_client.bucket,
&dobj.delete_object.object_name,
Some(delete_marker_version_id.to_string()),
target_delete_version_id(delete_marker_version_id, true),
replication_delete_marker_purge_remove_options(dobj.delete_object.delete_marker_mtime),
)
.await;
@@ -1796,6 +1859,14 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
}
}
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
if version_id.is_nil() {
version_purge.then(|| NULL_VERSION_ID.to_string())
} else {
Some(version_id.to_string())
}
}
async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
version_id.to_owned()
@@ -1835,11 +1906,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
return rinfo;
}
let version_id = if version_id.is_nil() {
None
} else {
Some(version_id.to_string())
};
let version_id = target_delete_version_id(version_id, is_version_purge);
if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
match head_object_with_proxy_stats(
@@ -3088,7 +3155,12 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
#[cfg(test)]
mod tests {
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
use super::*;
use s3s::dto::{
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
ReplicationRuleStatus, VersioningConfiguration,
};
use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
@@ -3536,7 +3608,9 @@ mod tests {
..Default::default()
};
let rcfg = ReplicationConfig::new(None, None);
let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
let roi = get_heal_replicate_object_info(&oi, &rcfg)
.await
.expect("non-delete heal classification should succeed");
assert_eq!(roi.replication_status, ReplicationStatusType::Failed);
assert_eq!(roi.op_type, ReplicationType::Heal);
@@ -3561,7 +3635,9 @@ mod tests {
};
let rcfg = ReplicationConfig::new(None, None);
let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
let roi = get_heal_replicate_object_info(&oi, &rcfg)
.await
.expect("non-delete heal classification should succeed");
assert!(roi.ssec);
assert_eq!(roi.checksum, Some(checksum));
@@ -3577,6 +3653,7 @@ mod tests {
version_purge_status: VersionPurgeStatusType::Pending,
version_id: Some(Uuid::nil()),
mod_time: Some(OffsetDateTime::now_utc()),
replication_decision: format!("{role}=true;false;{role};"),
..Default::default()
};
let rcfg = ReplicationConfig::new(
@@ -3586,13 +3663,176 @@ mod tests {
}),
None,
);
let roi = get_heal_replicate_object_info(&oi, &rcfg).await;
let roi = get_heal_replicate_object_info(&oi, &rcfg)
.await
.expect("stored purge admission should classify without a live versioning lookup");
assert_eq!(roi.replication_status_internal, None);
assert_eq!(roi.version_purge_status_internal.as_deref(), Some(format!("{role}=PENDING;").as_str()));
assert_eq!(roi.target_purge_statuses.get(role), Some(&VersionPurgeStatusType::Pending));
}
#[tokio::test]
async fn heal_pending_purge_reads_one_versioning_generation() {
let bucket = format!("heal-versioning-snapshot-{}", Uuid::new_v4());
let object = "archive/object";
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
ReplicationVersioningStore::install_prefix_state_test_config(
&bucket,
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
);
let rcfg = ReplicationConfig::new(
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: None,
id: Some("delete".to_string()),
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
Some(BucketTargets {
targets: vec![BucketTarget {
arn: arn.to_string(),
..Default::default()
}],
}),
);
let oi = ObjectInfo {
bucket,
name: object.to_string(),
version_id: Some(Uuid::nil()),
version_purge_status: VersionPurgeStatusType::Pending,
..Default::default()
};
let roi = get_heal_replicate_object_info(&oi, &rcfg)
.await
.expect("pending null purge classification should succeed");
assert!(roi.dsc.targets_map.get(arn).is_some_and(|target| target.replicate));
assert!(
roi.existing_obj_resync
.targets
.get(arn)
.is_some_and(|target| target.replicate)
);
}
#[tokio::test]
async fn heal_pending_purge_preserves_the_persisted_admission_decision() {
let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
let current_role = "arn:rustfs:replication:us-east-1:target:current";
let rcfg = ReplicationConfig::new(
Some(ReplicationConfiguration {
role: current_role.to_string(),
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
}),
destination: Destination {
bucket: current_role.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: None,
id: Some("delete".to_string()),
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
Some(BucketTargets {
targets: vec![
BucketTarget {
arn: admitted_arn.to_string(),
..Default::default()
},
BucketTarget {
arn: current_role.to_string(),
..Default::default()
},
],
}),
);
let oi = ObjectInfo {
bucket: "heal-persisted-delete-decision".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
version_purge_status: VersionPurgeStatusType::Pending,
version_purge_status_internal: Some(format!("{admitted_arn}=PENDING;")),
replication_decision: format!("{admitted_arn}=true;false;{admitted_arn};"),
..Default::default()
};
let roi = get_heal_replicate_object_info(&oi, &rcfg)
.await
.expect("persisted delete admission should survive live rule disablement");
assert_eq!(
roi.version_purge_status_internal.as_deref(),
Some(format!("{admitted_arn}=PENDING;").as_str())
);
assert!(roi.dsc.targets_map.get(admitted_arn).is_some_and(|target| target.replicate));
assert!(!roi.dsc.targets_map.contains_key(current_role));
assert!(
roi.existing_obj_resync
.targets
.get(admitted_arn)
.is_some_and(|target| target.replicate)
);
assert!(!roi.existing_obj_resync.targets.contains_key(current_role));
}
#[tokio::test]
async fn heal_rejects_semantically_invalid_replication_config() {
let rcfg = ReplicationConfig::new(
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: None,
destination: Destination {
bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
..Default::default()
},
existing_object_replication: None,
filter: None,
id: Some("invalid".to_string()),
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static("Enabld"),
}],
}),
Some(BucketTargets::default()),
);
let err = rcfg
.validate()
.expect_err("invalid string-backed statuses must fail before heal classification loop");
assert!(err.to_string().contains("Rule.Status"));
}
#[tokio::test]
async fn test_cancel_marks_only_matching_bucket_target_token() {
let resyncer = ReplicationResyncer::new().await;
@@ -3759,4 +3999,13 @@ mod tests {
assert_eq!(resync_status_duration(ResyncStatusType::ResyncStarted, Some(start), end), None);
assert_eq!(resync_status_duration(ResyncStatusType::ResyncFailed, None, end), None);
}
#[test]
fn target_delete_version_id_preserves_explicit_null_purges() {
let version_id = Uuid::new_v4();
assert_eq!(target_delete_version_id(version_id, true), Some(version_id.to_string()));
assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID));
assert_eq!(target_delete_version_id(Uuid::nil(), false), None);
}
}
@@ -22,9 +22,9 @@ use super::replication_stats_boundary::{
QueueCache, ReplicationMetricScope, SRMetricsSummary, XferStats,
};
use super::runtime_boundary as runtime_sources;
use std::collections::HashMap;
use std::collections::{HashMap, hash_map::Entry};
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
use std::time::{Duration, SystemTime};
use tokio::sync::{Mutex, RwLock};
use tokio::time::interval;
@@ -153,6 +153,83 @@ pub struct ReplicationStats {
pub most_recent_stats: Arc<Mutex<HashMap<String, BucketStats>>>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RuntimeReplicationTargetBacklog {
pub bucket: String,
pub target_arn: String,
pub count: u64,
pub bytes: u64,
}
type TargetQueueKey = (String, String);
type TargetQueueCache = HashMap<TargetQueueKey, InQueueMetric>;
struct TargetQueueCacheSlot {
owner: Weak<StdMutex<QueueCache>>,
metrics: TargetQueueCache,
}
impl TargetQueueCacheSlot {
fn new(owner: &Arc<StdMutex<QueueCache>>) -> Self {
Self {
owner: Arc::downgrade(owner),
metrics: TargetQueueCache::default(),
}
}
fn belongs_to(&self, owner: &Arc<StdMutex<QueueCache>>) -> bool {
self.owner.upgrade().is_some_and(|current| Arc::ptr_eq(&current, owner))
}
}
// Keep runtime target counters outside ReplicationStats to preserve its public struct shape.
static TARGET_QUEUE_CACHES: LazyLock<StdMutex<Vec<TargetQueueCacheSlot>>> = LazyLock::new(|| StdMutex::new(Vec::new()));
fn i64_to_u64_floor_zero(value: i64) -> u64 {
u64::try_from(value.max(0)).unwrap_or(0)
}
fn normalized_target_arns(target_arns: &[String]) -> Vec<&str> {
let mut target_arns = target_arns
.iter()
.map(String::as_str)
.filter(|target_arn| !target_arn.is_empty())
.collect::<Vec<_>>();
target_arns.sort_unstable();
target_arns.dedup();
target_arns
}
fn target_queue_cache_snapshot(cache: &TargetQueueCache) -> Vec<RuntimeReplicationTargetBacklog> {
cache
.iter()
.filter_map(|((bucket, target_arn), metric)| {
let count = i64_to_u64_floor_zero(metric.curr.get_current_count());
let bytes = i64_to_u64_floor_zero(metric.curr.get_current_bytes());
(count > 0 || bytes > 0).then(|| RuntimeReplicationTargetBacklog {
bucket: bucket.clone(),
target_arn: target_arn.clone(),
count,
bytes,
})
})
.collect()
}
fn prune_stale_target_queue_caches(caches: &mut Vec<TargetQueueCacheSlot>) {
caches.retain(|slot| slot.owner.strong_count() > 0);
}
fn with_target_queue_caches<T>(f: impl FnOnce(&mut Vec<TargetQueueCacheSlot>) -> T) -> T {
match TARGET_QUEUE_CACHES.lock() {
Ok(mut caches) => f(&mut caches),
Err(poisoned) => {
let mut caches = poisoned.into_inner();
f(&mut caches)
}
}
}
impl ReplicationStats {
pub fn new() -> Self {
Self {
@@ -684,6 +761,68 @@ impl ReplicationStats {
}
}
pub(crate) fn inc_target_q(&self, bucket: &str, target_arns: &[String], size: i64) {
let target_arns = normalized_target_arns(target_arns);
if target_arns.is_empty() {
return;
}
with_target_queue_caches(|caches| {
prune_stale_target_queue_caches(caches);
let slot_index = match caches.iter().position(|slot| slot.belongs_to(&self.q_cache)) {
Some(index) => index,
None => {
caches.push(TargetQueueCacheSlot::new(&self.q_cache));
caches.len() - 1
}
};
let slot = &mut caches[slot_index];
let bucket = bucket.to_string();
for target_arn in target_arns {
let metric = match slot.metrics.entry((bucket.clone(), target_arn.to_string())) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => entry.insert(InQueueMetric::default()),
};
metric.curr.add_current(size, 1);
}
});
}
pub(crate) fn dec_target_q(&self, bucket: &str, target_arns: &[String], size: i64) {
let target_arns = normalized_target_arns(target_arns);
if target_arns.is_empty() {
return;
}
with_target_queue_caches(|caches| {
prune_stale_target_queue_caches(caches);
if let Some(slot) = caches.iter_mut().find(|slot| slot.belongs_to(&self.q_cache)) {
let bucket = bucket.to_string();
for target_arn in target_arns {
if let Some(metric) = slot.metrics.get_mut(&(bucket.clone(), target_arn.to_string())) {
metric.curr.subtract_current(size, 1);
}
}
slot.metrics
.retain(|_, metric| metric.curr.get_current_count() > 0 || metric.curr.get_current_bytes() > 0);
}
});
}
pub fn runtime_target_backlog_snapshot(&self) -> Vec<RuntimeReplicationTargetBacklog> {
let mut snapshot = with_target_queue_caches(|caches| {
caches
.iter()
.find(|slot| slot.belongs_to(&self.q_cache))
.map(|slot| target_queue_cache_snapshot(&slot.metrics))
.unwrap_or_default()
});
snapshot.sort_by(|left, right| {
left.bucket
.cmp(&right.bucket)
.then_with(|| left.target_arn.cmp(&right.target_arn))
});
snapshot
}
/// Increase proxy metrics
pub async fn inc_proxy(&self, bucket: &str, api: &str, is_err: bool) {
let mut p_cache = self.p_cache.lock().await;
@@ -707,6 +846,94 @@ impl Default for ReplicationStats {
mod tests {
use super::*;
#[test]
fn runtime_target_backlog_snapshot_tracks_targets() {
let stats = ReplicationStats::new();
stats.inc_target_q(
"photos",
&[
"arn:rustfs:replication:target-b".to_string(),
"arn:rustfs:replication:target-a".to_string(),
"arn:rustfs:replication:target-a".to_string(),
],
1024,
);
let snapshot = stats.runtime_target_backlog_snapshot();
assert_eq!(
snapshot,
vec![
RuntimeReplicationTargetBacklog {
bucket: "photos".to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
count: 1,
bytes: 1024,
},
RuntimeReplicationTargetBacklog {
bucket: "photos".to_string(),
target_arn: "arn:rustfs:replication:target-b".to_string(),
count: 1,
bytes: 1024,
},
]
);
}
#[test]
fn runtime_target_backlog_ignores_empty_targets() {
let stats = ReplicationStats::new();
stats.inc_target_q("photos", &["".to_string()], 1024);
assert!(stats.runtime_target_backlog_snapshot().is_empty());
}
#[test]
fn runtime_target_backlog_is_scoped_to_stats_instance() {
let first = ReplicationStats::new();
let second = ReplicationStats::new();
first.inc_target_q("photos", &["arn:rustfs:replication:target-a".to_string()], 1024);
assert!(second.runtime_target_backlog_snapshot().is_empty());
assert_eq!(first.runtime_target_backlog_snapshot()[0].count, 1);
}
#[test]
fn runtime_target_backlog_decrements_with_saturation() {
let stats = ReplicationStats::new();
let target_arns = ["arn:rustfs:replication:target-a".to_string()];
stats.inc_target_q("photos", &target_arns, 1024);
stats.dec_target_q("photos", &target_arns, 2048);
assert!(stats.runtime_target_backlog_snapshot().is_empty());
}
#[test]
fn runtime_target_backlog_prunes_stale_sidecar_on_next_access() {
{
let stats = ReplicationStats::new();
stats.inc_target_q("photos", &["arn:rustfs:replication:target-a".to_string()], 1024);
assert!(
TARGET_QUEUE_CACHES
.lock()
.expect("target queue cache mutex")
.iter()
.any(|slot| slot.owner.strong_count() > 0)
);
}
let stats = ReplicationStats::new();
stats.inc_target_q("photos", &["arn:rustfs:replication:target-b".to_string()], 1024);
assert!(
TARGET_QUEUE_CACHES
.lock()
.expect("target queue cache mutex")
.iter()
.all(|slot| slot.owner.strong_count() > 0)
);
}
#[tokio::test]
async fn test_replication_stats_new() {
let stats = ReplicationStats::new();
@@ -35,6 +35,8 @@ use time::format_description::well_known::Rfc3339;
pub(crate) use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
};
#[cfg(test)]
pub(crate) use crate::bucket::target::BucketTarget;
pub(crate) use crate::bucket::target::BucketTargets;
use super::replication_config_store::ReplicationConfigStore;
@@ -12,11 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::bucket::versioning_sys::BucketVersioningSys;
use super::replication_error_boundary::Result;
use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
#[cfg(test)]
use s3s::dto::VersioningConfiguration;
#[cfg(test)]
use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
#[cfg(test)]
static PREFIX_STATE_TEST_CONFIGS: LazyLock<Mutex<HashMap<String, VersioningConfiguration>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
pub(crate) struct ReplicationVersioningStore;
impl ReplicationVersioningStore {
#[cfg(test)]
pub(crate) fn install_prefix_state_test_config(bucket: &str, config: VersioningConfiguration) {
PREFIX_STATE_TEST_CONFIGS
.lock()
.expect("replication versioning test config lock should not be poisoned")
.insert(bucket.to_string(), config);
}
pub(crate) async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
BucketVersioningSys::prefix_enabled(bucket, prefix).await
}
@@ -24,4 +41,18 @@ impl ReplicationVersioningStore {
pub(crate) async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
BucketVersioningSys::prefix_suspended(bucket, prefix).await
}
pub(crate) async fn prefix_state(bucket: &str, prefix: &str) -> Result<(bool, bool)> {
#[cfg(test)]
if let Some(config) = PREFIX_STATE_TEST_CONFIGS
.lock()
.expect("replication versioning test config lock should not be poisoned")
.remove(bucket)
{
return Ok((config.prefix_enabled(prefix), config.prefix_suspended(prefix)));
}
let config = BucketVersioningSys::get(bucket).await?;
Ok((config.prefix_enabled(prefix), config.prefix_suspended(prefix)))
}
}
@@ -19,6 +19,9 @@ pub trait VersioningApi {
fn enabled(&self) -> bool;
fn prefix_enabled(&self, prefix: &str) -> bool;
fn prefix_suspended(&self, prefix: &str) -> bool;
fn delete_state(&self, prefix: &str) -> (bool, bool) {
(self.prefix_enabled(prefix), self.suspended())
}
fn versioned(&self, prefix: &str) -> bool;
fn suspended(&self) -> bool;
}
@@ -92,3 +95,60 @@ impl VersioningApi for VersioningConfiguration {
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
}
}
#[cfg(test)]
mod tests {
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
use super::*;
struct LegacyVersioning;
impl VersioningApi for LegacyVersioning {
fn enabled(&self) -> bool {
false
}
fn prefix_enabled(&self, _prefix: &str) -> bool {
false
}
fn prefix_suspended(&self, _prefix: &str) -> bool {
false
}
fn versioned(&self, _prefix: &str) -> bool {
false
}
fn suspended(&self) -> bool {
false
}
}
#[test]
fn delete_state_has_a_backward_compatible_default() {
assert_eq!(LegacyVersioning.delete_state("object"), (false, false));
}
#[test]
fn delete_state_treats_excluded_prefixes_as_unversioned() {
let config = VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
};
assert_eq!(config.delete_state("archive/object"), (false, false));
assert!(config.prefix_suspended("archive/object"));
assert_eq!(config.delete_state("live/object"), (true, false));
let suspended = VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
..Default::default()
};
assert_eq!(suspended.delete_state("archive/object"), (false, true));
}
}
+1 -1
View File
@@ -615,7 +615,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
self.get_disks_by_key(object).delete_object(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
#[tracing::instrument(skip(self, objects, opts))]
async fn delete_objects(
&self,
bucket: &str,
+2 -2
View File
@@ -17,8 +17,8 @@
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::replication::{
ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_status_from_filemeta,
replication_statuses_map, version_purge_status_from_filemeta, version_purge_statuses_map,
DeleteReplicationConfigSnapshot, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
replication_status_from_filemeta, replication_statuses_map, version_purge_status_from_filemeta, version_purge_statuses_map,
};
use crate::bucket::versioning::VersioningApi as _;
use crate::config::storageclass;
+44
View File
@@ -20,6 +20,47 @@ use crate::storage_api_contracts::{
},
};
#[derive(Clone, Default)]
pub struct DeleteLockFence {
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
#[cfg(test)]
forced_lost: bool,
}
impl Debug for DeleteLockFence {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeleteLockFence")
.field("signal_count", &self.signals.len())
.finish()
}
}
impl DeleteLockFence {
pub(crate) fn new(signals: Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Self {
Self {
signals: Arc::new(signals),
#[cfg(test)]
forced_lost: false,
}
}
pub(crate) fn is_lock_lost(&self) -> bool {
#[cfg(test)]
if self.forced_lost {
return true;
}
self.signals.iter().any(|signal| signal.is_lost())
}
#[cfg(test)]
pub(crate) fn lost_for_test() -> Self {
Self {
signals: Arc::default(),
forced_lost: true,
}
}
}
#[derive(Debug, Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -54,8 +95,11 @@ pub struct ObjectOptions {
pub http_preconditions: Option<HTTPPreconditions>,
pub delete_replication: Option<ReplicationState>,
pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>,
pub delete_lock_fence: Option<DeleteLockFence>,
pub replication_request: bool,
pub delete_marker: bool,
pub synthetic_version_id: bool,
pub transition: TransitionOptions,
pub expiration: ExpirationOptions,
+439 -49
View File
@@ -35,14 +35,17 @@ use crate::bucket::lifecycle::{
save_transition_transaction_record,
},
};
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, VersionPurgeStatusType, version_purge_status_to_filemeta};
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::OldCurrentSize;
use crate::error::is_err_invalid_upload_id;
use crate::object_api::DeleteLockFence;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
use futures::FutureExt as _;
use http::HeaderValue;
use rustfs_utils::path::decode_dir_object;
use std::future::Future;
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
@@ -3111,25 +3114,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
quorum_result
}
#[tracing::instrument(skip(self))]
#[tracing::instrument(skip(self, objects, opts))]
async fn delete_objects(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let delete_config_snapshot = opts
.delete_replication_config_snapshot
.clone()
.unwrap_or_else(|| Arc::new(DeleteReplicationConfigSnapshot::default()));
for object in &objects {
self.invalidate_get_object_metadata_cache(bucket, &object.object_name).await;
}
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
del_errs.push(None)
}
let mut del_errs = (0..objects.len()).map(|_| None).collect::<Vec<_>>();
// Acquire locks in batch mode (best effort, matching previous behavior)
let mut batch = rustfs_lock::BatchLockRequest::new(self.locker_owner.as_str()).with_all_or_nothing(false);
@@ -3195,14 +3197,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
let ver_cfg = BucketVersioningSys::get_in(&self.ctx, bucket).await.unwrap_or_default();
// backlog#929 (HP-8): the per-object stat below exists solely to feed
// check_object_lock_delete (#4297). Resolve the bucket lock
// configuration once (in-memory cache) and skip the whole stat fanout
// for buckets without Object Lock; unknown metadata fails closed and
// keeps the stat, so the #4297 protection is preserved verbatim for
// every object-lock-enabled bucket.
// Resolve Object Lock once; replication still reads each matching
// object's tags below while the batch write lock is held.
let object_lock_checks_required =
object_lock_delete_check_required(metadata_sys::get_in(&self.ctx, bucket).await.ok().as_deref());
@@ -3213,31 +3209,75 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
continue;
}
let replication_object_name = decode_dir_object(&dobj.object_name);
let explicit_null_version = is_explicit_null_version(dobj.version_id);
let version_id = delete_file_info_version_id(dobj.version_id);
if object_lock_checks_required {
let check_opts = ObjectOptions {
version_id: version_id.map(|version_id| version_id.to_string()),
versioned: ver_cfg.prefix_enabled(dobj.object_name.as_str()),
version_suspended: ver_cfg.suspended(),
object_lock_delete: opts.object_lock_delete.clone(),
no_lock: true,
..Default::default()
};
let (versioned, version_suspended) = delete_config_snapshot
.versioning_config()
.delete_state(replication_object_name.as_str());
let check_opts = ObjectOptions {
version_id: version_id.map(|version_id| version_id.to_string()),
versioned,
version_suspended,
object_lock_delete: opts.object_lock_delete.clone(),
no_lock: true,
..Default::default()
};
let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
let replication_needs_source = replicate_delete
&& (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
let (goi, gerr) = if object_lock_checks_required || replication_needs_source {
let (goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, &dobj.object_name, &check_opts).await;
if gerr.is_none()
&& let Err(err) = check_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts).await
{
del_errs[i] = Some(err);
continue;
}
(goi, gerr)
} else {
(ObjectInfo::default(), None)
};
let source_missing = gerr
.as_ref()
.is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
if let Some(err) = gerr.as_ref()
&& !source_missing
{
del_errs[i] = Some(err.clone());
continue;
}
if object_lock_checks_required
&& !source_missing
&& let Err(err) = check_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts).await
{
del_errs[i] = Some(err);
continue;
}
let mut admitted = dobj.clone();
admitted.object_name = replication_object_name;
if admitted.synthetic_version_id {
admitted.version_id = None;
}
if replicate_delete {
let dsc = ReplicationObjectBridge::check_delete_with_snapshot(
&admitted,
&goi,
&check_opts,
source_missing,
&delete_config_snapshot,
);
if dsc.replicate_any() {
if admitted.version_id.is_some() {
admitted.version_purge_status = Some(version_purge_status_to_filemeta(VersionPurgeStatusType::Pending));
admitted.version_purge_statuses = dsc.pending_status();
} else {
admitted.delete_marker_replication_status = dsc.pending_status();
}
admitted.replicate_decision_str = Some(dsc.to_string());
}
}
let mut vr = FileInfo {
name: dobj.object_name.clone(),
version_id,
idx: i,
replication_state_internal: Some(dobj.replication_state()),
replication_state_internal: Some(admitted.replication_state()),
..Default::default()
};
@@ -3247,14 +3287,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// del_objects[i].object_name.clone_from(&vr.name);
// del_objects[i].version_id = vr.version_id.map(|v| v.to_string());
if dobj.version_id.is_none() {
let (suspended, versioned) = (ver_cfg.suspended(), ver_cfg.prefix_enabled(dobj.object_name.as_str()));
if suspended || versioned {
vr.mod_time = Some(OffsetDateTime::now_utc());
vr.deleted = true;
if versioned {
vr.version_id = Some(Uuid::new_v4());
}
if dobj.version_id.is_none() && (version_suspended || versioned) {
vr.mod_time = Some(OffsetDateTime::now_utc());
vr.deleted = true;
if versioned {
vr.version_id = Some(Uuid::new_v4());
}
}
@@ -3312,6 +3349,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
vers.push(fi_vers);
}
if opts.delete_lock_fence.as_ref().is_some_and(DeleteLockFence::is_lock_lost) {
if dist_erasure {
self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
}
for (index, object) in objects.iter().enumerate() {
if del_errs[index].is_none() {
del_errs[index] = Some(Error::NamespaceLockQuorumUnavailable {
mode: "delete_objects_commit",
bucket: bucket.to_string(),
object: decode_dir_object(&object.object_name),
required: 1,
achieved: 0,
});
}
}
return (del_objects, del_errs);
}
let rollback_dir = Uuid::new_v4();
let disks = self.disks.read().await;
@@ -3490,6 +3545,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
let preserve_delete_replication_state = should_preserve_delete_replication_state(&opts);
let delete_config_snapshot = if opts.delete_prefix || opts.transition.expire_restored || preserve_delete_replication_state
{
None
} else if let Some(snapshot) = opts.delete_replication_config_snapshot.clone() {
Some(snapshot)
} else {
Some(Arc::new(DeleteReplicationConfigSnapshot::default()))
};
self.invalidate_get_object_metadata_cache(bucket, object).await;
// Guard lock for single object delete
@@ -3580,18 +3645,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
let otd = ObjectToDelete {
object_name: object.to_string(),
version_id: opts
.version_id
.clone()
.map(|v| Uuid::parse_str(v.as_str()).ok().unwrap_or_default()),
object_name: decode_dir_object(object),
version_id: if opts.synthetic_version_id {
None
} else {
opts.version_id.as_deref().map(Uuid::parse_str).transpose()?
},
synthetic_version_id: opts.synthetic_version_id,
..Default::default()
};
let dsc = if should_preserve_delete_replication_state(&opts) {
ReplicateDecision::default()
let dsc = if let Some(snapshot) = delete_config_snapshot {
ReplicationObjectBridge::check_delete_with_snapshot(&otd, &goi, &opts, gerr.is_some(), &snapshot)
} else {
ReplicationObjectBridge::check_delete(bucket, &otd, &goi, &opts, gerr.map(|e| e.to_string())).await
ReplicateDecision::default()
};
if dsc.replicate_any() {
@@ -3649,6 +3716,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
let mut oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
oi.user_tags = Arc::clone(&goi.user_tags);
oi.replication_decision = goi.replication_decision;
self.invalidate_get_object_metadata_cache(bucket, object).await;
return Ok(oi);
@@ -3680,6 +3748,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
obj_info.size = goi.size;
obj_info.user_tags = Arc::clone(&goi.user_tags);
self.invalidate_get_object_metadata_cache(bucket, object).await;
Ok(obj_info)
}
@@ -7801,6 +7870,56 @@ mod object_tagging_namespace_lock_tests {
);
}
#[tokio::test]
async fn version_delete_returns_tags_read_under_the_delete_lock() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "delete-locked-tags";
let object = "object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut reader = PutObjReader::from_vec(b"body".to_vec());
let uploaded = set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("versioned object should be written");
let version_id = uploaded.version_id.expect("versioned PUT should return an ID").to_string();
let version_opts = ObjectOptions {
versioned: true,
version_id: Some(version_id),
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())),
..Default::default()
};
set_disks
.put_object_tags(bucket, object, "generation=stale", &version_opts)
.await
.expect("initial tags should be written");
let advisory = set_disks
.get_object_info(bucket, object, &version_opts)
.await
.expect("advisory pre-read should succeed");
set_disks
.put_object_tags(bucket, object, "generation=locked", &version_opts)
.await
.expect("concurrent tag update should commit before delete");
let deleted = set_disks
.delete_object(bucket, object, version_opts)
.await
.expect("version delete should succeed");
assert_eq!(advisory.user_tags.as_str(), "generation=stale");
assert_eq!(deleted.user_tags.as_str(), "generation=locked");
}
#[tokio::test]
#[serial_test::serial]
async fn tagging_serializes_with_put_and_delete_for_versioned_and_unversioned_objects() {
@@ -7815,6 +7934,7 @@ mod object_tagging_namespace_lock_tests {
let object_opts = ObjectOptions {
versioned,
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())),
..Default::default()
};
let original_body = b"original body".to_vec();
@@ -8014,6 +8134,276 @@ mod delete_objects_lock_gating_tests {
}
}
#[tokio::test]
async fn delete_objects_derives_per_object_versioning_from_the_request_snapshot() {
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "batch-versioning-snapshot-bucket";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
put_plain_object(&set_disks, bucket, "marker-object").await;
put_plain_object(&set_disks, bucket, "archive/unversioned-object").await;
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
None,
));
let objects = vec![
ObjectToDelete {
object_name: "marker-object".to_string(),
..Default::default()
},
ObjectToDelete {
object_name: "archive/unversioned-object".to_string(),
..Default::default()
},
];
let (deleted, errors) = set_disks
.delete_objects(
bucket,
objects,
ObjectOptions {
versioned: true,
delete_replication_config_snapshot: Some(snapshot),
..Default::default()
},
)
.await;
assert!(
errors.iter().all(Option::is_none),
"snapshot-backed batch delete should succeed: {errors:?}"
);
assert!(deleted[0].delete_marker);
assert!(deleted[0].delete_marker_version_id.is_some());
assert!(!deleted[1].delete_marker);
set_disks
.get_object_info(bucket, "archive/unversioned-object", &ObjectOptions::default())
.await
.expect_err("the snapshot-excluded object should be removed without a delete marker");
}
#[tokio::test]
async fn batch_version_delete_uses_tags_read_under_the_delete_lock() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use s3s::dto::{
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, Tag, VersioningConfiguration,
};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "batch-delete-locked-tags";
let object = "object";
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut reader = PutObjReader::from_vec(b"body".to_vec());
let uploaded = set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=stale".to_string())]),
..Default::default()
},
)
.await
.expect("versioned object should be written");
let version_id = uploaded.version_id.expect("versioned PUT should return an ID");
let version_opts = ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
};
set_disks
.put_object_tags(bucket, object, "generation=locked", &version_opts)
.await
.expect("tag update should commit before the batch delete");
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: Some(ReplicationRuleFilter {
tag: Some(Tag {
key: Some("generation".to_string()),
value: Some("locked".to_string()),
}),
..Default::default()
}),
id: Some("delete".to_string()),
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
));
let (deleted, errors) = set_disks
.delete_objects(
bucket,
vec![ObjectToDelete {
object_name: object.to_string(),
version_id: Some(version_id),
..Default::default()
}],
ObjectOptions {
versioned: true,
delete_replication_config_snapshot: Some(snapshot),
..Default::default()
},
)
.await;
assert!(errors.iter().all(Option::is_none), "batch version delete should succeed: {errors:?}");
let state = deleted[0]
.replication_state
.as_ref()
.expect("locked decision should be persisted");
assert_eq!(
state.version_purge_status_internal.as_deref(),
Some("arn:rustfs:replication:us-east-1:target:bucket=PENDING;")
);
assert!(state.replicate_decision_str.contains(arn));
}
#[tokio::test]
async fn synthetic_directory_delete_uses_decoded_prefix_and_marker_switch() {
use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter,
ReplicationRuleStatus, VersioningConfiguration,
};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "batch-directory-replication";
let object = encode_dir_object("photos/");
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
put_plain_object(&set_disks, bucket, &object).await;
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
}),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
}),
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: Some(ReplicationRuleFilter {
prefix: Some("photos/".to_string()),
..Default::default()
}),
id: Some("directory-marker".to_string()),
prefix: None,
priority: Some(1),
source_selection_criteria: None,
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
));
let (deleted, errors) = set_disks
.delete_objects(
bucket,
vec![ObjectToDelete {
object_name: object,
version_id: Some(Uuid::nil()),
synthetic_version_id: true,
..Default::default()
}],
ObjectOptions {
versioned: true,
delete_replication_config_snapshot: Some(snapshot),
..Default::default()
},
)
.await;
assert!(errors[0].is_none(), "directory delete should succeed: {:?}", errors[0]);
let state = deleted[0]
.replication_state
.as_ref()
.expect("marker replication decision should be persisted");
assert_eq!(state.replication_status_internal.as_deref(), Some(format!("{arn}=PENDING;").as_str()));
assert!(state.version_purge_status_internal.is_none());
}
#[tokio::test]
async fn delete_objects_aborts_before_disk_mutation_after_outer_lock_loss() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "batch-lost-outer-lock";
let object = "object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
put_plain_object(&set_disks, bucket, object).await;
let (_deleted, errors) = set_disks
.delete_objects(
bucket,
vec![ObjectToDelete {
object_name: object.to_string(),
..Default::default()
}],
ObjectOptions {
no_lock: true,
delete_lock_fence: Some(DeleteLockFence::lost_for_test()),
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())),
..Default::default()
},
)
.await;
assert!(
matches!(errors[0], Some(Error::NamespaceLockQuorumUnavailable { .. })),
"lost outer lock must fail the batch before disk mutation: {:?}",
errors[0]
);
set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("object must survive a lost-lock batch");
}
#[tokio::test]
async fn delete_objects_blocks_locked_object_and_deletes_the_rest() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+1 -1
View File
@@ -519,7 +519,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
result
}
#[instrument(skip(self))]
#[instrument(skip(self, objects, opts))]
async fn delete_objects(
&self,
bucket: &str,
+25 -1
View File
@@ -13,7 +13,9 @@
// limitations under the License.
use super::*;
use crate::bucket::replication::ReplicationObjectBridge;
use crate::disk::OldCurrentSize;
use crate::object_api::DeleteLockFence;
use crate::set_disk::{
get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
is_lock_optimization_enabled, is_object_lock_diag_enabled,
@@ -156,6 +158,13 @@ impl ObjectLockDiagGuard {
acquired_at: Instant::now(),
}
}
fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
match &self.guard {
rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
rustfs_lock::NamespaceLockGuard::Fast(_) => None,
}
}
}
/// Opaque write-lock guard for the RestoreObject accept path; see
@@ -594,6 +603,9 @@ impl ECStore {
guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
}
opts.no_lock = true;
opts.delete_lock_fence = Some(DeleteLockFence::new(
guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal).collect(),
));
Ok(guards)
}
@@ -1222,7 +1234,7 @@ impl ECStore {
Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()))
}
#[instrument(skip(self))]
#[instrument(skip(self, objects, opts))]
pub(super) async fn handle_delete_objects(
&self,
bucket: &str,
@@ -1248,6 +1260,17 @@ impl ECStore {
}
let mut opts = opts;
if opts.delete_replication_config_snapshot.is_none() {
match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
Err(err) => {
let message = err.to_string();
let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
return (del_objects, errors);
}
}
}
let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
Ok(guards) => guards,
Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err),
@@ -2540,6 +2563,7 @@ mod tests {
assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
assert!(opts.delete_lock_fence.is_some(), "set layer must receive the outer write-lock loss fence");
let alpha_lock = store
.handle_new_ns_lock("bucket", "alpha")
@@ -15,6 +15,8 @@
mod storage_api;
use s3s::dto::ReplicationConfiguration;
use std::future::Future;
use storage_api::contract_compat::{Error, ObjectInfo, ObjectOptions, ObjectToDelete};
use storage_api::replication_compat::{
BucketStats, DeletedObjectReplicationInfo, DynReplicationPool, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
ReplicationConfigurationExt, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationObjectBridge,
@@ -34,6 +36,8 @@ fn type_name_unsized<T: ?Sized>() -> &'static str {
fn assert_replication_config_ext<T: ReplicationConfigurationExt>() {}
fn assert_strict_delete_future<F: Future<Output = Result<ReplicateDecision, Error>>>(_: &F) {}
#[test]
fn replication_facade_exports_config_extension_contract() {
assert_replication_config_ext::<ReplicationConfiguration>();
@@ -58,6 +62,13 @@ fn replication_facade_exports_runtime_and_dto_types() {
assert!(type_name::<DeletedObjectReplicationInfo>().contains("DeletedObjectReplicationInfo"));
assert!(type_name::<ReplicationObjectBridge>().contains("ReplicationObjectBridge"));
assert!(type_name_unsized::<DynReplicationPool>().contains("ReplicationPoolTrait"));
let object = ObjectToDelete::default();
let source = ObjectInfo::default();
let opts = ObjectOptions::default();
let future = ReplicationObjectBridge::check_delete_strict("bucket", &object, &source, &opts, None);
assert_strict_delete_future(&future);
assert!(!std::any::type_name_of_val(&future).is_empty());
}
#[test]
+187
View File
@@ -0,0 +1,187 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Operator entry point for the KMS disaster-recovery drill.
//!
//! Runs one rehearsal inside a scratch workspace and writes the evidence
//! bundle. Everything it touches lives under that workspace: it never reads,
//! writes, or restores into a running deployment's key directory. See
//! `docs/operations/kms-disaster-recovery-drill.md` for the procedure and for
//! how to size the dataset so the measured recovery time transfers.
//!
//! Exit status is the verdict: 0 when every check held, 1 otherwise, so a
//! scheduled drill fails its job instead of quietly filing a bad report.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill};
use std::path::PathBuf;
use std::process::ExitCode;
use zeroize::Zeroizing;
const ENV_WORKSPACE: &str = "RUSTFS_KMS_DRILL_WORKSPACE";
const ENV_DRILL_ID: &str = "RUSTFS_KMS_DRILL_ID";
const ENV_DISASTER: &str = "RUSTFS_KMS_DRILL_DISASTER";
const ENV_DEPLOYMENT: &str = "RUSTFS_KMS_DRILL_DEPLOYMENT";
const ENV_MASTER_KEY: &str = "RUSTFS_KMS_DRILL_MASTER_KEY";
const ENV_KEYS: &str = "RUSTFS_KMS_DRILL_KEYS";
const ENV_OBJECTS_PER_KEY: &str = "RUSTFS_KMS_DRILL_OBJECTS_PER_KEY";
const ENV_OBJECT_BYTES: &str = "RUSTFS_KMS_DRILL_OBJECT_BYTES";
const ENV_EVIDENCE: &str = "RUSTFS_KMS_DRILL_EVIDENCE";
const ENV_FILE_PERMISSIONS: &str = "RUSTFS_KMS_DRILL_FILE_PERMISSIONS";
// Deliberately the same names the admin backup API reads: drilling with the
// KEK the real backups are sealed under is what proves that KEK is still
// retrievable, which is the part of a recovery no bundle can attest to.
const ENV_KEK: &str = "RUSTFS_KMS_BACKUP_KEK";
const ENV_KEK_ID: &str = "RUSTFS_KMS_BACKUP_KEK_ID";
const ENV_KEK_VERSION: &str = "RUSTFS_KMS_BACKUP_KEK_VERSION";
fn usage() -> String {
format!(
"KMS disaster-recovery drill.\n\
\n\
Required:\n \
{ENV_WORKSPACE} absolute path to a scratch directory the drill owns\n \
{ENV_MASTER_KEY} at-rest master key for the throwaway rehearsal deployment\n \
{ENV_KEK} base64 of the 32-byte backup KEK\n \
{ENV_KEK_ID} identifier recorded in the bundle manifest\n\
\n\
Optional:\n \
{ENV_KEK_VERSION} backup KEK version (default 1)\n \
{ENV_DRILL_ID} drill identifier and key-id prefix (default kms-dr-drill)\n \
{ENV_DEPLOYMENT} deployment identity recorded in the bundle (default kms-dr-drill)\n \
{ENV_DISASTER} key-directory-lost | master-key-salt-lost | key-record-corrupted\n \
{ENV_KEYS} master keys to rehearse with (default 4)\n \
{ENV_OBJECTS_PER_KEY} objects sealed per key (default 2)\n \
{ENV_OBJECT_BYTES} plaintext bytes per object (default 4096)\n \
{ENV_FILE_PERMISSIONS} octal key-file mode (default 600)\n \
{ENV_EVIDENCE} evidence output path (default <workspace>/evidence.json)"
)
}
fn required(name: &str) -> Result<String, String> {
std::env::var(name)
.ok()
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| format!("{name} is required\n\n{}", usage()))
}
fn optional_usize(name: &str, default: usize) -> Result<usize, String> {
match std::env::var(name) {
Ok(value) => value
.trim()
.parse()
.map_err(|_| format!("{name} must be an unsigned integer")),
Err(_) => Ok(default),
}
}
fn disaster_from_env() -> Result<DrillDisaster, String> {
match std::env::var(ENV_DISASTER).ok().as_deref().map(str::trim) {
None | Some("") | Some("key-directory-lost") => Ok(DrillDisaster::KeyDirectoryLost),
Some("master-key-salt-lost") => Ok(DrillDisaster::MasterKeySaltLost),
Some("key-record-corrupted") => Ok(DrillDisaster::KeyRecordCorrupted),
Some(other) => Err(format!(
"{ENV_DISASTER}={other:?} is not a known disaster; use key-directory-lost, master-key-salt-lost, or key-record-corrupted"
)),
}
}
fn kek_from_env() -> Result<BackupKek, String> {
let raw = Zeroizing::new(required(ENV_KEK)?);
let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?);
if decoded.len() != 32 {
return Err(format!("{ENV_KEK} must decode to exactly 32 bytes"));
}
let mut material = [0u8; 32];
material.copy_from_slice(&decoded);
let version = optional_usize(ENV_KEK_VERSION, 1)?;
let version = u32::try_from(version).map_err(|_| format!("{ENV_KEK_VERSION} is out of range"))?;
BackupKek::new(required(ENV_KEK_ID)?, version, material).map_err(|error| error.to_string())
}
async fn run() -> Result<DrillVerdict, String> {
let workspace = PathBuf::from(required(ENV_WORKSPACE)?);
if !workspace.is_absolute() {
return Err(format!("{ENV_WORKSPACE} must be an absolute path"));
}
let drill_id = std::env::var(ENV_DRILL_ID)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| "kms-dr-drill".to_string());
let file_permissions = match std::env::var(ENV_FILE_PERMISSIONS) {
Ok(value) => Some(u32::from_str_radix(value.trim(), 8).map_err(|_| format!("{ENV_FILE_PERMISSIONS} must be octal"))?),
Err(_) => Some(0o600),
};
let request = DrillRequest {
deployment_identity: std::env::var(ENV_DEPLOYMENT)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| drill_id.clone()),
drill_id,
workspace: workspace.clone(),
rustfs_version: env!("CARGO_PKG_VERSION").to_string(),
// A drill always restores into a target that has observed nothing, so
// any generation is monotonic; the number is recorded as evidence of
// which snapshot the rehearsal covered.
snapshot_generation: 1,
dataset: DrillDataset {
keys: optional_usize(ENV_KEYS, 4)?,
objects_per_key: optional_usize(ENV_OBJECTS_PER_KEY, 2)?,
object_bytes: optional_usize(ENV_OBJECT_BYTES, 4096)?,
},
disaster: disaster_from_env()?,
master_key: required(ENV_MASTER_KEY)?,
file_permissions,
};
let kek = kek_from_env()?;
let evidence = run_local_drill(&kek, &request).await.map_err(|error| error.to_string())?;
let evidence_path = std::env::var(ENV_EVIDENCE)
.ok()
.filter(|value| !value.trim().is_empty())
.map_or_else(|| workspace.join("evidence.json"), PathBuf::from);
let encoded = evidence.encode().map_err(|error| error.to_string())?;
std::fs::write(&evidence_path, &encoded).map_err(|error| format!("cannot write {}: {error}", evidence_path.display()))?;
eprintln!("verdict: {:?}", evidence.verdict);
eprintln!("disaster: {:?}", evidence.disaster);
eprintln!(
"objects: {}/{} historical objects decrypted after the restore",
evidence.envelope_probes.iter().filter(|probe| probe.verified).count(),
evidence.envelope_probes.len()
);
eprintln!("rpo: {} ms of work past the snapshot fence was lost", evidence.rpo.rpo_window_millis);
eprintln!("rto: {} ms of recovery work", evidence.rto_millis);
for finding in &evidence.findings {
eprintln!("finding: {finding}");
}
eprintln!("evidence: {}", evidence_path.display());
Ok(evidence.verdict)
}
#[tokio::main]
async fn main() -> ExitCode {
match run().await {
Ok(DrillVerdict::Passed) => ExitCode::SUCCESS,
Ok(DrillVerdict::Failed) => ExitCode::FAILURE,
Err(message) => {
eprintln!("{message}");
ExitCode::FAILURE
}
}
}
+194 -15
View File
@@ -15,9 +15,10 @@
//! API types for KMS dynamic configuration
use crate::config::{
BackendConfig, CacheConfig, DEFAULT_CACHE_TTL, DEFAULT_MAX_CACHED_KEYS, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend, KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod,
VaultConfig, VaultTransitConfig, allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option,
AwsKmsConfig, BackendConfig, CacheConfig, DEFAULT_CACHE_TTL, DEFAULT_MAX_CACHED_KEYS,
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend, KmsConfig, LocalConfig,
StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, allow_immediate_deletion_from_env,
redacted_secret, redacted_secret_option,
};
use crate::service_manager::KmsServiceStatus;
use crate::types::{KeyMetadata, KeyUsage};
@@ -165,6 +166,47 @@ pub struct ConfigureStaticKmsRequest {
pub allow_insecure_dev_defaults: Option<bool>,
}
/// Request to configure KMS with the AWS KMS backend.
///
/// Accepts no credential material by design: every node resolves AWS
/// credentials through the standard `aws-config` provider chain, so nothing
/// secret is submitted here, persisted with the cluster configuration, or
/// echoed back by the status API.
///
/// Keys are not created by this path. The backend refuses caller-named key
/// creation because AWS assigns identifiers itself, so `default_key_id` must
/// name a key that already exists in AWS, by key id or ARN.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ConfigureAwsKmsRequest {
/// AWS region hosting the keys.
///
/// Mandatory here, unlike the environment-variable path: this
/// configuration is persisted once and replayed on every node, so leaving
/// the region to each node's ambient provider chain would let nodes
/// silently address different regions — and therefore different keys —
/// while reporting the same configuration.
pub region: String,
/// Endpoint override for local emulators and private endpoints. Unset in
/// production, where the SDK derives the regional endpoint; a plaintext
/// endpoint stays gated behind the development opt-in.
pub endpoint_url: Option<String>,
/// Default master key ID for auto-encryption, as an AWS key id or ARN
pub default_key_id: Option<String>,
/// Operation timeout in seconds
pub timeout_seconds: Option<u64>,
/// Number of retry attempts
pub retry_attempts: Option<u32>,
/// Enable caching
pub enable_cache: Option<bool>,
/// Maximum number of keys to cache
pub max_cached_keys: Option<usize>,
/// Cache TTL in seconds
pub cache_ttl_seconds: Option<u64>,
/// Allow development-only insecure defaults
pub allow_insecure_dev_defaults: Option<bool>,
}
impl Drop for ConfigureStaticKmsRequest {
fn drop(&mut self) {
use zeroize::Zeroize;
@@ -211,6 +253,9 @@ pub enum ConfigureKmsRequest {
/// Configure with Static single-key backend
#[serde(rename = "Static", alias = "static")]
Static(ConfigureStaticKmsRequest),
/// Configure with the AWS KMS backend
#[serde(rename = "AWS", alias = "AwsKms", alias = "aws", alias = "aws-kms", alias = "aws_kms")]
Aws(ConfigureAwsKmsRequest),
}
/// KMS configuration response
@@ -636,6 +681,33 @@ impl ConfigureStaticKmsRequest {
}
}
impl ConfigureAwsKmsRequest {
/// Convert to KmsConfig
pub fn to_kms_config(&self) -> KmsConfig {
KmsConfig {
backend: KmsBackend::Aws,
default_key_id: self.default_key_id.clone(),
backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
region: Some(self.region.clone()),
endpoint_url: self.endpoint_url.clone(),
})),
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
// Read from server configuration, never from the request body: the
// gate must mean the same thing whether KMS was configured at
// startup or through this endpoint.
allow_immediate_deletion: allow_immediate_deletion_from_env(),
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
retry_attempts: self.retry_attempts.unwrap_or(3),
enable_cache: self.enable_cache.unwrap_or(true),
cache_config: CacheConfig {
max_keys: self.max_cached_keys.unwrap_or(DEFAULT_MAX_CACHED_KEYS),
ttl: self.cache_ttl_seconds.map_or(DEFAULT_CACHE_TTL, Duration::from_secs),
..CacheConfig::default()
},
}
}
}
impl ConfigureKmsRequest {
/// Convert to KmsConfig
pub fn to_kms_config(&self) -> KmsConfig {
@@ -644,6 +716,7 @@ impl ConfigureKmsRequest {
ConfigureKmsRequest::VaultKv2(req) => req.to_kms_config(),
ConfigureKmsRequest::VaultTransit(req) => req.to_kms_config(),
ConfigureKmsRequest::Static(req) => req.to_kms_config(),
ConfigureKmsRequest::Aws(req) => req.to_kms_config(),
}
}
}
@@ -829,6 +902,108 @@ mod tests {
assert!(request.to_kms_config().validate().is_ok());
}
#[test]
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
let raw = serde_json::json!({
"backend_type": backend_type,
"region": "eu-central-1",
"default_key_id": "arn:aws:kms:eu-central-1:111122223333:key/1234abcd"
});
let request: ConfigureKmsRequest = serde_json::from_value(raw).unwrap_or_else(|e| panic!("{backend_type}: {e}"));
let config = request.to_kms_config();
assert_eq!(config.backend, KmsBackend::Aws, "backend_type={backend_type}");
let aws = config.aws_kms_config().expect("aws backend config");
assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
assert_eq!(aws.endpoint_url, None);
assert!(config.validate().is_ok(), "backend_type={backend_type}");
}
}
/// A cluster-persisted AWS configuration must pin its own region: a
/// request that leaves it to each node's ambient provider chain is refused
/// rather than accepted into a configuration every node interprets
/// differently.
#[test]
fn test_aws_configure_request_requires_an_explicit_region() {
let missing = serde_json::json!({
"backend_type": "AWS",
"default_key_id": "arn:aws:kms:eu-central-1:111122223333:key/1234abcd"
});
let err = serde_json::from_value::<ConfigureKmsRequest>(missing).expect_err("a region-less AWS request must be refused");
assert!(err.to_string().contains("region"), "{err}");
let empty = serde_json::json!({ "backend_type": "AWS", "region": "" });
let request: ConfigureKmsRequest = serde_json::from_value(empty).expect("an empty region deserializes");
assert!(request.to_kms_config().validate().is_err(), "an empty region must not validate");
}
#[test]
fn test_aws_configure_request_rejects_plaintext_endpoint_without_opt_in() {
let raw = serde_json::json!({
"backend_type": "AWS",
"region": "us-east-1",
"endpoint_url": "http://localhost:4566"
});
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("aws request should deserialize");
assert!(request.to_kms_config().validate().is_err());
let opt_in = serde_json::json!({
"backend_type": "AWS",
"region": "us-east-1",
"endpoint_url": "http://localhost:4566",
"allow_insecure_dev_defaults": true
});
let request: ConfigureKmsRequest = serde_json::from_value(opt_in).expect("aws request should deserialize");
assert!(request.to_kms_config().validate().is_ok());
}
/// The AWS summary carries only non-credential settings, because the
/// backend never holds AWS credential material to begin with.
#[test]
fn test_aws_status_summary_reports_only_non_credential_settings() {
let config = ConfigureAwsKmsRequest {
region: "us-east-1".to_string(),
endpoint_url: None,
default_key_id: Some("arn:aws:kms:us-east-1:111122223333:key/1234abcd".to_string()),
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: None,
}
.to_kms_config();
let summary = KmsConfigSummary::from(&config);
assert_eq!(summary.backend_type, KmsBackend::Aws);
match &summary.backend_summary {
BackendSummary::Aws { region, endpoint_url } => {
assert_eq!(region.as_deref(), Some("us-east-1"));
assert_eq!(endpoint_url.as_deref(), None);
}
other => panic!("expected aws summary, got {other:?}"),
}
let response = KmsStatusResponse {
status: KmsServiceStatus::Running,
backend_type: Some(config.backend),
healthy: Some(true),
config_summary: Some(summary),
};
let rendered = format!(
"{}\n{response:?}",
serde_json::to_string(&response).expect("kms status response should serialize")
);
for credential_field in ["access_key", "secret_key", "session_token", "has_stored_credentials"] {
assert!(
!rendered.contains(credential_field),
"aws status output must not describe credential material: {rendered}"
);
}
}
#[test]
fn test_configure_request_rejects_unknown_fields() {
let raw = serde_json::json!({
@@ -853,6 +1028,17 @@ mod tests {
let err = serde_json::from_value::<ConfigureKmsRequest>(raw).expect_err("unknown auth field should fail");
assert!(err.to_string().contains("unknown field"));
// AWS credentials belong to the provider chain: a request that tries to
// smuggle them in must be refused, not silently ignored.
let raw = serde_json::json!({
"backend_type": "AWS",
"region": "us-east-1",
"secret_access_key": "AKIA-not-accepted-here"
});
let err = serde_json::from_value::<ConfigureKmsRequest>(raw).expect_err("unknown aws field should fail");
assert!(err.to_string().contains("unknown field"));
}
#[test]
@@ -1157,18 +1343,11 @@ pub struct CreateKeyResponse {
pub key_metadata: KeyMetadata,
}
/// Request to delete a key
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteKeyRequest {
/// Key ID to delete
pub key_id: String,
/// Number of days to wait before deletion (7-30 days, optional)
pub pending_window_in_days: Option<u32>,
/// Force immediate deletion (for development/testing only)
pub force_immediate: Option<bool>,
}
/// Response from delete key operation
/// JSON shape returned by the admin delete-key endpoint.
///
/// The delete *request* shape lives in [`crate::types::DeleteKeyRequest`] —
/// there is deliberately no copy here, because the immediate-deletion gate
/// (`force_immediate` + `confirm_key_id`) must have exactly one definition.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteKeyResponse {
/// Success flag
+2
View File
@@ -156,6 +156,8 @@ pub fn error_class(error: &KmsError) -> &'static str {
KmsError::UnsupportedCapability { .. } => "unsupported_capability",
KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
KmsError::BaselineVersionLost { .. } => "baseline_version_lost",
KmsError::KeyStillReferenced { .. } => "key_still_referenced",
KmsError::RewrapWouldExposePlaintext { .. } => "rewrap_would_expose_plaintext",
}
}
+28 -1
View File
@@ -67,7 +67,7 @@ use aws_smithy_types::Blob;
use jiff::Zoned;
use tokio_util::sync::CancellationToken;
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend};
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
use crate::config::{BackendConfig, KmsConfig};
use crate::error::{KmsError, Result};
use crate::policy::{self, AttemptError, ErrorClass, OpClass, RetryPolicy, classify_status};
@@ -577,6 +577,13 @@ impl KmsBackend for AwsKmsBackend {
/// and creation date every caller relies on costs one `DescribeKey` per
/// listed key. The page size is bounded by the caller's `limit`.
async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
// AWS rejects a `Limit` of zero, and clamping it up to one would return
// a key to a caller that asked for none; the empty page is answered
// here instead.
if list_keys_page_size(request.limit).is_none() {
return Ok(empty_key_page());
}
let limit = request
.limit
.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX).clamp(1, 1000));
@@ -1113,6 +1120,26 @@ mod tests {
assert_eq!(http_client.actual_requests().count(), 0, "no key may be created in AWS");
}
/// AWS rejects `Limit: 0` outright, so the request cannot be forwarded as
/// written; clamping it up to one would answer a caller that asked for no
/// keys with a key. The empty page is served locally instead.
#[tokio::test]
async fn zero_limit_list_returns_an_empty_page_without_calling_aws() {
let (http_client, backend) = scripted_backend(Vec::new());
let response = backend
.list_keys(ListKeysRequest {
limit: Some(0),
..Default::default()
})
.await
.expect("a zero-limit list must succeed");
assert!(response.keys.is_empty());
assert!(!response.truncated);
assert!(response.next_marker.is_none());
assert_eq!(http_client.actual_requests().count(), 0, "no request should reach AWS");
}
/// Signing keys are outside the envelope-encryption surface this backend
/// serves; creating one is refused rather than silently downgraded.
#[tokio::test]
+43 -4
View File
@@ -37,8 +37,9 @@ use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::service::ObjectEncryptionService;
use crate::types::{
CancelKeyDeletionRequest, CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, EncryptRequest,
GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext,
CancelKeyDeletionRequest, CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeDataKeyWrappingRequest,
DescribeKeyRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext,
RewrapDataKeyRequest,
};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
@@ -64,6 +65,23 @@ async fn expect_rotate_rejected(backend: &dyn KmsBackend, key_id: &str) {
}
}
/// Rewrap while not Enabled: it produces a new wrapping, so backends that
/// support it must gate it through the state machine exactly as encryption is
/// gated; backends without version history report the capability gap instead.
async fn expect_rewrap_rejected(backend: &dyn KmsBackend, ciphertext: Vec<u8>) {
let result = backend
.rewrap_data_key(RewrapDataKeyRequest {
ciphertext,
encryption_context: context(),
})
.await;
if backend.capabilities().rewrap {
expect_invalid_key_state(result, "");
} else {
expect_unsupported(result);
}
}
fn expect_invalid_key_state<T: std::fmt::Debug>(result: Result<T>, expected_fragment: &str) {
match result {
Err(KmsError::InvalidOperation { message }) => assert!(
@@ -158,6 +176,7 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) {
expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "disabled");
expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "disabled");
expect_rotate_rejected(backend, key_id).await;
expect_rewrap_rejected(backend, data_key.ciphertext_blob.clone()).await;
// ...but decryption of existing data keeps working (explicit AWS deviation)...
let decrypted = backend
.decrypt(decrypt_request(data_key.ciphertext_blob.clone()))
@@ -187,6 +206,7 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) {
expect_invalid_key_state(backend.enable_key(key_id).await, "pending deletion");
expect_invalid_key_state(backend.disable_key(key_id).await, "pending deletion");
expect_rotate_rejected(backend, key_id).await;
expect_rewrap_rejected(backend, data_key.ciphertext_blob.clone()).await;
expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "pending deletion");
let decrypted = backend
.decrypt(decrypt_request(data_key.ciphertext_blob.clone()))
@@ -282,11 +302,30 @@ async fn static_backend_stateless_contract() {
expect_invalid_key_state(backend.create_key(create_request("another-key".to_string())).await, "read-only");
expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "read-only");
expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "read-only");
// Enable/disable and rotation are capability gaps at the product
// surface, not state-machine rejections.
// Enable/disable, rotation and rewrap are capability gaps at the product
// surface, not state-machine rejections. A single fixed key has no second
// version to rewrap onto, so reporting the gap is the only honest answer —
// re-wrapping with the same material would look like progress while
// changing nothing.
expect_unsupported(backend.enable_key(key_id).await);
expect_unsupported(backend.disable_key(key_id).await);
expect_unsupported(backend.rotate_key(key_id).await);
expect_unsupported(
backend
.rewrap_data_key(RewrapDataKeyRequest {
ciphertext: data_key.ciphertext_blob.clone(),
encryption_context: context(),
})
.await,
);
expect_unsupported(
backend
.describe_data_key_wrapping(DescribeDataKeyWrappingRequest {
ciphertext: data_key.ciphertext_blob,
encryption_context: context(),
})
.await,
);
}
fn vault_dev_config(constructor: fn(url::Url, String) -> KmsConfig) -> KmsConfig {
+444 -48
View File
@@ -16,7 +16,7 @@
use crate::backends::{
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
ensure_tag_keys_are_mutable,
ensure_tag_keys_are_mutable, paginate_keys,
};
use crate::config::KmsConfig;
use crate::config::LocalConfig;
@@ -480,6 +480,36 @@ pub(crate) enum StoredKeyProtection {
PlaintextDevOnly,
}
/// The record's `at_rest_protection` value when this build cannot interpret
/// it, rendered for diagnostics. `Ok(None)` means the marker is absent
/// (pre-beta.9 records) or names a protection mode this build implements.
///
/// Every reader of a stored key record must consult this before its own
/// schema parse. Letting a strict [`StoredKeyProtection`] field fail inside a
/// larger struct collapses "written by a newer build" into "corrupt", and an
/// operator who reads corruption starts a disaster recovery instead of a
/// version rollback. The probe deliberately ignores every other field, so the
/// verdict is available even for records whose schema this build cannot
/// satisfy, and no key material is copied out of the caller's buffer.
///
/// `Err` carries the JSON error so callers can keep their own classification
/// for bytes that are not a record at all.
pub(crate) fn unknown_protection_marker(record: &[u8]) -> serde_json::Result<Option<String>> {
#[derive(Deserialize)]
struct MarkerProbe {
#[serde(default)]
at_rest_protection: Option<serde_json::Value>,
}
let Some(marker) = serde_json::from_slice::<MarkerProbe>(record)?.at_rest_protection else {
return Ok(None);
};
if serde_json::from_value::<StoredKeyProtection>(marker.clone()).is_ok() {
return Ok(None);
}
Ok(Some(marker.as_str().map(str::to_owned).unwrap_or_else(|| marker.to_string())))
}
/// Serializable representation of a master key stored on disk
#[derive(Debug, Clone, Serialize, Deserialize)]
struct StoredMasterKey {
@@ -741,10 +771,18 @@ impl LocalKmsClient {
/// real problem (restore the salt file or the whole directory) instead of
/// a generic decrypt failure.
///
/// Files that do not parse are ignored here — startup key validation
/// reports them with their own errors right after. Legacy pre-marker files
/// are also ignored: pre-beta.9 directories legitimately have no salt file
/// yet, and an empty directory must keep initializing as before.
/// A record this build cannot read or cannot interpret blocks generation
/// just as hard. Skipping it would publish a fresh salt over a directory
/// whose protection state is unknown, and that publication is
/// irreversible in the only way that matters: the next startup finds a
/// salt file, never re-enters this guard, and the evidence that the real
/// salt was missing is gone. Every record this rejects also fails startup
/// key validation a few lines later, so no directory that initializes
/// today stops initializing — the salt is simply no longer written first.
///
/// Legacy pre-marker files stay allowed: they parse here, pre-beta.9
/// directories legitimately have no salt file yet, and an empty directory
/// must keep initializing as before.
async fn ensure_missing_salt_can_be_generated(config: &LocalConfig) -> Result<()> {
#[derive(Deserialize)]
struct ProtectionProbe {
@@ -758,12 +796,23 @@ impl LocalKmsClient {
if path.extension().is_none_or(|extension| extension != "key") {
continue;
}
let Ok(content) = fs::read(&path).await else {
continue;
};
let Ok(probe) = serde_json::from_slice::<ProtectionProbe>(&content) else {
continue;
};
let content = fs::read(&path).await.map_err(|error| {
KmsError::configuration_error(format!(
"Local KMS master key salt at {} is missing and key record {} cannot be read ({error}); \
refusing to generate a replacement salt while a record's protection state is unknown",
Self::master_key_salt_path(config).display(),
path.display()
))
})?;
let probe = serde_json::from_slice::<ProtectionProbe>(&content).map_err(|error| {
KmsError::configuration_error(format!(
"Local KMS master key salt at {} is missing and key record {} is not interpretable by this build ({error}); \
refusing to generate a replacement salt restore the salt file from backup, or run a build that \
understands the record",
Self::master_key_salt_path(config).display(),
path.display()
))
})?;
if probe.at_rest_protection == StoredKeyProtection::EncryptedMasterKey {
return Err(KmsError::configuration_error(format!(
"Local KMS master key salt at {} is missing but {} is marked encrypted-master-key; \
@@ -805,15 +854,12 @@ impl LocalKmsClient {
// Two-stage parse so an unrecognised protection marker is reported as an
// unsupported format (a newer build may still read the key) instead of being
// folded into generic corruption with every other malformed record.
let raw: serde_json::Value = serde_json::from_slice(&content)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record is not valid JSON: {e}")))?;
if let Some(marker) = raw.get("at_rest_protection")
&& serde_json::from_value::<StoredKeyProtection>(marker.clone()).is_err()
{
let version = marker.as_str().map(str::to_owned).unwrap_or_else(|| marker.to_string());
let unknown_marker = unknown_protection_marker(&content)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record is not a readable JSON object: {e}")))?;
if let Some(version) = unknown_marker {
return Err(KmsError::unsupported_format_version(key_id, version));
}
let stored_key: StoredMasterKey = serde_json::from_value(raw)
let stored_key: StoredMasterKey = serde_json::from_slice(&content)
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record does not deserialize: {e}")))?;
if stored_key.key_id != key_id {
return Err(KmsError::invalid_key(format!(
@@ -1214,6 +1260,32 @@ impl LocalKmsClient {
Ok(master_key.into())
}
/// Every key identifier in the key directory, sorted.
///
/// `read_dir` order is arbitrary and may differ between calls over the same
/// directory, so it cannot carry a pagination cursor. Sorting gives the key
/// set a stable total order that a marker can point into.
async fn sorted_key_ids(&self) -> Result<Vec<String>> {
let mut key_ids = Vec::new();
let mut entries = fs::read_dir(&self.config.key_dir).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.extension().is_some_and(|ext| ext == "key")
&& let Some(key_id) = path.file_stem().and_then(|stem| stem.to_str())
{
key_ids.push(key_id.to_string());
}
}
key_ids.sort_unstable();
Ok(key_ids)
}
/// One page of the key set, ordered by key identifier.
///
/// Paging is real here rather than a first-page-only approximation:
/// callers that must see every key — the deletion sweep above all, which
/// only destroys expired material for keys it actually lists — depend on
/// `truncated` and `next_marker` to reach past the first page.
pub(crate) async fn list_keys(
&self,
request: &ListKeysRequest,
@@ -1221,44 +1293,48 @@ impl LocalKmsClient {
) -> Result<ListKeysResponse> {
debug!("Listing keys");
let mut keys = Vec::new();
let limit = request.limit.unwrap_or(100) as usize;
let mut count = 0;
let key_ids = self.sorted_key_ids().await?;
let page = paginate_keys(&key_ids, request, String::as_str);
let mut entries = fs::read_dir(&self.config.key_dir).await?;
// Only the page is read from disk, so the cost of a list stays bounded
// by the requested limit rather than by the size of the key set.
let mut keys = Vec::with_capacity(page.items.len());
for key_id in page.items {
let key_info = match self.describe_key(key_id, None).await {
Ok(key_info) => key_info,
// A key that vanished between the scan and the read is dropped
// from the page: concurrent removal is normal, and the cursor
// is derived from the identifier list, so the listing still
// advances past it.
Err(KmsError::KeyNotFound { .. }) => {
debug!(key_id, "skipping key removed while listing");
continue;
}
// Anything else means the record is still there and this build
// cannot interpret it. Dropping it would answer "these are
// your keys" with a set that silently omits one, and the
// deletion sweep would count a census it never fully saw.
Err(error) => return Err(error),
};
while let Some(entry) = entries.next_entry().await? {
if count >= limit {
break;
}
let path = entry.path();
if path.extension().is_some_and(|ext| ext == "key")
&& let Some(stem) = path.file_stem()
&& let Some(key_id) = stem.to_str()
&& let Ok(key_info) = self.describe_key(key_id, None).await
if let Some(ref status_filter) = request.status_filter
&& &key_info.status != status_filter
{
// Apply filters
if let Some(ref status_filter) = request.status_filter
&& &key_info.status != status_filter
{
continue;
}
if let Some(ref usage_filter) = request.usage_filter
&& &key_info.usage != usage_filter
{
continue;
}
keys.push(key_info);
count += 1;
continue;
}
if let Some(ref usage_filter) = request.usage_filter
&& &key_info.usage != usage_filter
{
continue;
}
keys.push(key_info);
}
Ok(ListKeysResponse {
keys,
next_marker: None, // Simple implementation without pagination
truncated: false,
next_marker: page.next_marker,
truncated: page.truncated,
})
}
@@ -2937,6 +3013,129 @@ mod tests {
);
}
/// The salt guard must fail closed on every record it cannot interpret,
/// not just on the ones that spell out `encrypted-master-key`.
///
/// Skipping such a record publishes a fresh salt, and that write is the
/// irreversible step: the next startup sees a salt file, never re-enters
/// the guard, and the only signal that the real salt was lost is gone.
/// Every input below also fails startup key validation, so this rejects
/// nothing that used to initialize — it only stops the salt from being
/// written before that failure.
#[tokio::test]
async fn missing_salt_with_uninterpretable_key_records_fails_closed_without_generating_a_salt() {
let (client, temp_dir) = create_test_client().await;
client.create_key("sealed-key", "AES_256", None).await.expect("create key");
let config = client.config.clone();
let key_path = client.master_key_path("sealed-key").expect("valid key id");
let salt_path = LocalKmsClient::master_key_salt_path(&config);
let pristine: serde_json::Value =
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode record");
drop(client);
let mut newer_build_record = pristine.clone();
newer_build_record["at_rest_protection"] = serde_json::json!("post-quantum-v2");
let newer_build_record = serde_json::to_vec_pretty(&newer_build_record).expect("encode record");
let truncated = {
let bytes = serde_json::to_vec_pretty(&pristine).expect("encode record");
bytes[..bytes.len() / 2].to_vec()
};
for (name, content) in [
("record from a newer build", newer_build_record),
("record that does not decode", truncated),
] {
fs::write(&key_path, &content).await.expect("write record");
fs::remove_file(&salt_path).await.ok();
let error = match LocalKmsClient::new(config.clone()).await {
Ok(_) => panic!("{name}: a missing salt must not be papered over"),
Err(error) => error,
};
// Asserted first: publishing a replacement salt is the
// irreversible step, and it happens before any error is produced.
assert!(!salt_path.exists(), "{name}: a replacement salt must never be generated");
assert!(
matches!(error, KmsError::ConfigurationError { .. }),
"{name}: expected a salt-specific configuration error, got {error:?}"
);
assert!(error.to_string().contains("salt"), "{name}: error must point at the salt: {error}");
assert_eq!(
sorted_dir_file_names(temp_dir.path()).await,
vec!["sealed-key.key".to_string()],
"{name}: the failed startup must not modify the key directory"
);
}
}
/// Compatibility floor for the guard above: a record written before the
/// protection marker existed carries no marker at all, and such a
/// directory legitimately has no salt yet. It must keep initializing.
/// A record this build cannot interpret must not be edited out of a
/// listing. The page would claim to describe the key set while omitting a
/// key that is still on disk, and the deletion sweep — which counts the
/// lifecycle gauges out of the pages it lists — would report a census it
/// never fully saw as complete.
#[tokio::test]
async fn list_keys_fails_closed_on_a_record_it_cannot_interpret() {
let (client, _temp_dir) = create_test_client().await;
client.create_key("alpha", "AES_256", None).await.expect("create alpha");
client.create_key("beta", "AES_256", None).await.expect("create beta");
let key_path = client.master_key_path("beta").expect("valid key id");
let mut record: serde_json::Value =
serde_json::from_slice(&fs::read(&key_path).await.expect("read record")).expect("decode record");
record["at_rest_protection"] = serde_json::json!("post-quantum-v2");
fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode record"))
.await
.expect("write record");
let error = client
.list_keys(&ListKeysRequest::default(), None)
.await
.expect_err("a listing must not quietly omit a key it cannot read");
assert!(
matches!(&error, KmsError::UnsupportedFormatVersion { key_id, version }
if key_id == "beta" && version == "post-quantum-v2"),
"got {error:?}"
);
}
#[tokio::test]
async fn missing_salt_with_pre_marker_key_records_still_initializes() {
let (dev_client, temp_dir) = create_dev_mode_client().await;
dev_client
.create_key("pre-marker-key", "AES_256", None)
.await
.expect("create key");
let key_path = dev_client.master_key_path("pre-marker-key").expect("valid key id");
let mut record: serde_json::Value =
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode record");
drop(dev_client);
// Pre-beta.9 records carry no protection marker at all, and their
// directories legitimately hold no salt file yet.
record
.as_object_mut()
.expect("record is an object")
.remove("at_rest_protection");
fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode record"))
.await
.expect("write record");
let client = LocalKmsClient::new(test_config(temp_dir.path()))
.await
.expect("a pre-marker directory must keep initializing");
assert!(
LocalKmsClient::master_key_salt_path(&client.config).exists(),
"first-time salt creation must still happen"
);
client
.describe_key("pre-marker-key", None)
.await
.expect("the pre-marker record must stay readable");
}
#[tokio::test]
async fn missing_salt_with_only_plaintext_dev_keys_still_initializes() {
let (dev_client, temp_dir) = create_dev_mode_client().await;
@@ -3087,4 +3286,201 @@ mod tests {
.expect("repeat removal");
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
}
// -- Listing and pagination ---------------------------------------------
fn page_request(limit: u32, marker: Option<&str>) -> ListKeysRequest {
ListKeysRequest {
limit: Some(limit),
marker: marker.map(str::to_string),
usage_filter: None,
status_filter: None,
}
}
fn filtered_page_request(limit: u32, marker: Option<&str>, status: KeyStatus) -> ListKeysRequest {
ListKeysRequest {
status_filter: Some(status),
..page_request(limit, marker)
}
}
async fn create_keys(client: &LocalKmsClient, key_ids: &[String]) {
for key_id in key_ids {
client
.create_key(key_id, "AES_256", None)
.await
.expect("key should be created");
}
}
/// Paging must reach every key exactly once. A backend that reports the
/// first page as the whole key set strands everything behind it — the
/// deletion sweep would never destroy expired material past page one.
#[tokio::test]
async fn list_keys_pages_through_the_whole_key_set() {
let (client, _temp_dir) = create_test_client().await;
let expected: Vec<String> = (0..7).map(|index| format!("page-key-{index:02}")).collect();
create_keys(&client, &expected).await;
let mut seen = Vec::new();
let mut marker: Option<String> = None;
// Bounded so a listing that cannot advance fails the assertion below
// instead of hanging the test run.
for _ in 0..expected.len() + 1 {
let response = client
.list_keys(&page_request(3, marker.as_deref()), None)
.await
.expect("list should succeed");
assert!(response.keys.len() <= 3, "a page must not exceed the requested limit");
seen.extend(response.keys.iter().map(|key| key.key_id.clone()));
if !response.truncated {
assert!(response.next_marker.is_none(), "a final page must not offer a cursor");
break;
}
marker = Some(response.next_marker.expect("a truncated page must offer a cursor"));
}
assert_eq!(seen, expected, "paging must visit every key exactly once, in identifier order");
}
/// Exact-limit boundary: a page that ends on the last key is complete.
#[tokio::test]
async fn list_keys_page_ending_on_the_last_key_is_not_truncated() {
let (client, _temp_dir) = create_test_client().await;
let key_ids: Vec<String> = (0..3).map(|index| format!("exact-key-{index}")).collect();
create_keys(&client, &key_ids).await;
let response = client
.list_keys(&page_request(3, None), None)
.await
.expect("list should succeed");
assert_eq!(response.keys.len(), 3);
assert!(!response.truncated);
assert!(response.next_marker.is_none());
// One key short of the set, the same listing is truncated.
let response = client
.list_keys(&page_request(2, None), None)
.await
.expect("list should succeed");
assert!(response.truncated);
assert_eq!(response.next_marker.as_deref(), Some("exact-key-1"));
}
/// The cursor is an identifier, not an index, so it keeps working after the
/// key it names is destroyed — which is exactly what the deletion sweep
/// does to the keys it retires between pages.
#[tokio::test]
async fn list_keys_resumes_after_a_deleted_marker_key() {
let (client, _temp_dir) = create_test_client().await;
let key_ids: Vec<String> = ["marker-a", "marker-b", "marker-c"].iter().map(|id| id.to_string()).collect();
create_keys(&client, &key_ids).await;
fs::remove_file(client.master_key_path("marker-b").expect("key path"))
.await
.expect("key file should be removable");
let response = client
.list_keys(&page_request(10, Some("marker-b")), None)
.await
.expect("list should succeed");
let listed: Vec<&str> = response.keys.iter().map(|key| key.key_id.as_str()).collect();
assert_eq!(listed, vec!["marker-c"], "a vanished marker must not restart the listing");
assert!(!response.truncated);
}
/// A zero limit means zero keys, and no cursor to loop on.
#[tokio::test]
async fn list_keys_with_zero_limit_returns_an_empty_page() {
let (client, _temp_dir) = create_test_client().await;
create_keys(&client, &["zero-limit-key".to_string()]).await;
let response = client
.list_keys(&page_request(0, None), None)
.await
.expect("a zero-limit list must succeed");
assert!(response.keys.is_empty());
assert!(!response.truncated);
assert!(response.next_marker.is_none());
}
/// A filter narrows a page after it has been cut, so a page can come back
/// empty while keys still remain. The traversal has to continue on
/// `truncated` rather than on a short page, and the cursor has to advance
/// across the keys the filter removed — otherwise a filtered listing ends
/// at the first run of non-matching keys and reports a partial key set as
/// the whole one.
#[tokio::test]
async fn filtered_paging_crosses_a_page_that_matches_nothing() {
let (client, _temp_dir) = create_test_client().await;
let key_ids: Vec<String> = (0..12).map(|index| format!("filter-key-{index:02}")).collect();
create_keys(&client, &key_ids).await;
// A page worth of adjacent keys is excluded, so one page of the
// traversal matches nothing at all.
for key_id in &key_ids[3..6] {
client.disable_key(key_id, None).await.expect("key should be disabled");
}
let still_active: Vec<String> = key_ids
.iter()
.enumerate()
.filter(|(index, _)| !(3..6).contains(index))
.map(|(_, key_id)| key_id.clone())
.collect();
let mut seen = Vec::new();
let mut pages_without_a_match = 0;
let mut marker: Option<String> = None;
// Bounded so a listing that cannot advance fails the assertions below
// instead of hanging the test run.
for _ in 0..key_ids.len() + 1 {
let response = client
.list_keys(&filtered_page_request(3, marker.as_deref(), KeyStatus::Active), None)
.await
.expect("list should succeed");
assert!(response.keys.len() <= 3, "a page must not exceed the requested limit");
assert!(
response.keys.iter().all(|key| key.status == KeyStatus::Active),
"a filtered page must carry matches only"
);
if response.keys.is_empty() {
pages_without_a_match += 1;
}
seen.extend(response.keys.iter().map(|key| key.key_id.clone()));
if !response.truncated {
assert!(response.next_marker.is_none(), "a final page must not offer a cursor");
break;
}
marker = Some(response.next_marker.expect("a truncated page must offer a cursor"));
}
assert_eq!(seen, still_active, "filtered paging must visit every match exactly once");
assert_eq!(pages_without_a_match, 1, "the excluded run must produce a page with no match");
// The keys the first filter excluded are exactly the ones the opposite
// filter lists, so nothing fell out of the key set on the way.
let response = client
.list_keys(&filtered_page_request(key_ids.len() as u32, None, KeyStatus::Disabled), None)
.await
.expect("list should succeed");
let listed: Vec<&str> = response.keys.iter().map(|key| key.key_id.as_str()).collect();
assert_eq!(listed, key_ids[3..6].iter().map(String::as_str).collect::<Vec<_>>());
assert!(!response.truncated, "a page covering the whole key set is complete");
}
/// A limit past the end of the key set returns everything, once.
#[tokio::test]
async fn list_keys_limit_beyond_the_key_set_returns_one_complete_page() {
let (client, _temp_dir) = create_test_client().await;
let key_ids: Vec<String> = (0..3).map(|index| format!("huge-limit-key-{index}")).collect();
create_keys(&client, &key_ids).await;
let response = client
.list_keys(&page_request(u32::MAX, None), None)
.await
.expect("list should succeed");
assert_eq!(response.keys.len(), 3);
assert!(!response.truncated);
assert!(response.next_marker.is_none());
}
}
+348 -1
View File
@@ -126,6 +126,129 @@ pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item =
Ok(())
}
/// Reject a rewrap whose caller cannot reproduce the envelope's encryption
/// context.
///
/// Deliberately the same rule the decrypt paths apply — every context entry the
/// envelope carries must be matched, and an entirely empty request context is
/// accepted for envelopes written before contexts were recorded. Rewrap must
/// never be *stricter* than decrypt: a retirement sweep has to be able to
/// rewrap every envelope that still reads, or the old master key version it is
/// trying to retire stays referenced forever by objects that are perfectly
/// readable.
///
/// It must not be *laxer* either. The context binds an envelope to one
/// bucket/object pair, and a rewrap that skipped the check would let a caller
/// launder an envelope it has no claim on into a freshly wrapped one.
pub(crate) fn ensure_rewrap_context_matches(
envelope_context: &HashMap<String, String>,
request_context: &HashMap<String, String>,
) -> Result<()> {
for (key, expected_value) in envelope_context {
match request_context.get(key) {
Some(actual_value) if actual_value == expected_value => {}
Some(actual_value) => {
return Err(KmsError::context_mismatch(format!(
"Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
)));
}
None if request_context.is_empty() => {}
None => return Err(KmsError::context_mismatch(format!("Missing context key '{key}'"))),
}
}
Ok(())
}
/// Page size used when a [`ListKeysRequest`] does not ask for one.
pub(crate) const DEFAULT_LIST_KEYS_PAGE_SIZE: u32 = 100;
/// One page of a key set the backend has to slice itself.
pub(crate) struct KeyPage<'a, T> {
/// The identifiers this page covers, in listing order.
pub(crate) items: &'a [T],
/// Where the next page resumes; `None` when this page is the last one.
pub(crate) next_marker: Option<String>,
/// Whether keys remain beyond this page.
pub(crate) truncated: bool,
}
/// Cut the page `request` asks for out of `sorted`.
///
/// `sorted` must be ordered by the identifier `key_id_of` returns, and the
/// marker is an *exclusive lower bound* on that identifier rather than an index
/// into the sequence. That is what makes paging survive concurrent mutation:
/// the next page resumes at the first identifier greater than the marker, so a
/// key added or removed elsewhere in the ordering — including the marker key
/// itself, which the deletion sweep routinely destroys — cannot make the
/// listing skip keys or restart from the beginning.
///
/// A `limit` of zero is honoured as written: the caller asked for no keys and
/// gets an empty, non-truncated page (see [`list_keys_page_size`]).
///
/// Filters are applied by the caller to `items` after this slice, so a filtered
/// page can be shorter than `limit` — and even empty — while more keys remain.
/// Callers must page until `truncated` is false rather than until a page comes
/// back short.
pub(crate) fn paginate_keys<'a, T>(sorted: &'a [T], request: &ListKeysRequest, key_id_of: impl Fn(&T) -> &str) -> KeyPage<'a, T> {
let Some(limit) = list_keys_page_size(request.limit) else {
return KeyPage {
items: &[],
next_marker: None,
truncated: false,
};
};
let start = match request.marker.as_deref() {
Some(marker) => sorted.partition_point(|item| key_id_of(item) <= marker),
None => 0,
};
// `partition_point` never exceeds the length, so both bounds stay in range
// however large `limit` is.
let end = start.saturating_add(limit).min(sorted.len());
let items = &sorted[start..end];
let truncated = end < sorted.len();
KeyPage {
items,
// Resuming from the last identifier on the page, not from an index,
// keeps the cursor meaningful after the key it names disappears.
next_marker: if truncated {
items.last().map(|item| key_id_of(item).to_string())
} else {
None
},
truncated,
}
}
/// Resolve the page size of a [`ListKeysRequest`]; `None` means the caller
/// asked for no keys at all.
///
/// `Some(0)` is a well-formed request for an empty page — the reading rustfs
/// already gives `max-keys=0` on the S3 listing path — not a malformed one and
/// not an omitted value. Rounding it up to a default would hand back a full
/// page of keys to a caller that explicitly asked for none.
pub(crate) fn list_keys_page_size(limit: Option<u32>) -> Option<usize> {
match limit.unwrap_or(DEFAULT_LIST_KEYS_PAGE_SIZE) {
0 => None,
size => Some(size as usize),
}
}
/// The response to a request for zero keys.
///
/// `truncated` is false even when keys exist: a zero-length page carries no
/// identifier to resume from, so claiming more results would hand back a cursor
/// the caller can never advance and turn a `while truncated` loop into a
/// non-terminating one.
pub(crate) fn empty_key_page() -> ListKeysResponse {
ListKeysResponse {
keys: Vec::new(),
next_marker: None,
truncated: false,
}
}
/// Simplified KMS backend interface for manager
#[async_trait]
pub trait KmsBackend: Send + Sync {
@@ -144,7 +267,18 @@ pub trait KmsBackend: Send + Sync {
/// Describe a key
async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse>;
/// List keys
/// List keys.
///
/// `status_filter` and `usage_filter` narrow a page after it has been cut,
/// so a filtered page can be shorter than the requested limit — and even
/// empty — while keys remain: a caller must page until `truncated` is false
/// rather than until a page comes back short. Every backend applies both
/// filters; a backend that cannot answer a filter must fail rather than
/// return the unfiltered set.
///
/// Backends that slice their own key set do so with [`paginate_keys`],
/// which fixes the ordering and the marker semantics; a backend paging
/// through a remote API passes that API's own cursor through instead.
async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse>;
/// Delete a key
@@ -181,6 +315,75 @@ pub trait KmsBackend: Send + Sync {
Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
}
/// Re-wrap an existing data key envelope under the key's current version,
/// returning an envelope that protects the same data key.
///
/// This is the primitive that makes retiring an old master key version
/// possible at all: until every envelope a version wrapped has been moved
/// onto the current version, destroying that version's material orphans
/// every object whose data key it wrapped. Rotation alone does not shrink
/// the blast radius of a leaked master key version, because envelopes
/// written before it stay decryptable under the leaked material forever.
///
/// Contract for implementations:
///
/// - The plaintext data key must not be persisted, logged, or returned. It
/// is either never materialized at all (backends with a native rewrap) or
/// held only for the length of the re-wrap.
/// - The destination version is always the key's *current* version. Wrapping
/// to any older version would create objects that nodes which resolve
/// envelopes to the current version cannot read.
/// - Everything except the wrapping is carried over verbatim, encryption
/// context included, so the result is the same data key rewrapped and not
/// a new one. An envelope must never be moved between objects.
/// - Idempotence: an envelope already wrapped by the current version comes
/// back byte for byte with [`RewrapDataKeyResponse::rewrapped`] false, so
/// re-running a sweep converges and performs no metadata writes.
///
/// Only backends that advertise [`BackendCapabilities::rewrap`] may override
/// this method; the default rejects the operation. A backend without
/// retained version history has nothing to rewrap *to*, so it reports the
/// capability gap rather than performing a re-wrap that changes no version
/// and buys no security.
async fn rewrap_data_key(&self, _request: RewrapDataKeyRequest) -> Result<RewrapDataKeyResponse> {
Err(KmsError::unsupported_capability(
"backend without versioned key material to rewrap onto",
"rewrap_data_key",
))
}
/// Report which master key version wraps an existing data key envelope, and
/// whether that is the key's current version.
///
/// The read-only counterpart of [`Self::rewrap_data_key`], and the only
/// supported way to ask the question: each rotating backend records the
/// version somewhere else, so every caller that parsed envelopes itself
/// would carry its own copy of that knowledge outside the KMS boundary.
///
/// [`DescribeDataKeyWrappingResponse::is_current`] must be the exact
/// negation of what [`RewrapDataKeyResponse::rewrapped`] would report for
/// the same envelope. The two answers drive a single loop — scan to find
/// work, rewrap to do it, scan again to prove it is done — and a
/// disagreement would make that loop either never terminate or declare
/// success while envelopes still reference a version somebody is about to
/// destroy.
///
/// Deliberately not gated on key state: an inventory has to be able to
/// count envelopes under keys that are disabled or already scheduled for
/// deletion, which are precisely the keys whose retirement is in question.
///
/// Gated by the same [`BackendCapabilities::rewrap`] flag, because a backend
/// that cannot rewrap has no version to report progress against.
async fn describe_data_key_wrapping(
&self,
_request: DescribeDataKeyWrappingRequest,
) -> Result<DescribeDataKeyWrappingResponse> {
Err(KmsError::unsupported_capability(
"backend without versioned key material to rewrap onto",
"describe_data_key_wrapping",
))
}
/// Replace a key's free-form description; `None` clears it.
///
/// Backends that advertise [`BackendCapabilities::update_key_metadata`]
@@ -297,6 +500,8 @@ pub struct BackendCapabilities {
pub physical_delete: bool,
/// Updating a key's description and tags after creation
pub update_key_metadata: bool,
/// Re-wrapping an existing data key envelope onto the key's current version
pub rewrap: bool,
}
impl BackendCapabilities {
@@ -314,6 +519,7 @@ impl BackendCapabilities {
versioning: false,
physical_delete: false,
update_key_metadata: false,
rewrap: false,
}
}
@@ -370,6 +576,12 @@ impl BackendCapabilities {
self.update_key_metadata = update_key_metadata;
self
}
/// Set whether envelope rewrap onto the current key version is supported
pub const fn with_rewrap(mut self, rewrap: bool) -> Self {
self.rewrap = rewrap;
self
}
}
impl Default for BackendCapabilities {
@@ -449,6 +661,7 @@ mod tests {
assert!(!capabilities.versioning);
assert!(!capabilities.physical_delete);
assert!(!capabilities.update_key_metadata);
assert!(!capabilities.rewrap);
}
#[tokio::test]
@@ -463,6 +676,26 @@ mod tests {
),
("tag_key", MinimalBackend.tag_key("any-key", &HashMap::new()).await),
("untag_key", MinimalBackend.untag_key("any-key", &[]).await),
(
"rewrap_data_key",
MinimalBackend
.rewrap_data_key(RewrapDataKeyRequest {
ciphertext: b"{}".to_vec(),
encryption_context: HashMap::new(),
})
.await
.map(|_| ()),
),
(
"describe_data_key_wrapping",
MinimalBackend
.describe_data_key_wrapping(DescribeDataKeyWrappingRequest {
ciphertext: b"{}".to_vec(),
encryption_context: HashMap::new(),
})
.await
.map(|_| ()),
),
] {
let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
assert!(
@@ -484,6 +717,40 @@ mod tests {
);
}
/// The rewrap context guard has to sit exactly where the decrypt guard
/// sits: strict enough that an envelope cannot be laundered under a
/// different context, lax enough that everything decrypt accepts can still
/// be rewrapped — otherwise a retirement sweep stalls on readable objects.
#[test]
fn rewrap_context_guard_matches_the_decrypt_rule() {
let envelope = HashMap::from([
("bucket".to_string(), "b/o".to_string()),
("tenant".to_string(), "acme".to_string()),
]);
ensure_rewrap_context_matches(&envelope, &envelope).expect("an exact context must be accepted");
// An empty request context is the pre-context-recording compatibility
// case decrypt allows; rewrap must not be stricter.
ensure_rewrap_context_matches(&envelope, &HashMap::new()).expect("an empty request context must stay accepted");
// Extra entries the envelope does not carry are ignored, as on decrypt.
let mut superset = envelope.clone();
superset.insert("extra".to_string(), "ignored".to_string());
ensure_rewrap_context_matches(&envelope, &superset).expect("extra request context entries must be ignored");
let mut tampered = envelope.clone();
tampered.insert("bucket".to_string(), "other/o".to_string());
assert!(
matches!(ensure_rewrap_context_matches(&envelope, &tampered), Err(KmsError::ContextMismatch { .. })),
"a tampered context value must be rejected"
);
let partial = HashMap::from([("tenant".to_string(), "acme".to_string())]);
assert!(
matches!(ensure_rewrap_context_matches(&envelope, &partial), Err(KmsError::ContextMismatch { .. })),
"a non-empty context missing an envelope entry must be rejected"
);
}
#[test]
fn identity_tag_is_rejected_by_metadata_updates() {
let error = ensure_tag_keys_are_mutable([RESERVED_KEY_NAME_TAG])
@@ -546,4 +813,84 @@ mod tests {
insta::assert_json_snapshot!("static_backend_capabilities", capabilities_snapshot(backend.capabilities()));
}
// -- Pagination boundaries ----------------------------------------------
fn key_ids(count: usize) -> Vec<String> {
(0..count).map(|index| format!("key-{index:02}")).collect()
}
fn page_request(limit: Option<u32>, marker: Option<&str>) -> ListKeysRequest {
ListKeysRequest {
limit,
marker: marker.map(str::to_string),
usage_filter: None,
status_filter: None,
}
}
fn page_of(keys: &[String], limit: Option<u32>, marker: Option<&str>) -> (Vec<String>, Option<String>, bool) {
let page = paginate_keys(keys, &page_request(limit, marker), String::as_str);
(page.items.to_vec(), page.next_marker, page.truncated)
}
/// Zero keys requested, zero keys returned — and no cursor, so a caller
/// looping on `truncated` terminates instead of asking forever. Slicing a
/// zero-length page out of a non-empty key set must not reach for the
/// element before the page either.
#[test]
fn zero_limit_returns_an_empty_untruncated_page() {
let keys = key_ids(3);
assert_eq!(page_of(&keys, Some(0), None), (Vec::new(), None, false));
assert_eq!(page_of(&keys, Some(0), Some("key-01")), (Vec::new(), None, false));
// Also at the ends of the key set, where a page has no predecessor.
assert_eq!(page_of(&[], Some(0), None), (Vec::new(), None, false));
assert_eq!(page_of(&keys, Some(0), Some("key-02")), (Vec::new(), None, false));
assert_eq!(list_keys_page_size(Some(0)), None);
assert_eq!(list_keys_page_size(None), Some(DEFAULT_LIST_KEYS_PAGE_SIZE as usize));
assert_eq!(list_keys_page_size(Some(7)), Some(7));
}
/// A limit past the end of the key set is not an overflow.
#[test]
fn oversized_limit_returns_the_whole_key_set_once() {
let keys = key_ids(3);
assert_eq!(page_of(&keys, Some(u32::MAX), None), (keys.clone(), None, false));
assert_eq!(page_of(&keys, Some(u32::MAX), Some("key-01")), (vec![keys[2].clone()], None, false));
}
/// The cursor is an identifier, so a marker naming a key that no longer
/// exists resumes after where it would have been instead of restarting.
#[test]
fn marker_for_a_removed_key_resumes_after_it() {
let keys = vec!["key-00".to_string(), "key-02".to_string()];
assert_eq!(page_of(&keys, Some(10), Some("key-01")), (vec!["key-02".to_string()], None, false));
// A marker past every key ends the listing rather than wrapping.
assert_eq!(page_of(&keys, Some(10), Some("key-99")), (Vec::new(), None, false));
// A marker before every key yields the whole set.
assert_eq!(page_of(&keys, Some(10), Some("key")), (keys.clone(), None, false));
}
/// Truncation flips exactly at the page boundary, and paging covers the
/// key set once end to end.
#[test]
fn pages_tile_the_key_set_exactly_at_the_limit_boundary() {
let keys = key_ids(4);
assert_eq!(page_of(&keys, Some(4), None), (keys.clone(), None, false));
assert_eq!(page_of(&keys, Some(3), None), (keys[..3].to_vec(), Some("key-02".to_string()), true));
let mut seen = Vec::new();
let mut marker = None;
loop {
let (items, next_marker, truncated) = page_of(&keys, Some(2), marker.as_deref());
seen.extend(items);
if !truncated {
assert!(next_marker.is_none(), "a final page must not offer a cursor");
break;
}
marker = Some(next_marker.expect("a truncated page must offer a cursor"));
}
assert_eq!(seen, keys, "paging must tile the key set exactly once");
}
}
@@ -8,6 +8,7 @@ expression: capabilities_snapshot(backend.capabilities())
"encrypt": true,
"generate_data_key": true,
"physical_delete": false,
"rewrap": false,
"rotate": true,
"schedule_deletion": true,
"update_key_metadata": false,
@@ -8,6 +8,7 @@ expression: capabilities_snapshot(backend.capabilities())
"encrypt": true,
"generate_data_key": true,
"physical_delete": true,
"rewrap": false,
"rotate": false,
"schedule_deletion": true,
"update_key_metadata": true,
@@ -8,6 +8,7 @@ expression: capabilities_snapshot(backend.capabilities())
"encrypt": true,
"generate_data_key": true,
"physical_delete": false,
"rewrap": false,
"rotate": false,
"schedule_deletion": false,
"update_key_metadata": false,
@@ -8,6 +8,7 @@ expression: capabilities_snapshot(backend.capabilities())
"encrypt": true,
"generate_data_key": true,
"physical_delete": true,
"rewrap": true,
"rotate": true,
"schedule_deletion": true,
"update_key_metadata": true,
@@ -8,6 +8,7 @@ expression: capabilities_snapshot(backend.capabilities())
"encrypt": true,
"generate_data_key": true,
"physical_delete": true,
"rewrap": true,
"rotate": true,
"schedule_deletion": true,
"update_key_metadata": true,
+82 -9
View File
@@ -19,9 +19,12 @@
//!
//! ## Ciphertext format
//!
//! encrypted_data(plaintext_len+16) || nonce (12 bytes)
//! A JSON-serialized `DataKeyEnvelope` carrying the AES-256-GCM ciphertext, the
//! 12-byte nonce and the authenticated encryption context. This is a
//! RustFS-internal format; it is not interchangeable with MinIO's KMS
//! ciphertext (see the note on `StaticConfig`).
use crate::backends::{BackendCapabilities, KmsBackend};
use crate::backends::{BackendCapabilities, KmsBackend, empty_key_page, list_keys_page_size};
use crate::config::{BackendConfig, KmsConfig};
use crate::encryption::DataKeyEnvelope;
use crate::error::{KmsError, Result};
@@ -260,8 +263,15 @@ impl StaticKmsBackend {
})
}
/// List the single configured key, honouring the pagination marker.
/// List the single configured key, honouring the pagination marker and the
/// status and usage filters.
pub(crate) fn list_configured_key(&self, request: &ListKeysRequest) -> Result<ListKeysResponse> {
// A caller asking for no keys gets none, even from a backend whose
// whole key set is one key.
if list_keys_page_size(request.limit).is_none() {
return Ok(empty_key_page());
}
let key_info = KeyInfo {
key_id: self.key_id.clone(),
description: Some("Static single-key KMS backend".to_string()),
@@ -276,15 +286,25 @@ impl StaticKmsBackend {
created_by: None,
};
// Apply prefix filter if provided
// The marker is an exclusive lower bound on the identifier, as it is
// for every other backend.
if let Some(ref marker) = request.marker
&& self.key_id <= *marker
{
return Ok(ListKeysResponse {
keys: vec![],
next_marker: None,
truncated: false,
});
return Ok(empty_key_page());
}
// The configured key is filtered like any other: a caller narrowing the
// listing to disabled or signing keys must get an empty page rather
// than this active encryption key, which it would otherwise have to
// recognise as a non-match on its own.
if request
.status_filter
.as_ref()
.is_some_and(|status| status != &key_info.status)
|| request.usage_filter.as_ref().is_some_and(|usage| usage != &key_info.usage)
{
return Ok(empty_key_page());
}
Ok(ListKeysResponse {
@@ -657,6 +677,59 @@ mod tests {
assert!(!response.truncated);
}
/// A zero limit means zero keys, even for a backend whose whole key set is
/// a single configured key.
#[tokio::test]
async fn zero_limit_list_returns_an_empty_page() {
let (backend, _key_id, _key) = create_test_backend().await;
let response = backend
.list_configured_key(&ListKeysRequest {
limit: Some(0),
..Default::default()
})
.expect("a zero-limit list must succeed");
assert!(response.keys.is_empty());
assert!(!response.truncated);
assert!(response.next_marker.is_none());
}
/// A filter that excludes the configured key must empty the page. Handing
/// the key back regardless would answer "list the disabled keys" with an
/// active one, and the response says nothing about the filter having been
/// dropped.
#[tokio::test]
async fn a_filter_the_configured_key_does_not_match_empties_the_page() {
let (backend, key_id, _key) = create_test_backend().await;
for request in [
ListKeysRequest {
status_filter: Some(KeyStatus::Disabled),
..Default::default()
},
ListKeysRequest {
usage_filter: Some(KeyUsage::SignVerify),
..Default::default()
},
] {
let response = backend.list_configured_key(&request).expect("a filtered list must succeed");
assert!(response.keys.is_empty(), "excluded key was listed for {request:?}");
assert!(!response.truncated);
assert!(response.next_marker.is_none());
}
// The filters the key does match still list it.
let response = backend
.list_configured_key(&ListKeysRequest {
status_filter: Some(KeyStatus::Active),
usage_filter: Some(KeyUsage::EncryptDecrypt),
..Default::default()
})
.expect("a matching list must succeed");
assert_eq!(response.keys.len(), 1);
assert_eq!(response.keys[0].key_id, key_id);
}
#[tokio::test]
async fn lifecycle_mutations_are_unsupported_at_the_product_surface() {
let (backend, key_id, _key) = create_test_backend().await;
+837 -32
View File
@@ -19,8 +19,8 @@ use crate::backends::vault_credentials::{
token_source_for,
};
use crate::backends::{
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
ensure_tag_keys_are_mutable,
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, empty_key_page, ensure_key_state_permits,
ensure_key_status_permits, ensure_rewrap_context_matches, ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys,
};
use crate::config::{KmsConfig, VaultConfig};
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -38,6 +38,7 @@ use std::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use vaultrs::{api::kv2::requests::SetSecretRequestOptions, error::ClientError, kv2};
use zeroize::Zeroize as _;
/// Vault KMS client implementation
pub struct VaultKmsClient {
@@ -80,6 +81,19 @@ struct VaultKeyData {
/// persistence landed, so it must stay optional for backward compatibility.
#[serde(default)]
deletion_date: Option<Zoned>,
/// When the key's material last became current through a rotation.
///
/// Written by [`VaultKmsClient::rotate_key`] as part of the same
/// check-and-set that switches the current version, so it can only be set on
/// a rotation that actually committed. `None` means "no rotation time on
/// record", which covers two cases that are deliberately not distinguished
/// here: a key that was never rotated, and a key rotated by a build that
/// predates this field. Nothing is back-filled — inventing a timestamp for
/// the second case would report a rotation that this node never observed.
/// See [`crate::deletion_worker`] for why collapsing the two is safe for the
/// rotation-age gauge.
#[serde(default)]
rotated_at: Option<Zoned>,
/// Encrypted key material (base64 encoded)
encrypted_key_material: String,
/// Version that pre-versioning envelopes (no `master_key_version`) resolve to.
@@ -880,6 +894,137 @@ impl VaultKmsClient {
Ok(plaintext)
}
/// Report which master key version wraps an envelope, and whether that is
/// the key's current version.
///
/// Reads the key record only; it never unwraps anything, so it works on keys
/// whose state forbids new cryptographic use — which is exactly the
/// population a retirement inventory has to cover.
pub(crate) async fn describe_data_key_wrapping(
&self,
request: &DescribeDataKeyWrappingRequest,
) -> Result<DescribeDataKeyWrappingResponse> {
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
.map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?;
ensure_rewrap_context_matches(&envelope.encryption_context, &request.encryption_context)?;
let key_data = self.get_key_data(&envelope.master_key_id).await?;
let current_version = key_data.version;
Ok(DescribeDataKeyWrappingResponse {
key_id: envelope.master_key_id,
key_version: Some(resolve_envelope_master_key_version(
envelope.master_key_version,
key_data.baseline_version,
current_version,
)),
current_key_version: Some(current_version),
// Deliberately not `key_version == current_version`. A
// pre-versioning envelope resolves to the current version while
// saying nothing, and `rewrap_data_key` rewrites exactly those to
// stamp the version — so reporting them as current here would leave
// the sweep and the scan permanently disagreeing.
is_current: envelope.master_key_version == Some(current_version),
})
}
/// Re-wrap an existing envelope with the key's current master key version.
///
/// The plaintext data key is unwrapped with the version that actually
/// wrapped it and immediately re-wrapped with the current material. It is
/// zeroized before this returns, never persisted, never logged and never
/// handed to the caller: the whole point of rewrap is that the data key is
/// re-protected without anyone above this layer holding it.
///
/// Everything except the wrapping is carried over verbatim — the data key's
/// own id, its spec, its encryption context and its creation time — so the
/// result is the same data key under new wrapping. That keeps the DEK's
/// recorded age honest and makes the operation invisible to every consumer
/// of the envelope other than the version stamp.
///
/// A pre-versioning envelope (no `master_key_version`) is rewrapped even
/// when it resolves to the current version and its bytes would be unwrapped
/// with the very material they are about to be re-wrapped with. That is not
/// wasted work: the version stamp is the only evidence a retirement scan can
/// read, and an envelope that does not state its version can never be
/// counted as migrated.
pub(crate) async fn rewrap_data_key(&self, request: &RewrapDataKeyRequest) -> Result<RewrapDataKeyResponse> {
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
.map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?;
ensure_rewrap_context_matches(&envelope.encryption_context, &request.encryption_context)?;
// Single read of the key record: the version that unwraps, the material
// that re-wraps and the version stamped into the result must all come
// from one snapshot. Reading them separately would let a concurrent
// rotation produce an envelope stamped with a version whose material
// never wrapped it — an envelope that then fails to decrypt forever.
let key_data = self.get_key_data(&envelope.master_key_id).await?;
ensure_key_status_permits(&envelope.master_key_id, &key_data.status, StateGatedOperation::Encrypt)?;
let current_version = key_data.version;
if envelope.master_key_version == Some(current_version) {
// Already on the current version and saying so. Hand the input back
// untouched rather than producing an equivalent envelope with a
// fresh nonce: a re-run of a sweep must converge to zero writes, and
// the storage layer keys its write decision off these bytes.
return Ok(RewrapDataKeyResponse {
ciphertext: request.ciphertext.clone(),
key_id: envelope.master_key_id,
source_key_version: Some(current_version),
destination_key_version: Some(current_version),
rewrapped: false,
});
}
let source_version =
resolve_envelope_master_key_version(envelope.master_key_version, key_data.baseline_version, current_version);
// Both materials are resolved before anything is unwrapped, so no
// fallible step sits between the plaintext data key coming into
// existence and the zeroize that removes it again.
let source_material = self
.get_key_material_for_version(&envelope.master_key_id, &key_data, source_version)
.await?;
let destination_material = decode_stored_key_material(&envelope.master_key_id, &key_data.encrypted_key_material)
.inspect_err(|error| warn!(key_id = %envelope.master_key_id, %error, "Vault KMS key material failed validation"))?;
let mut plaintext_key = match self
.dek_crypto
.decrypt(&source_material, &envelope.encrypted_key, &envelope.nonce)
.await
{
Ok(plaintext) => plaintext,
Err(error) => {
return Err(self
.explain_unwrap_failure(&envelope.master_key_id, &key_data, envelope.master_key_version, error)
.await);
}
};
let rewrapped = self.dek_crypto.encrypt(&destination_material, &plaintext_key).await;
plaintext_key.zeroize();
let (encrypted_key, nonce) = rewrapped?;
let rewrapped_envelope = DataKeyEnvelope {
key_id: envelope.key_id,
master_key_id: envelope.master_key_id,
key_spec: envelope.key_spec,
encrypted_key,
nonce,
encryption_context: envelope.encryption_context,
created_at: envelope.created_at,
master_key_version: Some(current_version),
};
let ciphertext = serde_json::to_vec(&rewrapped_envelope)?;
debug!(key_id = %rewrapped_envelope.master_key_id, source_version, current_version, "Vault KMS data key rewrapped");
Ok(RewrapDataKeyResponse {
ciphertext,
key_id: rewrapped_envelope.master_key_id,
source_key_version: Some(source_version),
destination_key_version: Some(current_version),
rewrapped: true,
})
}
/// Re-report a failure to unwrap a data key as the lost-baseline diagnosis
/// when that is what the key record shows.
///
@@ -966,7 +1111,7 @@ impl VaultKmsClient {
description: existing.description,
metadata: existing.metadata,
created_at: existing.created_at,
rotated_at: None,
rotated_at: existing.rotated_at,
created_by: None,
deletion_date: existing.deletion_date,
})
@@ -993,6 +1138,7 @@ impl VaultKmsClient {
metadata: HashMap::new(),
tags: HashMap::new(),
deletion_date: None,
rotated_at: None,
encrypted_key_material: encrypted_material,
baseline_version: None,
};
@@ -1039,7 +1185,7 @@ impl VaultKmsClient {
metadata: key_data.metadata,
tags: key_data.tags,
created_at: key_data.created_at,
rotated_at: None,
rotated_at: key_data.rotated_at,
created_by: None,
})
}
@@ -1051,37 +1197,42 @@ impl VaultKmsClient {
) -> Result<ListKeysResponse> {
debug!("Listing keys with limit: {:?}", request.limit);
let all_keys = self.list_vault_keys().await?;
let limit = request.limit.unwrap_or(100) as usize;
// Simple pagination implementation
let start_idx = request
.marker
.as_ref()
.and_then(|m| all_keys.iter().position(|k| k == m))
.map(|idx| idx + 1)
.unwrap_or(0);
let end_idx = std::cmp::min(start_idx + limit, all_keys.len());
let keys_page = &all_keys[start_idx..end_idx];
let mut key_infos = Vec::new();
for key_id in keys_page {
if let Ok(key_info) = self.describe_key(key_id, None).await {
key_infos.push(key_info);
}
// A caller asking for no keys is answered without reaching Vault.
if list_keys_page_size(request.limit).is_none() {
return Ok(empty_key_page());
}
let next_marker = if end_idx < all_keys.len() {
Some(all_keys[end_idx - 1].clone())
} else {
None
};
let mut all_keys = self.list_vault_keys().await?;
// Vault's own LIST ordering is not part of its contract, so the sort is
// what makes the marker a stable cursor across calls.
all_keys.sort_unstable();
let page = paginate_keys(&all_keys, request, String::as_str);
let mut key_infos = Vec::with_capacity(page.items.len());
for key_id in page.items {
// A key that disappeared between the listing and the read is
// dropped from the page rather than failing it; the cursor comes
// from the identifier list, so the listing still advances past it.
let Ok(key_info) = self.describe_key(key_id, None).await else {
continue;
};
if request
.status_filter
.as_ref()
.is_some_and(|status| status != &key_info.status)
{
continue;
}
if request.usage_filter.as_ref().is_some_and(|usage| usage != &key_info.usage) {
continue;
}
key_infos.push(key_info);
}
Ok(ListKeysResponse {
keys: key_infos,
next_marker,
truncated: end_idx < all_keys.len(),
next_marker: page.next_marker,
truncated: page.truncated,
})
}
@@ -1263,8 +1414,15 @@ impl VaultKmsClient {
// Step 3: switch the current pointer. The top-level copy of the material is
// the fast path for new encryptions and must always match `version`.
//
// The rotation timestamp rides along on this same write: it marks the
// moment the new material became current, and persisting it here means it
// commits if and only if the rotation does. A rotation that fails after
// freezing the version record leaves the key unrotated and unstamped, so
// the recorded time never runs ahead of the current version.
key_data.version = new_version;
key_data.encrypted_key_material = new_material;
key_data.rotated_at = Some(Zoned::now());
self.cas_store_key_data(key_id, &key_data, cas).await?;
info!(key_id, version = new_version, "Vault KMS master key rotated");
@@ -1278,7 +1436,9 @@ impl VaultKmsClient {
description: key_data.description.clone(),
metadata: key_data.metadata.clone(),
created_at: key_data.created_at.clone(),
rotated_at: Some(Zoned::now()),
// The persisted value, not a fresh `now()`: what the caller is told
// must be what a later describe of the same key reports.
rotated_at: key_data.rotated_at.clone(),
created_by: None,
deletion_date: key_data.deletion_date.clone(),
})
@@ -1421,6 +1581,17 @@ impl KmsBackend for VaultKmsBackend {
})
}
async fn rewrap_data_key(&self, request: RewrapDataKeyRequest) -> Result<RewrapDataKeyResponse> {
self.client.rewrap_data_key(&request).await
}
async fn describe_data_key_wrapping(
&self,
request: DescribeDataKeyWrappingRequest,
) -> Result<DescribeDataKeyWrappingResponse> {
self.client.describe_data_key_wrapping(&request).await
}
async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
let generate_request = GenerateKeyRequest {
master_key_id: request.key_id.clone(),
@@ -1630,7 +1801,9 @@ impl KmsBackend for VaultKmsBackend {
// Rotation freezes the outgoing material as an immutable version
// record before switching the current pointer, and envelopes resolve
// their wrapping version on decrypt, so every historical version
// stays decryptable after a rotation.
// stays decryptable after a rotation. Those same immutable records are
// what lets an envelope be unwrapped with the version that wrapped it
// and re-wrapped onto the current one.
BackendCapabilities::minimal()
.with_rotate(true)
.with_enable_disable(true)
@@ -1638,6 +1811,7 @@ impl KmsBackend for VaultKmsBackend {
.with_versioning(true)
.with_physical_delete(true)
.with_update_key_metadata(true)
.with_rewrap(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -1732,6 +1906,7 @@ mod tests {
metadata: HashMap::new(),
tags: HashMap::new(),
deletion_date: None,
rotated_at: None,
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
baseline_version: None,
}
@@ -1751,6 +1926,35 @@ mod tests {
})
}
/// A caller asking for no keys gets an empty page, and the page arithmetic
/// never reaches for the element before an empty page. The scripted key
/// listing stays unused: a request for zero keys has nothing to ask Vault.
#[tokio::test]
async fn zero_limit_list_returns_an_empty_page_without_calling_vault() {
let (vault, client) =
scripted_client(vec![ScriptedResponse::ok(serde_json::json!({ "keys": ["key-a", "key-b"] }))]).await;
let response = client
.list_keys(
&ListKeysRequest {
limit: Some(0),
..Default::default()
},
None,
)
.await
.expect("a zero-limit list must succeed");
assert!(response.keys.is_empty());
assert!(!response.truncated);
assert!(response.next_marker.is_none());
assert!(
vault.requests().is_empty(),
"a request for no keys must not reach Vault: {:?}",
vault.requests()
);
}
#[tokio::test]
async fn wired_read_retries_transient_status_then_succeeds() {
let (vault, client) = scripted_client(vec![
@@ -2048,6 +2252,7 @@ mod tests {
encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
baseline_version: Some(1),
deletion_date: None,
rotated_at: None,
};
let mut value = serde_json::to_value(&key_data).expect("serialize key data");
@@ -2411,6 +2616,7 @@ mod tests {
metadata: HashMap::new(),
tags: HashMap::new(),
deletion_date: Some(deadline.clone()),
rotated_at: None,
encrypted_key_material: "material".to_string(),
baseline_version: None,
};
@@ -3416,6 +3622,141 @@ mod tests {
);
}
/// The rotation-age gauge ages a key from `rotated_at`, falling back to
/// `created_at`. A rotation that commits without recording its time makes a
/// key rotated many times read exactly like one that was never rotated, so
/// the commit must carry the timestamp and a later describe must report it.
/// Reverting either half turns this test red.
#[tokio::test]
async fn wired_rotate_persists_rotation_time_and_describe_reports_it() {
let created_at = Zoned::now() - Duration::from_secs(365 * 86400);
let mut key_data = healthy_key_data();
key_data.created_at = created_at.clone();
key_data.version = 2;
key_data.baseline_version = Some(1);
key_data.description = Some("payload key".to_string());
key_data.metadata.insert("owner".to_string(), "platform".to_string());
key_data.tags.insert("env".to_string(), "prod".to_string());
let (vault, client) = scripted_client(vec![
ScriptedResponse::ok(kv2_metadata_read_data(3)),
ScriptedResponse::ok(kv2_read_data(&key_data)),
ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })),
// Version 3's material record, then the pointer switch.
ScriptedResponse::ok(kv2_write_ack()),
ScriptedResponse::ok(kv2_write_ack()),
])
.await;
let rotated = client.rotate_key("wired-key", None).await.expect("rotate a healthy key");
let reported = rotated.rotated_at.clone().expect("a committed rotation must report its time");
let bodies = vault.request_bodies();
let committed = parse_write_body(&bodies[4]);
let persisted: VaultKeyData =
serde_json::from_value(committed["data"].clone()).expect("the committed record must deserialize");
assert_eq!(
persisted.rotated_at.as_ref().map(Zoned::timestamp),
Some(reported.timestamp()),
"the rotation must persist the time it reports: {committed}"
);
// The rest of the record rides through the read-modify-write untouched;
// a rotation that dropped any of it would corrupt the key.
assert_eq!(persisted.version, 3, "{committed}");
assert_eq!(persisted.created_at.timestamp(), created_at.timestamp(), "{committed}");
assert_eq!(persisted.baseline_version, Some(1), "{committed}");
assert_eq!(persisted.description.as_deref(), Some("payload key"), "{committed}");
assert_eq!(persisted.metadata.get("owner").map(String::as_str), Some("platform"), "{committed}");
assert_eq!(persisted.tags.get("env").map(String::as_str), Some("prod"), "{committed}");
// Describing the committed record must report the rotation, not the
// creation a year earlier that the gauge would otherwise fall back to.
let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&persisted))]).await;
let described = client
.describe_key("wired-key", None)
.await
.expect("describe the rotated key");
assert_eq!(
described.rotated_at.as_ref().map(Zoned::timestamp),
Some(reported.timestamp()),
"describe must report the persisted rotation time"
);
assert_ne!(
described.rotated_at.as_ref().map(Zoned::timestamp),
Some(created_at.timestamp()),
"a rotated key must not be aged from its creation"
);
}
/// A record written before rotation timestamps were persisted carries no
/// rotation time. Reporting one anyway — the current time, the read time —
/// would tell the rotation-age gauge the key was just rotated and silence a
/// genuinely overdue key, so the absence has to travel as `None`.
#[tokio::test]
async fn wired_describe_key_invents_no_rotation_time_for_legacy_records() {
let mut key_data = healthy_key_data();
key_data.created_at = Zoned::now() - Duration::from_secs(365 * 86400);
key_data.version = 4;
key_data.baseline_version = Some(1);
let mut record = kv2_read_data(&key_data);
record["data"]
.as_object_mut()
.expect("key record must be a JSON object")
.remove("rotated_at")
.expect("current records must carry the field");
let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(record)]).await;
let described = client
.describe_key("wired-key", None)
.await
.expect("a record without the field must still describe");
assert!(
described.rotated_at.is_none(),
"an unstamped record must not be reported as freshly rotated, got {:?}",
described.rotated_at
);
assert_eq!(
described.created_at.timestamp(),
key_data.created_at.timestamp(),
"the rest of the legacy record must survive the read"
);
assert_eq!(described.version, 4);
}
/// The persisted KV2 record round-trips its rotation time, and records
/// written before the field existed keep deserializing (as None) with the
/// rest of their contents intact.
#[test]
fn vault_key_data_rotated_at_round_trips_and_stays_backward_compatible() {
let rotated_at = Zoned::now();
let mut key_data = healthy_key_data();
key_data.rotated_at = Some(rotated_at.clone());
key_data.baseline_version = Some(1);
key_data.version = 2;
key_data.tags.insert("env".to_string(), "prod".to_string());
let mut value = serde_json::to_value(&key_data).expect("serialize");
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
assert_eq!(
restored.rotated_at.as_ref().map(Zoned::timestamp),
Some(rotated_at.timestamp()),
"the rotation time must survive the KV2 round trip"
);
value
.as_object_mut()
.expect("record must be a JSON object")
.remove("rotated_at")
.expect("current records must carry the field");
let legacy: VaultKeyData = serde_json::from_value(value).expect("legacy record must deserialize");
assert!(legacy.rotated_at.is_none());
assert_eq!(legacy.version, 2);
assert_eq!(legacy.baseline_version, Some(1));
assert_eq!(legacy.tags.get("env").map(String::as_str), Some("prod"));
}
/// Reading a pre-versioning envelope against a key whose baseline was erased
/// resolves to the current version, whose material never wrapped it. The
/// unwrap therefore fails (AES-GCM cannot yield plaintext under the wrong
@@ -3571,4 +3912,468 @@ mod tests {
"a successful read must not pay for the lost-baseline diagnosis"
);
}
/// The Vault-side state of one KV2 key: the top-level record plus the
/// immutable version records rotations froze.
///
/// The scripted responder serves canned responses, so a multi-operation
/// scenario has to carry the state between operations itself. Rotations
/// fold what they *wrote* back into this state (see [`Self::apply_writes`]),
/// which is what makes the rotation regressions below real: the material a
/// later decrypt resolves is the material the rotation persisted, not a
/// fixture the test invented.
struct KeyState {
key_data: VaultKeyData,
version_records: Vec<VaultKeyVersionRecord>,
}
impl KeyState {
/// A never-rotated key: no version records exist yet.
fn new(key_data: VaultKeyData) -> Self {
Self {
key_data,
version_records: Vec::new(),
}
}
fn version_record(&self, version: u32) -> &VaultKeyVersionRecord {
self.version_records
.iter()
.find(|record| record.version == version)
.unwrap_or_else(|| panic!("no version record was frozen for version {version}"))
}
/// The versions-directory listing; a key with no records has no
/// directory at all.
fn versions_listing(&self) -> ScriptedResponse {
if self.version_records.is_empty() {
return ScriptedResponse::error(404, "not found");
}
let keys: Vec<String> = self.version_records.iter().map(|record| record.version.to_string()).collect();
ScriptedResponse::ok(serde_json::json!({ "keys": keys }))
}
/// Fold the writes an operation made into the state, so the next
/// operation reads exactly what Vault would now hold.
fn apply_writes(&mut self, requests: &[String], bodies: &[String]) {
for (line, body) in requests.iter().zip(bodies) {
let Some(path) = line.strip_prefix("POST ") else {
continue;
};
let data = parse_write_body(body)["data"].clone();
if path.contains("/versions/") {
let record: VaultKeyVersionRecord = serde_json::from_value(data).expect("version record write body");
self.version_records.retain(|existing| existing.version != record.version);
self.version_records.push(record);
} else {
self.key_data = serde_json::from_value(data).expect("key record write body");
}
}
}
}
/// Encrypt against a scripted Vault serving `state`.
async fn encrypt_scripted(state: &KeyState, plaintext: &[u8]) -> EncryptResponse {
let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&state.key_data))]).await;
client
.encrypt(
&EncryptRequest {
key_id: "wired-key".to_string(),
plaintext: plaintext.to_vec(),
encryption_context: HashMap::new(),
grant_tokens: Vec::new(),
},
None,
)
.await
.expect("encrypt must produce an envelope")
}
/// Rotate against a scripted Vault seeded with `state`, and return the state
/// Vault holds afterwards.
///
/// The first rotation freezes the baseline before creating the next version
/// (four writes); later rotations skip that step (two writes).
async fn rotate_scripted(state: &KeyState) -> KeyState {
let writes = if state.key_data.baseline_version.is_none() { 4 } else { 2 };
let mut responses = vec![
ScriptedResponse::ok(kv2_metadata_read_data(1)),
ScriptedResponse::ok(kv2_read_data(&state.key_data)),
state.versions_listing(),
];
responses.extend((0..writes).map(|_| ScriptedResponse::ok(kv2_write_ack())));
let (vault, client) = scripted_client(responses).await;
let rotated = client.rotate_key("wired-key", None).await.expect("rotation must commit");
assert_eq!(rotated.version, state.key_data.version + 1, "a rotation must advance the version");
let mut next = KeyState {
key_data: state.key_data.clone(),
version_records: state.version_records.clone(),
};
next.apply_writes(&vault.requests(), &vault.request_bodies());
next
}
/// Decrypt against a scripted Vault serving `state`, scripting the
/// version-record read the envelope's own version calls for. Returns the
/// plaintext together with the requests the decrypt made.
async fn decrypt_scripted(state: &KeyState, ciphertext: &[u8]) -> (Vec<u8>, Vec<String>) {
let envelope: DataKeyEnvelope = serde_json::from_slice(ciphertext).expect("envelope must parse");
let mut responses = vec![ScriptedResponse::ok(kv2_read_data(&state.key_data))];
if let Some(version) = envelope.master_key_version
&& version != state.key_data.version
{
responses.push(ScriptedResponse::ok(kv2_read_version_record_data(state.version_record(version))));
}
let (vault, client) = scripted_client(responses).await;
let plaintext = client
.decrypt(
&DecryptRequest {
ciphertext: ciphertext.to_vec(),
encryption_context: HashMap::new(),
grant_tokens: Vec::new(),
},
None,
)
.await
.expect("the envelope must decrypt against the rotated key");
(plaintext, vault.requests())
}
/// The forward half of the rotation contract: data written before a rotation
/// stays readable after it, with no live Vault involved.
///
/// The negative half (a regressed pointer must fail closed) is covered by
/// `wired_decrypt_fails_closed_when_current_version_regressed`; this pins the
/// path that must keep working, which the fail-closed guards could otherwise
/// tighten into a rotation that orphans every existing object.
#[tokio::test]
async fn wired_kv2_envelope_from_before_rotation_still_decrypts() {
const PLAINTEXT: &[u8] = b"written-before-the-rotation";
let state_v1 = KeyState::new(healthy_key_data());
let encrypted_v1 = encrypt_scripted(&state_v1, PLAINTEXT).await;
let envelope_v1: DataKeyEnvelope = serde_json::from_slice(&encrypted_v1.ciphertext).expect("envelope must parse");
assert_eq!(envelope_v1.master_key_version, Some(1));
let state_v2 = rotate_scripted(&state_v1).await;
assert_eq!(state_v2.key_data.version, 2);
assert_eq!(state_v2.key_data.baseline_version, Some(1));
assert_ne!(
state_v2.key_data.encrypted_key_material, state_v1.key_data.encrypted_key_material,
"the rotation must have replaced the current material, or the decrypt below proves nothing"
);
let (plaintext, requests) = decrypt_scripted(&state_v2, &encrypted_v1.ciphertext).await;
assert_eq!(
plaintext, PLAINTEXT,
"the pre-rotation envelope must yield its original plaintext, not merely avoid an error"
);
assert_eq!(
requests,
vec![
"GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string(),
"GET /v1/secret/data/rustfs/kms/keys/wired-key/versions/1".to_string(),
],
"the old envelope must resolve through the immutable v1 record"
);
// The rotation is not cosmetic: new writes go to the rotated version and
// still round-trip, so both generations are live at once.
let encrypted_v2 = encrypt_scripted(&state_v2, b"written-after-the-rotation").await;
let envelope_v2: DataKeyEnvelope = serde_json::from_slice(&encrypted_v2.ciphertext).expect("envelope must parse");
assert_eq!(envelope_v2.master_key_version, Some(2), "new envelopes must carry the rotated version");
assert_eq!(encrypted_v2.key_version, 2);
let (plaintext_v2, requests_v2) = decrypt_scripted(&state_v2, &encrypted_v2.ciphertext).await;
assert_eq!(plaintext_v2, b"written-after-the-rotation".to_vec());
assert_eq!(
requests_v2.len(),
1,
"an envelope on the current version must not read a version record: {requests_v2:?}"
);
}
/// Old envelopes must survive more than one generation: the baseline is
/// frozen once and every intermediate version keeps its own record, so both
/// a pre-rotation envelope and one written between the two rotations still
/// decrypt after the second.
#[tokio::test]
async fn wired_kv2_envelopes_survive_consecutive_rotations() {
let state_v1 = KeyState::new(healthy_key_data());
let encrypted_v1 = encrypt_scripted(&state_v1, b"generation-1").await;
let state_v2 = rotate_scripted(&state_v1).await;
let encrypted_v2 = encrypt_scripted(&state_v2, b"generation-2").await;
assert_eq!(encrypted_v2.key_version, 2);
let state_v3 = rotate_scripted(&state_v2).await;
assert_eq!(state_v3.key_data.version, 3);
assert_eq!(
state_v3.key_data.baseline_version,
Some(1),
"the baseline is frozen once and carried through later rotations"
);
let mut recorded: Vec<u32> = state_v3.version_records.iter().map(|record| record.version).collect();
recorded.sort_unstable();
assert_eq!(recorded, vec![1, 2, 3], "every version that ever wrapped a DEK must keep a record");
for (ciphertext, expected, version) in [
(&encrypted_v1.ciphertext, b"generation-1".as_slice(), 1u32),
(&encrypted_v2.ciphertext, b"generation-2".as_slice(), 2),
] {
let (plaintext, requests) = decrypt_scripted(&state_v3, ciphertext).await;
assert_eq!(
plaintext, expected,
"an envelope from version {version} must survive two rotations intact"
);
assert!(
requests[1].ends_with(&format!("/versions/{version}")),
"the decrypt must resolve the version that wrapped it: {requests:?}"
);
}
}
/// Rewrap against a scripted Vault serving `state`, scripting the key record
/// plus every version record the state holds, so the implementation — not
/// the harness — decides which of them it needs. Returns the response
/// together with the requests the rewrap made.
async fn rewrap_scripted(state: &KeyState, ciphertext: &[u8]) -> (RewrapDataKeyResponse, Vec<String>) {
let mut responses = vec![ScriptedResponse::ok(kv2_read_data(&state.key_data))];
responses.extend(
state
.version_records
.iter()
.map(|record| ScriptedResponse::ok(kv2_read_version_record_data(record))),
);
let (vault, client) = scripted_client(responses).await;
let response = client
.rewrap_data_key(&RewrapDataKeyRequest {
ciphertext: ciphertext.to_vec(),
encryption_context: HashMap::new(),
})
.await
.expect("rewrap must produce an envelope on the current version");
(response, vault.requests())
}
/// Describe the wrapping of `ciphertext` against a scripted Vault serving
/// `state`.
async fn describe_wrapping_scripted(state: &KeyState, ciphertext: &[u8]) -> DescribeDataKeyWrappingResponse {
let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&state.key_data))]).await;
client
.describe_data_key_wrapping(&DescribeDataKeyWrappingRequest {
ciphertext: ciphertext.to_vec(),
encryption_context: HashMap::new(),
})
.await
.expect("describing an envelope's wrapping must succeed")
}
/// The whole point of the primitive: an envelope wrapped by a superseded
/// master key version comes back wrapped by the current one, carrying the
/// same data key.
///
/// Two independent things pin that the *old* material did the unwrapping.
/// The frozen version-1 record is read — an implementation that reached for
/// the current material would never ask for it — and the wrapping is
/// AES-256-GCM, so unwrapping with the wrong material cannot yield the
/// original data key at all, only an authentication failure. The closing
/// decrypt then shows the result is genuinely bound to version 2: it
/// resolves on the key record alone, with no version record in sight.
#[tokio::test]
async fn wired_kv2_rewrap_moves_an_old_envelope_onto_the_current_version() {
const PLAINTEXT: &[u8] = b"data-key-material-that-must-survive";
let state_v1 = KeyState::new(healthy_key_data());
let encrypted_v1 = encrypt_scripted(&state_v1, PLAINTEXT).await;
let state_v2 = rotate_scripted(&state_v1).await;
assert_ne!(
state_v2.key_data.encrypted_key_material, state_v1.key_data.encrypted_key_material,
"the rotation must have replaced the current material, or this test proves nothing"
);
let before = describe_wrapping_scripted(&state_v2, &encrypted_v1.ciphertext).await;
assert_eq!(before.key_version, Some(1));
assert_eq!(before.current_key_version, Some(2));
assert!(!before.is_current, "a version-1 envelope on a version-2 key is not current");
let (response, requests) = rewrap_scripted(&state_v2, &encrypted_v1.ciphertext).await;
assert!(response.rewrapped);
assert_eq!(response.source_key_version, Some(1));
assert_eq!(response.destination_key_version, Some(2));
assert_eq!(
requests,
vec![
"GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string(),
"GET /v1/secret/data/rustfs/kms/keys/wired-key/versions/1".to_string(),
],
"the unwrap must resolve the frozen version-1 material: {requests:?}"
);
let original: DataKeyEnvelope = serde_json::from_slice(&encrypted_v1.ciphertext).expect("envelope must parse");
let rewrapped: DataKeyEnvelope = serde_json::from_slice(&response.ciphertext).expect("rewrapped envelope must parse");
assert_eq!(rewrapped.master_key_version, Some(2), "the result must name the current version");
assert_ne!(rewrapped.encrypted_key, original.encrypted_key, "the wrapping must actually change");
// Everything but the wrapping is carried over, so the data key keeps its
// identity and its recorded age.
assert_eq!(rewrapped.key_id, original.key_id);
assert_eq!(rewrapped.key_spec, original.key_spec);
assert_eq!(rewrapped.encryption_context, original.encryption_context);
assert_eq!(rewrapped.created_at, original.created_at);
let (plaintext, decrypt_requests) = decrypt_scripted(&state_v2, &response.ciphertext).await;
assert_eq!(
plaintext, PLAINTEXT,
"the rewrapped envelope must yield the original data key byte for byte"
);
assert_eq!(
decrypt_requests.len(),
1,
"the rewrapped envelope must resolve on the current record alone: {decrypt_requests:?}"
);
let after = describe_wrapping_scripted(&state_v2, &response.ciphertext).await;
assert!(after.is_current, "the scan must agree the envelope no longer needs rewrapping");
}
/// Re-running a sweep must converge. An envelope already on the current
/// version comes back byte for byte with nothing to persist, rather than as
/// an equivalent envelope with a fresh nonce that would make every pass
/// rewrite every object's metadata forever.
#[tokio::test]
async fn wired_kv2_rewrap_of_a_current_envelope_is_a_no_op() {
let state_v1 = KeyState::new(healthy_key_data());
let state_v2 = rotate_scripted(&state_v1).await;
let encrypted_v2 = encrypt_scripted(&state_v2, b"written-after-the-rotation").await;
let described = describe_wrapping_scripted(&state_v2, &encrypted_v2.ciphertext).await;
assert!(described.is_current);
let (response, requests) = rewrap_scripted(&state_v2, &encrypted_v2.ciphertext).await;
assert!(!response.rewrapped, "an already-current envelope has nothing to rewrap");
assert_eq!(
response.ciphertext, encrypted_v2.ciphertext,
"a no-op rewrap must hand the input back unchanged"
);
assert_eq!(response.source_key_version, Some(2));
assert_eq!(response.destination_key_version, Some(2));
assert_eq!(requests.len(), 1, "a no-op must not reach for any version record: {requests:?}");
// Idempotence in the literal sense: feeding the result back in changes
// nothing again.
let (again, _) = rewrap_scripted(&state_v2, &response.ciphertext).await;
assert!(!again.rewrapped);
assert_eq!(again.ciphertext, encrypted_v2.ciphertext);
}
/// A pre-versioning envelope carries no version at all, so it can never
/// satisfy a retirement scan however current its material happens to be.
/// Rewrap therefore rewrites it to stamp the version — including on a
/// never-rotated key, where the material it is unwrapped with and the
/// material it is re-wrapped with are the same bytes.
#[tokio::test]
async fn wired_kv2_rewrap_stamps_a_pre_versioning_envelope() {
const PLAINTEXT: &[u8] = b"written-before-versioning-existed";
let state_v1 = KeyState::new(healthy_key_data());
let encrypted_v1 = encrypt_scripted(&state_v1, PLAINTEXT).await;
let legacy = strip_master_key_version(&encrypted_v1.ciphertext);
let legacy_envelope: DataKeyEnvelope = serde_json::from_slice(&legacy).expect("legacy envelope must parse");
assert_eq!(legacy_envelope.master_key_version, None);
// Never rotated: the resolved version is already the current one, and
// the envelope is still rewritten purely to record it.
let described = describe_wrapping_scripted(&state_v1, &legacy).await;
assert_eq!(described.key_version, Some(1));
assert_eq!(described.current_key_version, Some(1));
assert!(
!described.is_current,
"an envelope that does not state its version can never count as migrated"
);
let (response, requests) = rewrap_scripted(&state_v1, &legacy).await;
assert!(response.rewrapped);
assert_eq!(response.source_key_version, Some(1));
assert_eq!(response.destination_key_version, Some(1));
assert_eq!(requests.len(), 1, "a never-rotated key has no version record to read: {requests:?}");
let stamped: DataKeyEnvelope = serde_json::from_slice(&response.ciphertext).expect("stamped envelope must parse");
assert_eq!(stamped.master_key_version, Some(1));
let (plaintext, _) = decrypt_scripted(&state_v1, &response.ciphertext).await;
assert_eq!(plaintext, PLAINTEXT);
// After a rotation the same legacy envelope resolves through the frozen
// baseline instead, and lands on the rotated version.
let state_v2 = rotate_scripted(&state_v1).await;
assert_eq!(state_v2.key_data.baseline_version, Some(1));
let (rotated_response, rotated_requests) = rewrap_scripted(&state_v2, &legacy).await;
assert_eq!(rotated_response.source_key_version, Some(1), "the baseline is what wrapped it");
assert_eq!(rotated_response.destination_key_version, Some(2));
assert!(
rotated_requests[1].ends_with("/versions/1"),
"the unwrap must resolve the baseline material: {rotated_requests:?}"
);
let (rotated_plaintext, _) = decrypt_scripted(&state_v2, &rotated_response.ciphertext).await;
assert_eq!(rotated_plaintext, PLAINTEXT);
}
/// Drop the `master_key_version` field to produce the envelope shape a
/// pre-versioning build wrote.
fn strip_master_key_version(ciphertext: &[u8]) -> Vec<u8> {
let mut value: serde_json::Value = serde_json::from_slice(ciphertext).expect("envelope must parse");
value
.as_object_mut()
.expect("envelope is a JSON object")
.remove("master_key_version");
serde_json::to_vec(&value).expect("serialize legacy envelope")
}
/// The encryption context binds an envelope to one object. A caller that
/// cannot reproduce it is refused before any Vault read, so rewrap cannot be
/// used to launder an envelope onto a fresh wrapping.
#[tokio::test]
async fn wired_kv2_rewrap_rejects_a_tampered_encryption_context() {
let context = HashMap::from([("bucket".to_string(), "photos/cat.jpg".to_string())]);
let (vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&healthy_key_data()))]).await;
let encrypted = client
.encrypt(
&EncryptRequest {
key_id: "wired-key".to_string(),
plaintext: b"bound-to-one-object".to_vec(),
encryption_context: context.clone(),
grant_tokens: Vec::new(),
},
None,
)
.await
.expect("encrypt must produce an envelope");
let error = client
.rewrap_data_key(&RewrapDataKeyRequest {
ciphertext: encrypted.ciphertext.clone(),
encryption_context: HashMap::from([("bucket".to_string(), "photos/other.jpg".to_string())]),
})
.await
.expect_err("a context that does not match the envelope must be refused");
assert!(matches!(error, KmsError::ContextMismatch { .. }), "got {error:?}");
let error = client
.describe_data_key_wrapping(&DescribeDataKeyWrappingRequest {
ciphertext: encrypted.ciphertext,
encryption_context: HashMap::from([("bucket".to_string(), "photos/other.jpg".to_string())]),
})
.await
.expect_err("the read-only accessor must apply the same guard");
assert!(matches!(error, KmsError::ContextMismatch { .. }), "got {error:?}");
assert_eq!(
vault.requests().len(),
1,
"the context guard must run before any Vault read: {:?}",
vault.requests()
);
}
}
+563 -25
View File
@@ -19,8 +19,8 @@ use crate::backends::vault_credentials::{
token_source_for,
};
use crate::backends::{
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
ensure_tag_keys_are_mutable,
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, empty_key_page, ensure_key_state_permits,
ensure_rewrap_context_matches, ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys,
};
use crate::config::{KmsConfig, VaultTransitConfig};
use crate::encryption::{DataKeyEnvelope, generate_key_material};
@@ -45,6 +45,7 @@ use vaultrs::{
requests::{
CreateKeyRequestBuilder, DecryptDataRequestBuilder, EncryptDataRequestBuilder, UpdateKeyConfigurationRequestBuilder,
},
responses::ReadKeyData,
},
error::ClientError,
kv2,
@@ -73,6 +74,19 @@ const METADATA_CACHE_TTL: Duration = Duration::from_secs(300);
/// grow process memory without limit.
const METADATA_CACHE_CAPACITY: u64 = 1024;
/// Read the key version out of a Transit ciphertext's `vault:vN:` prefix.
///
/// Transit ciphertext self-describes the version that wrapped it, which is why
/// [`DataKeyEnvelope::master_key_version`] stays `None` on this backend. The
/// prefix is therefore the only place a rewrap can learn whether it changed
/// anything. `None` means the ciphertext is not in a shape this backend
/// produced, and callers must treat the version as unknown rather than assume
/// one.
fn transit_ciphertext_version(ciphertext: &str) -> Option<u32> {
let (version, _) = ciphertext.strip_prefix("vault:v")?.split_once(':')?;
version.parse().ok()
}
/// Whether a KV2 write failed its check-and-set precondition.
///
/// Mirrors the helper of the same name in `vault.rs`; the two backends keep
@@ -358,6 +372,47 @@ impl VaultTransitKmsClient {
.map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))
}
/// Re-encrypt a Transit ciphertext under the key's latest version without
/// the plaintext ever leaving Vault.
///
/// Classified as an idempotent read because it is one: Vault mutates
/// nothing, and a replayed attempt only produces another ciphertext of the
/// same data key under the same version.
async fn transit_rewrap(&self, key_id: &str, ciphertext: &str) -> Result<String> {
let response = self
.run("vault_transit_rewrap", OpClass::ReadIdempotent, move || async move {
let vault = self.vault().map_err(AttemptError::fatal)?;
data::rewrap(&vault.client, &self.config.mount_path, key_id, ciphertext, None)
.await
.map_err(|e| AttemptError::from_vaultrs(e, |e| Self::map_vault_error(key_id, e, "rewrap")))
})
.await?;
Ok(response.ciphertext)
}
/// Vault's own newest version of a transit key.
///
/// `transit/keys/:name` reports retained versions as a version-number to
/// creation-time map rather than as a single "latest" field, so the newest
/// version is the largest entry in it.
///
/// Read from Vault rather than taken from the RustFS metadata record's
/// `current_version` counter: that counter only advances when a rotation
/// goes through this process, while `transit/rewrap` always targets Vault's
/// notion of latest. The scan that decides whether a rewrap is still needed
/// and the rewrap that acts on it must answer to the same authority, or an
/// operator-side `vault write -f transit/keys/x/rotate` would leave the two
/// permanently disagreeing.
async fn latest_transit_key_version(&self, key_id: &str) -> Result<Option<u32>> {
let response = self.read_transit_key(key_id).await?;
let latest = match &response.keys {
ReadKeyData::Symmetric(versions) => versions.keys().filter_map(|version| version.parse::<u32>().ok()).max(),
ReadKeyData::Asymmetric(versions) => versions.keys().filter_map(|version| version.parse::<u32>().ok()).max(),
};
Ok(latest)
}
fn metadata_key_path(&self, key_id: &str) -> String {
format!("{}/{}", self.metadata_key_prefix, key_id)
}
@@ -758,6 +813,125 @@ impl VaultTransitKmsClient {
}
}
/// Report which transit key version wraps an envelope, and whether that is
/// the key's latest version.
///
/// Reads only key metadata, never the ciphertext's contents, so it answers
/// for AAD-bound envelopes that [`Self::rewrap_data_key`] has to refuse — an
/// inventory must be able to count exactly the envelopes that are stuck.
pub(crate) async fn describe_data_key_wrapping(
&self,
request: &DescribeDataKeyWrappingRequest,
) -> Result<DescribeDataKeyWrappingResponse> {
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
.map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?;
ensure_rewrap_context_matches(&envelope.encryption_context, &request.encryption_context)?;
let source_ciphertext = std::str::from_utf8(&envelope.encrypted_key)
.map_err(|e| KmsError::cryptographic_error("utf8", format!("Invalid Transit ciphertext: {e}")))?;
let key_version = transit_ciphertext_version(source_ciphertext);
let current_key_version = self.latest_transit_key_version(&envelope.master_key_id).await?;
Ok(DescribeDataKeyWrappingResponse {
key_id: envelope.master_key_id,
key_version,
current_key_version,
// An unreadable prefix on either side means the version is unknown,
// and unknown must never read as "already current" — that is the
// answer that lets an operator destroy a version still in use.
is_current: key_version.is_some() && key_version == current_key_version,
})
}
/// Re-wrap an existing envelope onto the transit key's latest version using
/// Vault's native rewrap endpoint.
///
/// The data key is never decrypted into this process: Vault re-encrypts the
/// ciphertext internally and hands back only the new ciphertext, so no
/// `transit/decrypt` is issued and no plaintext data key exists here to
/// leak, log or persist.
///
/// # Envelopes bound to an encryption context cannot be rewrapped
///
/// This backend binds the encryption context into the wrapping as AEAD
/// associated data ([`Self::transit_encrypt`]), and Vault's `transit/rewrap`
/// endpoint accepts no `associated_data` parameter — the only way to move
/// such a ciphertext onto a newer version is `transit/decrypt` followed by
/// `transit/encrypt`, which materializes the plaintext data key inside
/// RustFS. That trade is refused here rather than made silently: it would
/// hand back a valid envelope while dropping the very property that makes a
/// backend-side rewrap worth having. Every object-level envelope carries a
/// bucket/object context, so in practice this rejects them all until the
/// context binding or the endpoint changes.
///
/// The context guard still runs first, so a caller that cannot reproduce the
/// envelope's context is told that rather than being told about the AAD
/// limitation of an envelope it has no claim on.
pub(crate) async fn rewrap_data_key(&self, request: &RewrapDataKeyRequest) -> Result<RewrapDataKeyResponse> {
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
.map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?;
ensure_rewrap_context_matches(&envelope.encryption_context, &request.encryption_context)?;
self.ensure_key_state_allows(&envelope.master_key_id, StateGatedOperation::Encrypt)
.await?;
if !envelope.encryption_context.is_empty() {
return Err(KmsError::rewrap_would_expose_plaintext(
&envelope.master_key_id,
"the envelope binds its encryption context as AEAD associated data, which Vault Transit's rewrap endpoint \
cannot carry; rewrapping it would require decrypting the data key inside RustFS",
));
}
let source_ciphertext = std::str::from_utf8(&envelope.encrypted_key)
.map_err(|e| KmsError::cryptographic_error("utf8", format!("Invalid Transit ciphertext: {e}")))?;
let source_key_version = transit_ciphertext_version(source_ciphertext);
let rewrapped_ciphertext = match self.transit_rewrap(&envelope.master_key_id, source_ciphertext).await {
Ok(ciphertext) => ciphertext,
Err(error) => {
self.invalidate_metadata_on_state_error(&envelope.master_key_id, &error).await;
return Err(error);
}
};
let destination_key_version = transit_ciphertext_version(&rewrapped_ciphertext);
// Vault re-encrypts unconditionally, so an already-current ciphertext
// comes back changed but no newer. Report that as "nothing to persist"
// and hand the input back untouched, or a repeated sweep would rewrite
// every object's metadata on every pass forever.
if source_key_version.is_some() && source_key_version == destination_key_version {
return Ok(RewrapDataKeyResponse {
ciphertext: request.ciphertext.clone(),
key_id: envelope.master_key_id,
source_key_version,
destination_key_version,
rewrapped: false,
});
}
let rewrapped_envelope = DataKeyEnvelope {
key_id: envelope.key_id,
master_key_id: envelope.master_key_id,
key_spec: envelope.key_spec,
encrypted_key: rewrapped_ciphertext.into_bytes(),
nonce: envelope.nonce,
encryption_context: envelope.encryption_context,
created_at: envelope.created_at,
// Transit ciphertext still self-describes its version, so the
// envelope field stays absent exactly as generate_data_key leaves it.
master_key_version: None,
};
let ciphertext = serde_json::to_vec(&rewrapped_envelope)?;
Ok(RewrapDataKeyResponse {
ciphertext,
key_id: rewrapped_envelope.master_key_id,
source_key_version,
destination_key_version,
rewrapped: true,
})
}
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
#[cfg(test)]
pub(crate) async fn create_key(
@@ -852,7 +1026,12 @@ impl VaultTransitKmsClient {
request: &ListKeysRequest,
_context: Option<&OperationContext>,
) -> Result<ListKeysResponse> {
let all_keys = self
// A caller asking for no keys is answered without reaching Vault.
if list_keys_page_size(request.limit).is_none() {
return Ok(empty_key_page());
}
let mut all_keys = self
.run("vault_transit_list_keys", OpClass::ReadIdempotent, move || async move {
let vault = self.vault().map_err(AttemptError::fatal)?;
key::list(&vault.client, &self.config.mount_path).await.map_err(|e| {
@@ -861,36 +1040,27 @@ impl VaultTransitKmsClient {
})
.await?
.keys;
// Vault's own LIST ordering is not part of its contract, so the sort is
// what makes the marker a stable cursor across calls.
all_keys.sort_unstable();
let page = paginate_keys(&all_keys, request, String::as_str);
let mut filtered = Vec::new();
for key_id in all_keys {
let key_info = self.key_info(&key_id).await?;
// Reading metadata only for the page keeps a list bounded by the
// requested limit instead of by the size of the transit mount.
let mut keys = Vec::with_capacity(page.items.len());
for key_id in page.items {
let key_info = self.key_info(key_id).await?;
let usage_matches = request.usage_filter.as_ref().is_none_or(|usage| usage == &key_info.usage);
let status_matches = request.status_filter.as_ref().is_none_or(|status| status == &key_info.status);
if usage_matches && status_matches {
filtered.push(key_info);
keys.push(key_info);
}
}
let start_idx = request
.marker
.as_ref()
.and_then(|marker| filtered.iter().position(|info| &info.key_id == marker))
.map(|idx| idx + 1)
.unwrap_or(0);
let limit = request.limit.unwrap_or(100) as usize;
let end_idx = std::cmp::min(start_idx + limit, filtered.len());
let keys = filtered[start_idx..end_idx].to_vec();
let next_marker = if end_idx < filtered.len() {
Some(filtered[end_idx - 1].key_id.clone())
} else {
None
};
Ok(ListKeysResponse {
keys,
next_marker,
truncated: end_idx < filtered.len(),
next_marker: page.next_marker,
truncated: page.truncated,
})
}
@@ -1170,6 +1340,17 @@ impl KmsBackend for VaultTransitKmsBackend {
})
}
async fn rewrap_data_key(&self, request: RewrapDataKeyRequest) -> Result<RewrapDataKeyResponse> {
self.client.rewrap_data_key(&request).await
}
async fn describe_data_key_wrapping(
&self,
request: DescribeDataKeyWrappingRequest,
) -> Result<DescribeDataKeyWrappingResponse> {
self.client.describe_data_key_wrapping(&request).await
}
async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
let generate_request = GenerateKeyRequest {
master_key_id: request.key_id.clone(),
@@ -1312,7 +1493,11 @@ impl KmsBackend for VaultTransitKmsBackend {
fn capabilities(&self) -> BackendCapabilities {
// Vault Transit natively supports version-retaining rotation, keeps
// prior versions addressable for decryption, and allows physical
// deletion once a key is pending deletion.
// deletion once a key is pending deletion. Rewrap is advertised because
// the endpoint exists and works; envelopes whose encryption context is
// bound as associated data are still refused per envelope (see
// `VaultTransitKmsClient::rewrap_data_key`), which is a property of the
// envelope rather than of the backend.
BackendCapabilities::minimal()
.with_rotate(true)
.with_enable_disable(true)
@@ -1320,6 +1505,7 @@ impl KmsBackend for VaultTransitKmsBackend {
.with_versioning(true)
.with_physical_delete(true)
.with_update_key_metadata(true)
.with_rewrap(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -1453,6 +1639,35 @@ mod tests {
serde_json::to_value(&response).expect("serialize transit key read response")
}
/// A caller asking for no keys gets an empty page, and the page arithmetic
/// never reaches for the element before an empty page. The scripted key
/// listing stays unused: a request for zero keys has nothing to ask Vault.
#[tokio::test]
async fn zero_limit_list_returns_an_empty_page_without_calling_vault() {
let (vault, client) =
scripted_client(vec![ScriptedResponse::ok(serde_json::json!({ "keys": ["key-a", "key-b"] }))]).await;
let response = client
.list_keys(
&ListKeysRequest {
limit: Some(0),
..Default::default()
},
None,
)
.await
.expect("a zero-limit list must succeed");
assert!(response.keys.is_empty());
assert!(!response.truncated);
assert!(response.next_marker.is_none());
assert!(
vault.requests().is_empty(),
"a request for no keys must not reach Vault: {:?}",
vault.requests()
);
}
#[tokio::test]
async fn wired_transit_encrypt_retries_transient_status() {
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
@@ -2258,4 +2473,327 @@ mod tests {
"the sweep must not touch the transit key after backing off: {requests:?}"
);
}
/// The forward half of the rotation contract on the Transit path: a data key
/// generated before a rotation must still be decryptable after it.
///
/// Transit key material never leaves Vault, so an offline responder cannot
/// prove the cryptographic round trip — `test_transit_old_ciphertext_decrypts_after_rotate`
/// keeps that against a live Vault. What this pins is the wiring the client
/// owns and could regress on its own: the historical `vault:v1:` ciphertext
/// is forwarded to Vault byte for byte after the rotation bumped the
/// recorded version, and nothing on the decrypt path gates on that version.
#[tokio::test]
async fn wired_transit_pre_rotation_data_key_is_decrypted_unchanged() {
const CIPHERTEXT_V1: &str = "vault:v1:scripted-pre-rotation";
const RECOVERED_DEK: [u8; 32] = [0x37u8; 32];
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let (vault, client) = scripted_client(vec![
// generate_data_key: state gate reads the metadata record, then the
// transit encrypt returns first-version ciphertext.
ScriptedResponse::ok(metadata_read_data(&metadata)),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": CIPHERTEXT_V1 })),
// rotate: the state gate hits the metadata cache, the rotation
// commits, and the versioned read+write records the version bump.
ScriptedResponse::ok(serde_json::json!({})),
ScriptedResponse::ok(kv2_metadata_read_data(1)),
ScriptedResponse::ok(metadata_read_data(&metadata)),
ScriptedResponse::ok(kv2_write_ack()),
// decrypt of the pre-rotation envelope; Vault owns the transit
// crypto, so the recovered material is the responder's to hand back.
ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
])
.await;
let data_key = client
.generate_data_key(
&GenerateKeyRequest {
master_key_id: "wired-key".to_string(),
key_spec: "AES_256".to_string(),
key_length: Some(32),
encryption_context: HashMap::new(),
grant_tokens: Vec::new(),
},
None,
)
.await
.expect("generate_data_key must produce an envelope");
let envelope: DataKeyEnvelope = serde_json::from_slice(&data_key.ciphertext).expect("envelope must parse");
assert_eq!(envelope.encrypted_key, CIPHERTEXT_V1.as_bytes());
let rotated = client.rotate_key("wired-key", None).await.expect("rotation must commit");
assert_eq!(rotated.version, 2, "the rotation must record the version bump");
let plaintext = client
.decrypt(
&DecryptRequest {
ciphertext: data_key.ciphertext.clone(),
encryption_context: HashMap::new(),
grant_tokens: Vec::new(),
},
None,
)
.await
.expect("a pre-rotation data key must stay decryptable");
assert_eq!(
plaintext,
RECOVERED_DEK.to_vec(),
"the decrypt must hand back the recovered material, not merely avoid an error"
);
let requests = vault.requests();
assert_eq!(requests.len(), 7, "{requests:?}");
assert_eq!(requests[6], "POST /v1/transit/decrypt/wired-key", "{requests:?}");
// The version the rotation recorded, and the ciphertext the decrypt sent:
// the client must forward the historical version verbatim instead of
// re-stamping it to (or pinning the request at) the current one.
let bodies = vault.request_bodies();
let recorded: serde_json::Value = serde_json::from_str(&bodies[5]).expect("metadata write body must be JSON");
assert_eq!(recorded["data"]["current_version"], serde_json::json!(2), "{recorded}");
let decrypt_body: serde_json::Value = serde_json::from_str(&bodies[6]).expect("decrypt body must be JSON");
assert_eq!(
decrypt_body["ciphertext"],
serde_json::json!(CIPHERTEXT_V1),
"the pre-rotation ciphertext must reach Vault unchanged: {decrypt_body}"
);
}
/// Transit read-key payload for a key that has been rotated up to `latest`.
fn transit_key_read_data_up_to(key_id: &str, latest: u32) -> serde_json::Value {
let mut response: serde_json::Value = transit_key_read_data(key_id);
let keys: serde_json::Map<String, serde_json::Value> = (1..=latest)
.map(|version| (version.to_string(), serde_json::json!(1_700_000_000_u64 + u64::from(version))))
.collect();
response["keys"] = serde_json::Value::Object(keys);
response
}
fn wired_key_request(context: HashMap<String, String>) -> GenerateKeyRequest {
GenerateKeyRequest {
master_key_id: "wired-key".to_string(),
key_spec: "AES_256".to_string(),
key_length: Some(32),
encryption_context: context,
grant_tokens: Vec::new(),
}
}
#[test]
fn transit_ciphertext_version_reads_only_a_well_formed_prefix() {
assert_eq!(transit_ciphertext_version("vault:v1:abc"), Some(1));
assert_eq!(transit_ciphertext_version("vault:v27:abc"), Some(27));
// Anything else leaves the version unknown rather than guessing one; an
// invented version is what would let a still-referenced key version be
// reported as retired.
assert_eq!(transit_ciphertext_version("vault:v:abc"), None);
assert_eq!(transit_ciphertext_version("vault:vx:abc"), None);
assert_eq!(transit_ciphertext_version("vault:v1"), None);
assert_eq!(transit_ciphertext_version("v1:abc"), None);
assert_eq!(transit_ciphertext_version(""), None);
}
/// The property that justifies having a Transit-specific rewrap at all: the
/// data key is re-encrypted by Vault, so no `transit/decrypt` is issued and
/// no plaintext data key ever exists inside this process.
#[tokio::test]
async fn wired_transit_rewrap_uses_the_native_endpoint_and_never_decrypts() {
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let (vault, client) = scripted_client(vec![
// generate_data_key: metadata state gate, then the transit encrypt.
ScriptedResponse::ok(metadata_read_data(&metadata)),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })),
// rewrap: the state gate hits the metadata cache, so the only call
// is the native rewrap.
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v3:rewrapped" })),
])
.await;
let data_key = client
.generate_data_key(&wired_key_request(HashMap::new()), None)
.await
.expect("generate_data_key must produce an envelope");
let original: DataKeyEnvelope = serde_json::from_slice(&data_key.ciphertext).expect("envelope must parse");
let response = client
.rewrap_data_key(&RewrapDataKeyRequest {
ciphertext: data_key.ciphertext.clone(),
encryption_context: HashMap::new(),
})
.await
.expect("rewrap must move the ciphertext onto the latest version");
assert!(response.rewrapped);
assert_eq!(response.source_key_version, Some(1));
assert_eq!(response.destination_key_version, Some(3));
let rewrapped: DataKeyEnvelope = serde_json::from_slice(&response.ciphertext).expect("rewrapped envelope must parse");
assert_eq!(rewrapped.encrypted_key, b"vault:v3:rewrapped".to_vec());
assert_eq!(
rewrapped.master_key_version, None,
"Transit ciphertext self-describes its version, so the envelope field must stay absent"
);
assert_eq!(rewrapped.key_id, original.key_id);
assert_eq!(rewrapped.created_at, original.created_at);
assert_eq!(rewrapped.encryption_context, original.encryption_context);
let requests = vault.requests();
assert!(
requests.contains(&"POST /v1/transit/rewrap/wired-key".to_string()),
"the native rewrap endpoint must be used: {requests:?}"
);
assert!(
!requests.iter().any(|request| request.contains("/transit/decrypt/")),
"no decrypt may be issued: the plaintext data key must never enter this process: {requests:?}"
);
// The ciphertext Vault was asked to rewrap is the one that was stored,
// byte for byte.
let bodies = vault.request_bodies();
let rewrap_index = requests
.iter()
.position(|request| request == "POST /v1/transit/rewrap/wired-key")
.expect("the rewrap request must be recorded");
let body: serde_json::Value = serde_json::from_str(&bodies[rewrap_index]).expect("rewrap body must be JSON");
assert_eq!(body["ciphertext"], serde_json::json!("vault:v1:scripted"), "{body}");
}
/// Vault re-encrypts unconditionally, so an already-latest ciphertext comes
/// back different but no newer. That must report as "nothing to persist", or
/// every sweep pass would rewrite every object's metadata forever.
#[tokio::test]
async fn wired_transit_rewrap_of_a_current_ciphertext_is_a_no_op() {
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let (_vault, client) = scripted_client(vec![
ScriptedResponse::ok(metadata_read_data(&metadata)),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })),
// Same version back, different bytes.
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:re-encrypted" })),
])
.await;
let data_key = client
.generate_data_key(&wired_key_request(HashMap::new()), None)
.await
.expect("generate_data_key must produce an envelope");
let response = client
.rewrap_data_key(&RewrapDataKeyRequest {
ciphertext: data_key.ciphertext.clone(),
encryption_context: HashMap::new(),
})
.await
.expect("rewrap must succeed");
assert!(!response.rewrapped, "a ciphertext already on the latest version is a no-op");
assert_eq!(
response.ciphertext, data_key.ciphertext,
"a no-op must hand the stored envelope back unchanged, not Vault's fresh re-encryption"
);
assert_eq!(response.source_key_version, Some(1));
assert_eq!(response.destination_key_version, Some(1));
}
/// Vault's `transit/rewrap` endpoint takes no `associated_data` parameter,
/// and this backend binds the encryption context as exactly that. The only
/// remaining route would decrypt the data key inside RustFS, so the request
/// is refused rather than silently downgraded — and refused without any call
/// to Vault at all.
#[tokio::test]
async fn wired_transit_rewrap_refuses_an_aad_bound_envelope() {
let context = HashMap::from([("bucket".to_string(), "photos/cat.jpg".to_string())]);
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let (vault, client) = scripted_client(vec![
ScriptedResponse::ok(metadata_read_data(&metadata)),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })),
// Only the read-only accessor below is allowed to consume this.
ScriptedResponse::ok(transit_key_read_data_up_to("wired-key", 2)),
])
.await;
let data_key = client
.generate_data_key(&wired_key_request(context.clone()), None)
.await
.expect("generate_data_key must produce an envelope");
let error = client
.rewrap_data_key(&RewrapDataKeyRequest {
ciphertext: data_key.ciphertext.clone(),
encryption_context: context.clone(),
})
.await
.expect_err("an AAD-bound envelope must not be rewrapped by decrypting it here");
assert!(
matches!(&error, KmsError::RewrapWouldExposePlaintext { key_id, .. } if key_id == "wired-key"),
"got {error:?}"
);
// The stuck envelope must still be countable, or an inventory could not
// report how much of the key version is unmigratable.
let described = client
.describe_data_key_wrapping(&DescribeDataKeyWrappingRequest {
ciphertext: data_key.ciphertext.clone(),
encryption_context: context,
})
.await
.expect("describing the wrapping must work even when rewrapping it cannot");
assert_eq!(described.key_version, Some(1));
assert_eq!(described.current_key_version, Some(2));
assert!(!described.is_current);
let requests = vault.requests();
assert!(
!requests.iter().any(|request| request.contains("/transit/rewrap/")),
"the refusal must happen before any rewrap call: {requests:?}"
);
assert!(
!requests.iter().any(|request| request.contains("/transit/decrypt/")),
"and above all before any decrypt: {requests:?}"
);
}
/// The current version comes from Vault's own key record rather than from
/// the RustFS metadata counter, which only advances on rotations this
/// process performed.
#[tokio::test]
async fn wired_transit_describe_wrapping_reads_vaults_latest_version() {
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
assert_eq!(metadata.current_version, 1, "the RustFS counter still says version 1");
let (vault, client) = scripted_client(vec![
ScriptedResponse::ok(metadata_read_data(&metadata)),
ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })),
// Vault has been rotated behind RustFS's back.
ScriptedResponse::ok(transit_key_read_data_up_to("wired-key", 4)),
])
.await;
let data_key = client
.generate_data_key(&wired_key_request(HashMap::new()), None)
.await
.expect("generate_data_key must produce an envelope");
let described = client
.describe_data_key_wrapping(&DescribeDataKeyWrappingRequest {
ciphertext: data_key.ciphertext.clone(),
encryption_context: HashMap::new(),
})
.await
.expect("describing the wrapping must succeed");
assert_eq!(described.key_id, "wired-key");
assert_eq!(described.key_version, Some(1));
assert_eq!(
described.current_key_version,
Some(4),
"the latest version must come from Vault, not from the RustFS metadata counter"
);
assert!(!described.is_current);
let requests = vault.requests();
assert!(
requests.contains(&"GET /v1/transit/keys/wired-key".to_string()),
"the latest version must be read from the transit key record: {requests:?}"
);
}
}
+4 -3
View File
@@ -65,9 +65,10 @@ pub enum AtRestProtection {
/// for bundling purposes: re-wrap is mandatory.
LegacyUnspecified,
/// Vault KV2 as currently shipped: material confidentiality relies on
/// Vault ACLs, KV2 at-rest encryption, and TLS only (the backend reports
/// `at_rest_protection = "vault-kv2-acl"`); RustFS applies no
/// cryptographic wrapping of its own.
/// Vault ACLs, KV2 at-rest encryption, and TLS only; RustFS applies no
/// cryptographic wrapping of its own. The backend reports nothing about
/// that boundary at runtime, so this value is a backup-manifest
/// declaration, not a backend self-report.
StorageOnly,
/// Vault KV2 with material wrapped by Vault Transit before storage. Not
/// produced by any current backend; the row exists so a future direction
File diff suppressed because it is too large Load Diff
+5 -2
View File
@@ -27,8 +27,10 @@ use serde::{Deserialize, Serialize};
/// Machine-readable category of a restore blocker.
///
/// The first six codes mirror the [`BackupError`] variants; the remaining
/// codes cover preflight conditions that are not bundle defects.
/// The first six codes mirror the [`BackupError`] variants — an unknown
/// format version reports the same code whether it came from the manifest or
/// from a bundled key record — and the remaining codes cover preflight
/// conditions that are not bundle defects.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum RestoreBlockerCode {
@@ -73,6 +75,7 @@ impl From<&BackupError> for RestoreBlocker {
BackupError::WrongKek { .. } => RestoreBlockerCode::WrongBackupKek,
BackupError::MissingArtifact { .. } => RestoreBlockerCode::MissingArtifact,
BackupError::IncompleteBundle { .. } => RestoreBlockerCode::IncompleteBundle,
BackupError::UnsupportedRecordVersion { .. } => RestoreBlockerCode::UnknownFormatVersion,
};
Self {
code,
+16
View File
@@ -60,6 +60,13 @@ pub enum BackupError {
/// restored, regardless of how much of it is readable.
#[error("backup bundle is incomplete ({reason}); incomplete bundles must never be restored")]
IncompleteBundle { reason: String },
/// A bundled key record declares an at-rest protection mode this build
/// does not implement. Deliberately distinct from [`Self::Corrupted`]:
/// the record is intact and a newer build reads it fine, so the operator
/// response is a version change, not a disaster recovery.
#[error("bundled key record '{key_id}' declares unsupported at-rest protection {version:?}; this build cannot restore it")]
UnsupportedRecordVersion { key_id: String, version: String },
}
impl BackupError {
@@ -129,5 +136,14 @@ mod tests {
BackupError::incomplete_bundle("manifest has no completeness marker").to_string(),
"backup bundle is incomplete (manifest has no completeness marker); incomplete bundles must never be restored"
);
let unsupported_record = BackupError::UnsupportedRecordVersion {
key_id: "alpha".to_string(),
version: "post-quantum-v2".to_string(),
};
assert_eq!(
unsupported_record.to_string(),
"bundled key record 'alpha' declares unsupported at-rest protection \"post-quantum-v2\"; this build cannot restore it"
);
}
}
+37 -2
View File
@@ -48,7 +48,7 @@
//! published. A crash at any earlier point leaves a bundle without a
//! manifest, which decodes as an incomplete bundle and can never be restored.
use crate::backends::local::{LocalKmsClient, StoredKeyProtection};
use crate::backends::local::{LocalKmsClient, StoredKeyProtection, unknown_protection_marker};
use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
use crate::backup::error::BackupError;
use crate::backup::manifest::{
@@ -376,7 +376,16 @@ async fn collect_snapshot(client: &LocalKmsClient) -> Result<CollectedSnapshot>
let raw = Zeroizing::new(fs::read(&path).await?);
// Any unreadable record aborts the export: a bundle silently missing
// one key is worse than no bundle at all.
// one key is worse than no bundle at all. The protection marker is
// classified first so a record from a newer build keeps its own
// verdict — an operator who reads "material corrupt" starts a
// disaster recovery for what is only a version mismatch.
let unknown_marker = unknown_protection_marker(&raw).map_err(|error| {
KmsError::material_corrupt(&stem, format!("stored key record is not a readable JSON object: {error}"))
})?;
if let Some(version) = unknown_marker {
return Err(KmsError::unsupported_format_version(&stem, version));
}
let probe: StoredRecordProbe = serde_json::from_slice(&raw)
.map_err(|error| KmsError::material_corrupt(&stem, format!("stored key record does not deserialize: {error}")))?;
if probe.key_id != stem {
@@ -1115,4 +1124,30 @@ mod tests {
.expect_err("identity mismatch must abort the export");
assert!(matches!(error, KmsError::InvalidKey { .. }), "got {error:?}");
}
/// A record written by a newer build must abort the export as an
/// unsupported format, not as corrupt material: the two verdicts send the
/// operator down completely different runbooks, and only one of them is
/// true here.
#[tokio::test]
async fn record_from_a_newer_build_aborts_export_as_unsupported_format() {
let (client, _key_dir) = encrypted_client().await;
client.create_key("alpha", "AES_256", None).await.expect("create key");
let record_path = client.key_directory().join("alpha.key");
let mut record: serde_json::Value =
serde_json::from_slice(&std::fs::read(&record_path).expect("read record")).expect("decode record");
record["at_rest_protection"] = serde_json::json!("post-quantum-v2");
std::fs::write(&record_path, serde_json::to_vec_pretty(&record).expect("encode record")).expect("write record");
let bundle = TempDir::new().expect("bundle dir");
let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle")))
.await
.expect_err("an uninterpretable record must abort the export");
assert!(
matches!(&error, KmsError::UnsupportedFormatVersion { key_id, version }
if key_id == "alpha" && version == "post-quantum-v2"),
"got {error:?}"
);
}
}
+87 -1
View File
@@ -54,7 +54,7 @@
use crate::backends::local::{
LOCAL_KMS_MASTER_KEY_SALT_FILE, LOCAL_KMS_MASTER_KEY_SALT_LEN, LOCAL_RESTORE_COMMIT_MARKER_FILE, LocalKmsClient,
StoredKeyProtection, durable_file, is_orphan_commit_temp_name, validate_key_id,
StoredKeyProtection, durable_file, is_orphan_commit_temp_name, unknown_protection_marker, validate_key_id,
};
use crate::backup::capability::AtRestProtection;
use crate::backup::dry_run::{
@@ -605,6 +605,14 @@ fn decode_key_record(
.to_string();
validate_key_id(&stem)?;
// Classify the protection marker before the schema parse: a record from a
// newer build is not a damaged bundle, and reporting it as corruption
// sends the operator into disaster recovery instead of a version change.
let unknown_marker = unknown_protection_marker(&plaintext)
.map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' is not a readable JSON object: {error}")))?;
if let Some(version) = unknown_marker {
return Err(BackupError::UnsupportedRecordVersion { key_id: stem, version }.into());
}
let probe: RestoredRecordProbe = serde_json::from_slice(&plaintext)
.map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' does not deserialize: {error}")))?;
if probe.key_id != stem {
@@ -2056,4 +2064,82 @@ mod tests {
.find(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted)
.expect("a tampered config artifact must be reported as corruption");
}
/// A bundled record from a newer build is not a damaged bundle: the bytes
/// are intact and the build that wrote them reads them back. Reporting it
/// as corruption sends the operator into a disaster recovery for what a
/// version change fixes, so the verdict must stay separate all the way
/// out to the dry-run blocker code.
#[test]
fn bundled_record_from_a_newer_build_is_not_reported_as_corruption() {
let record = serde_json::json!({
"key_id": "alpha",
"at_rest_protection": "post-quantum-v2",
"encrypted_key_material": "AAAAAAAAAAAAAAAAAAAAAA==",
"nonce": vec![0u8; 12],
});
let error = match decode_key_record(
"artifacts/keys/alpha.key.enc",
Zeroizing::new(serde_json::to_vec(&record).expect("encode record")),
&[AtRestProtection::EncryptedMasterKey],
) {
Ok(_) => panic!("a record this build cannot interpret must be rejected"),
Err(error) => error,
};
let KmsError::Backup(inner) = &error else {
panic!("expected a backup error, got {error:?}");
};
assert!(
matches!(inner, BackupError::UnsupportedRecordVersion { key_id, version }
if key_id == "alpha" && version == "post-quantum-v2"),
"got {inner:?}"
);
assert_eq!(
RestoreBlocker::from(inner).code,
RestoreBlockerCode::UnknownFormatVersion,
"a dry run must report a version blocker, not a corruption blocker"
);
}
/// The commit marker's format-version branch, symmetric with the Vault
/// restore marker's: an unknown version is refused outright everywhere the
/// marker is read, and the backend refuses to start while it is present.
#[tokio::test]
async fn restore_commit_marker_from_a_newer_build_is_refused() {
let mut marker = serde_json::to_value(RestoreCommitMarker {
format_version: RESTORE_MARKER_FORMAT_VERSION,
backup_id: BACKUP_ID.to_string(),
manifest_digest: ContentDigest::sha256_of(b"manifest"),
files: vec![LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string()],
})
.expect("marker json");
marker["format_version"] = serde_json::json!(99);
let marker = serde_json::to_vec_pretty(&marker).expect("marker bytes");
let error = RestoreCommitMarker::decode(&marker).expect_err("an unknown marker version must be refused");
assert!(error.to_string().contains("unknown format version 99"), "got {error}");
// Reachable from the restore entry point, not just from the decoder.
let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await;
let work = TempDir::new().expect("work dir");
let (bundle, _manifest) = export_bundle(&client, work.path()).await;
let target = TempDir::new().expect("target dir");
fs::write(target.path().join(LOCAL_RESTORE_COMMIT_MARKER_FILE), &marker)
.await
.expect("seed marker");
let before = snapshot_tree(target.path());
let error = restore_local_backup(&test_kek(), &restore_request(&bundle, target.path()))
.await
.expect_err("an unknown marker version must block the restore");
assert!(error.to_string().contains("unknown format version 99"), "got {error}");
assert_eq!(snapshot_tree(target.path()), before, "the refused restore must not write");
// And the backend itself refuses to start on any marker at all.
let error = match LocalKmsClient::new(local_config(target.path(), Some(TEST_MASTER_KEY))).await {
Ok(_) => panic!("a restore marker must block startup"),
Err(error) => error,
};
assert!(error.to_string().contains("unfinished restore"), "got {error}");
}
}
+7 -1
View File
@@ -21,7 +21,8 @@
//! [`vault_restore`] orchestrates the consumer side for the Vault backends,
//! whose cryptographic root is restored by Vault's own disaster-recovery
//! flow. All are crate-internal APIs; the admin API builds on these pieces in
//! follow-up changes.
//! follow-up changes. [`drill`] rehearses the whole loop end to end and turns
//! it into machine-readable evidence.
//!
//! # Bundle model
//!
@@ -51,6 +52,7 @@
//! format version.
mod capability;
pub mod drill;
mod dry_run;
mod error;
pub mod local_export;
@@ -59,6 +61,10 @@ mod manifest;
pub mod vault_restore;
pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
pub use drill::{
DRILL_EVIDENCE_FORMAT_VERSION, DrillBundleEvidence, DrillDataset, DrillDisaster, DrillEvidence, DrillPhase, DrillPhaseTiming,
DrillRecoveryEvidence, DrillRequest, DrillRpoEvidence, DrillVerdict, EnvelopeProbe, run_local_drill,
};
pub use dry_run::{
ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport,
};
+5 -2
View File
@@ -114,10 +114,13 @@ use vaultrs::api::transit::responses::ReadKeyData;
use vaultrs::error::ClientError;
use vaultrs::{kv2, transit::key};
// The bundle layout names are pub(crate) so the drill harness
// (`crate::backup::drill`) addresses the exact paths a Vault bundle uses
// instead of copies that could drift.
/// Bundle-relative directory holding one KV record artifact per key.
const VAULT_RECORD_ARTIFACT_DIR: &str = "vault/records";
pub(crate) const VAULT_RECORD_ARTIFACT_DIR: &str = "vault/records";
/// Bundle-relative suffix of a KV record artifact.
const VAULT_RECORD_ARTIFACT_SUFFIX: &str = ".json.enc";
pub(crate) const VAULT_RECORD_ARTIFACT_SUFFIX: &str = ".json.enc";
/// KV path segment (under the bundle's path prefix) of the restore commit
/// marker. The leading dot keeps it out of the key id space the backends
/// address; a bundle naming a key id equal to it is rejected.
+9 -2
View File
@@ -297,8 +297,15 @@ impl Default for LocalConfig {
/// Static single-key KMS backend configuration
///
/// Uses a pre-configured AES-256 key to derive data encryption keys via
/// HMAC-SHA256 + AES-256-GCM, matching the MinIO builtin/static KMS wire format.
/// The configured 32-byte key is used directly as the AES-256-GCM key that
/// wraps data encryption keys — there is no HMAC-SHA256 derivation step — and
/// each wrapped DEK is serialized as a RustFS `DataKeyEnvelope` JSON blob.
///
/// This mirrors the *concept* of MinIO's builtin/static single-key KMS, but is
/// not wire-compatible with it: MinIO wraps DEKs in a different (`{"aead": ...}`)
/// blob that this backend neither produces nor accepts, so KMS ciphertext
/// written by MinIO cannot be opened here. Reading MinIO-written SSE objects is
/// tracked separately in rustfs/backlog#1638.
#[derive(Clone, Default, Serialize, Deserialize)]
pub struct StaticConfig {
/// Key identifier (name) for the single configured key
+93 -1
View File
@@ -36,6 +36,11 @@ use tracing::{debug, info, warn};
/// How often the worker looks for expired pending deletions.
pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
/// Keys requested per `ListKeys` call while sweeping. The sweep follows
/// `truncated`/`next_marker` until the key set is exhausted, so this bounds the
/// working set of one call, not how many keys a sweep can reach.
const SWEEP_PAGE_SIZE: u32 = 100;
// ---------------------------------------------------------------------------
// Metrics
//
@@ -102,6 +107,17 @@ impl KeyCensus {
KeyStatus::PendingDeletion => self.pending_deletion += 1,
KeyStatus::Deleted => self.tombstones += 1,
KeyStatus::Active | KeyStatus::Disabled => {
// A missing rotation time means either "never rotated" or "the
// build that rotated it did not record when". Both fall back to
// creation, and the two are not worth separate series: for the
// first, creation *is* the correct baseline; for the second it
// is an upper bound on the true age, so the gauge can only
// overstate how overdue a key is, never hide an overdue one.
// The overstatement is self-healing — the next rotation stamps
// the record — and erring loud is the right direction for a
// rotation-readiness alert. Backends never fabricate a
// timestamp to close the gap, so nothing here can turn an
// unstamped key into a freshly rotated one.
let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
}
@@ -146,6 +162,17 @@ fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
/// referencing configuration lives (for example bucket encryption settings in
/// the server) and are injected via
/// [`crate::service_manager::KmsServiceManager::set_deletion_reference_checker`].
///
/// # Blocks only
///
/// This gate is one-directional and must stay that way: a checker may add a
/// reason to keep material, never a reason to destroy it. Nothing that
/// enumerates key usage — least of all a scan whose coverage depends on how
/// far a background sweep got — may ever be wired in as a condition that
/// releases a removal this gate is holding. One incomplete scan would then be
/// enough to destroy the only copy of a key that still has data behind it,
/// which is why [`crate::key_impact::KeyImpactReport`] exposes no clearance
/// and is consumed only where a refusal is being decided.
#[async_trait]
pub trait DeletionReferenceChecker: Send + Sync {
/// Identifiers of configuration still referencing `key_id` (bucket names,
@@ -240,7 +267,7 @@ impl DeletionWorker {
let mut marker: Option<String> = None;
let listed_everything = loop {
let request = ListKeysRequest {
limit: Some(100),
limit: Some(SWEEP_PAGE_SIZE),
marker: marker.clone(),
usage_filter: None,
status_filter: None,
@@ -270,6 +297,13 @@ impl DeletionWorker {
break true;
}
match response.next_marker {
// A backend that hands back the cursor it was just given cannot
// advance. Following it would re-list the same page forever, so
// the sweep stops and reports the key set as partially seen.
Some(ref next_marker) if marker.as_ref() == Some(next_marker) => {
warn!(marker = %next_marker, "KMS deletion sweep listing did not advance");
break false;
}
Some(next_marker) => marker = Some(next_marker),
// Truncated without a marker: the rest of the key set is out
// of reach, so the census is incomplete.
@@ -426,6 +460,41 @@ mod tests {
assert_eq!(report, SweepReport::default());
}
/// Schedule `count` keys for deletion and report their identifiers.
async fn schedule_keys(backend: &LocalKmsBackend, count: usize) -> Vec<String> {
let mut key_ids = Vec::with_capacity(count);
for index in 0..count {
let key_id = create_key(backend, &format!("expiring-{index:04}")).await;
schedule(backend, &key_id).await;
key_ids.push(key_id);
}
key_ids
}
/// A deployment with more keys than fit in one listing page must still have
/// every expired key destroyed: the sweep has to follow the cursor instead
/// of stopping at the first page.
#[tokio::test]
async fn sweep_removes_expired_keys_beyond_the_first_page() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let total = SWEEP_PAGE_SIZE as usize + 5;
let key_ids = schedule_keys(&backend, total).await;
let report = worker(backend.clone()).sweep(&after_window()).await;
assert_eq!(report.failed, 0);
assert_eq!(
report.removed.len(),
total,
"every expired key must be swept, not just the ones on the first page"
);
// The keys sorting last are the ones a first-page-only sweep leaves
// behind for good.
for key_id in key_ids.iter().rev().take(5) {
assert_key_gone(&backend, key_id).await;
}
}
#[tokio::test]
async fn cancelled_deletion_always_beats_the_sweep() {
let temp_dir = tempfile::tempdir().expect("temp dir");
@@ -757,4 +826,27 @@ mod tests {
}
}
}
/// The census counts the whole key set, not the first page of it. A sweep
/// that stops after one page would publish a gauge that understates every
/// large deployment by exactly the keys it never listed.
#[test]
fn lifecycle_gauges_count_keys_beyond_the_first_page() {
let total = SWEEP_PAGE_SIZE as usize + 5;
let (snapshot, ()) = record_metrics(|| {
Box::pin(async move {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
schedule_keys(&backend, total).await;
// Well inside the seven-day window: every key is observed as
// pending rather than removed.
let report = worker(backend.clone()).sweep(&Zoned::now()).await;
assert_eq!(report.skipped, total, "every scheduled key must be inspected");
assert!(report.removed.is_empty());
})
});
assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(total as f64));
}
}
+35
View File
@@ -144,6 +144,25 @@ pub enum KmsError {
"Baseline version lost for key {key_id}: master key version records exist (oldest {oldest_version}) but the key record carries no baseline version, so data keys written before versioned rotation can no longer be resolved to the master key version that wrapped them. A node older than versioned rotation rewrote the key record and dropped the field. Finish upgrading every node, restore baseline_version to {oldest_version} on the key record, then retry"
)]
BaselineVersionLost { key_id: String, oldest_version: u32 },
/// Configuration still points at the key, so its material must not be
/// destroyed. Distinct from the generic invalid-operation errors so that
/// callers can tell "this key is still wired into the deployment" apart
/// from a malformed request and act on the listed references.
#[error(
"Key {key_id} is still referenced by configuration and its material must not be destroyed: {}. Remove or repoint the listed configuration, then retry",
.references.join(", ")
)]
KeyStillReferenced { key_id: String, references: Vec<String> },
/// The only available way to rewrap this envelope would pull the plaintext
/// data key into the RustFS process. Refused rather than performed: the
/// point of a backend-side rewrap is that the data key stays inside the
/// backend, so silently falling back to unwrap-then-rewrap would hand back
/// a correct envelope while quietly dropping the property that justified
/// the operation.
#[error("Cannot rewrap a data key of key {key_id} without exposing its plaintext: {reason}")]
RewrapWouldExposePlaintext { key_id: String, reason: String },
}
impl KmsError {
@@ -252,6 +271,14 @@ impl KmsError {
Self::OperationCancelled { message: message.into() }
}
/// Create a still-referenced error
pub fn key_still_referenced<S: Into<String>>(key_id: S, references: Vec<String>) -> Self {
Self::KeyStillReferenced {
key_id: key_id.into(),
references,
}
}
/// Create a material missing error
pub fn material_missing<S: Into<String>>(key_id: S) -> Self {
Self::MaterialMissing { key_id: key_id.into() }
@@ -310,6 +337,14 @@ impl KmsError {
oldest_version,
}
}
/// Create a rewrap-would-expose-plaintext error
pub fn rewrap_would_expose_plaintext<S1: Into<String>, S2: Into<String>>(key_id: S1, reason: S2) -> Self {
Self::RewrapWouldExposePlaintext {
key_id: key_id.into(),
reason: reason.into(),
}
}
}
/// Convert from standard library errors
+290
View File
@@ -0,0 +1,290 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! What still points at a KMS key, as far as the server can prove it.
//!
//! # This report never says a key is unused
//!
//! There is deliberately no `in_use`, no `unreferenced`, and no
//! `safe_to_delete`: the report states which sources were consulted
//! ([`ReferenceCoverage`]), how exhaustively they could be read
//! ([`ReferenceCompleteness`]), and what was found. An empty
//! [`KeyImpactReport::references`] therefore means "nothing was found in the
//! scanned sources", never "nothing references this key" — object envelopes
//! written under a key are not, and cannot cheaply be, enumerated here.
//! Collapsing that distinction into a boolean would hand callers a green
//! checkmark backed by an unscanned half of the problem, so the distinction
//! lives in the type rather than in prose a UI can skip.
//!
//! # It may only ever add a reason to refuse
//!
//! A report is an input to refusing destruction, never to permitting it.
//! [`KeyImpactReport::blocks_destruction`] is phrased so its `false` case
//! carries no authority: it means this report found no reason to refuse, not
//! that any other gate has been satisfied. The deletion worker's own
//! [`crate::DeletionReferenceChecker`] stays the gate that decides whether
//! expired material is destroyed.
use serde::{Deserialize, Serialize};
/// A place where references to a key can live.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ReferenceScope {
/// A bucket's default server-side encryption configuration.
BucketDefaultEncryption,
/// The KMS service's configured default key.
ServiceDefaultKey,
/// Data-key envelopes stored on object versions.
ObjectEnvelopes,
/// Session envelopes of multipart uploads that have not completed yet.
InProgressMultipartUploads,
}
/// Why an entry appears in [`KeyImpactReport::references`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum KeyReferenceKind {
/// A bucket's default encryption configuration names the key.
BucketDefaultEncryption,
/// The key is the KMS service's configured default key.
ServiceDefaultKey,
/// A whole source in [`ReferenceCoverage::scanned`] could not be
/// enumerated. Reported as a reference, not as an absence: a source that
/// cannot be read may hold references, and destroying material on the
/// strength of an unanswered question is the one outcome that cannot be
/// undone.
UnreadableSource,
/// One resource inside an otherwise readable source could not be
/// inspected. Reported for the same reason as [`Self::UnreadableSource`].
UnreadableResource,
}
/// One thing that points at a key, or one place that could not be checked.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct KeyReference {
/// Machine-readable category.
pub kind: KeyReferenceKind,
/// Identifier of the referencing resource: a bucket name for bucket
/// configuration, the key id for the service default key, the affected
/// source or resource for the unreadable kinds. Never key material.
pub id: String,
/// Human-readable detail. Identifiers only; never secrets or material.
pub detail: String,
}
/// Which sources a report consulted, and which it did not look at at all.
///
/// `not_scanned` is mandatory rather than implied: a caller reading an empty
/// reference list has to be able to see, from the report alone, what was left
/// out of it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReferenceCoverage {
/// Sources this report enumerated.
pub scanned: Vec<ReferenceScope>,
/// Sources this report does not cover at all.
pub not_scanned: Vec<ReferenceScope>,
}
/// How far a report's reference list can be trusted.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ReferenceCompleteness {
/// Every reference within [`ReferenceCoverage::scanned`] was enumerated.
/// Reserved for configuration-layer facts, which are finite, cheap to
/// read, and therefore exhaustively decidable.
Exact,
/// The list holds what a snapshot happened to observe within the scanned
/// scopes. Absence of a reference is not evidence that none exists.
ObservedOnly,
/// At least one scanned source could not be read, so the list is not a
/// statement about the key at all.
Unavailable,
}
/// What the server can currently say about who points at a key.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct KeyImpactReport {
/// Key the report is about.
pub key_id: String,
/// How far [`Self::references`] can be trusted.
pub completeness: ReferenceCompleteness,
/// Which sources were consulted and which were not.
pub coverage: ReferenceCoverage,
/// References found, plus one entry per source that could not be read.
pub references: Vec<KeyReference>,
}
impl KeyImpactReport {
/// An empty report over the configuration layer: bucket default encryption
/// settings and the service default key.
///
/// Both are finite and cheap to enumerate, so a report that reads them all
/// stays [`ReferenceCompleteness::Exact`]; object-level scopes are
/// declared as not scanned and stay that way.
pub fn configuration_layer(key_id: impl Into<String>) -> Self {
Self {
key_id: key_id.into(),
completeness: ReferenceCompleteness::Exact,
coverage: ReferenceCoverage {
scanned: vec![ReferenceScope::BucketDefaultEncryption, ReferenceScope::ServiceDefaultKey],
not_scanned: vec![ReferenceScope::ObjectEnvelopes, ReferenceScope::InProgressMultipartUploads],
},
references: Vec::new(),
}
}
/// Record one reference.
///
/// An unreadable source or resource downgrades the report to
/// [`ReferenceCompleteness::Unavailable`] here rather than at the call
/// site, so no producer can report a partially read source as `Exact`.
pub fn push_reference(&mut self, reference: KeyReference) {
if matches!(reference.kind, KeyReferenceKind::UnreadableSource | KeyReferenceKind::UnreadableResource) {
self.completeness = ReferenceCompleteness::Unavailable;
}
self.references.push(reference);
}
/// Whether this report on its own is reason to refuse destroying the key's
/// material right now.
///
/// The two answers are not symmetric. `true` is a decision: something
/// points at the key, or a source that might could not be read. `false`
/// only means this report contributes no objection — it is never a
/// clearance, and must not be used to skip, shorten, or satisfy any other
/// check on the deletion path.
pub fn blocks_destruction(&self) -> bool {
// The completeness test is redundant while every producer records an
// unreadable source as a reference, and stays here so that a future
// producer which forgets to cannot turn an unanswered question into a
// silent clearance.
!self.references.is_empty() || matches!(self.completeness, ReferenceCompleteness::Unavailable)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn bucket_reference(bucket: &str) -> KeyReference {
KeyReference {
kind: KeyReferenceKind::BucketDefaultEncryption,
id: bucket.to_string(),
detail: format!("bucket {bucket} encrypts new objects with this key by default"),
}
}
#[test]
fn a_fresh_configuration_report_declares_the_object_layer_unscanned() {
let report = KeyImpactReport::configuration_layer("kms-key-1");
assert_eq!(report.completeness, ReferenceCompleteness::Exact);
assert!(report.references.is_empty());
assert_eq!(
report.coverage.not_scanned,
vec![ReferenceScope::ObjectEnvelopes, ReferenceScope::InProgressMultipartUploads],
"an empty reference list is only readable next to what was left unscanned"
);
assert!(!report.blocks_destruction());
}
#[test]
fn any_reference_blocks_destruction() {
let mut report = KeyImpactReport::configuration_layer("kms-key-1");
report.push_reference(bucket_reference("sse-bucket"));
assert!(report.blocks_destruction());
assert_eq!(
report.completeness,
ReferenceCompleteness::Exact,
"a fully readable configuration layer stays exact even when it holds references"
);
}
#[test]
fn an_unreadable_source_is_never_reported_as_an_absence() {
for kind in [KeyReferenceKind::UnreadableSource, KeyReferenceKind::UnreadableResource] {
let mut report = KeyImpactReport::configuration_layer("kms-key-1");
report.push_reference(KeyReference {
kind,
id: "bucket-default-encryption".to_string(),
detail: "configuration could not be read".to_string(),
});
assert_eq!(
report.completeness,
ReferenceCompleteness::Unavailable,
"{kind:?} must downgrade completeness"
);
assert!(report.blocks_destruction(), "{kind:?} must block destruction");
}
}
#[test]
fn unavailable_completeness_blocks_even_without_references() {
// Guards the fail-closed fallback for a producer that marks a report
// unavailable without recording why.
let report = KeyImpactReport {
completeness: ReferenceCompleteness::Unavailable,
..KeyImpactReport::configuration_layer("kms-key-1")
};
assert!(report.references.is_empty());
assert!(report.blocks_destruction());
}
#[test]
fn report_round_trips_through_json() {
let mut report = KeyImpactReport::configuration_layer("kms-key-1");
report.push_reference(bucket_reference("sse-bucket"));
report.push_reference(KeyReference {
kind: KeyReferenceKind::ServiceDefaultKey,
id: "kms-key-1".to_string(),
detail: "configured as the KMS service default key".to_string(),
});
let json = serde_json::to_string(&report).expect("serialization should succeed");
let decoded: KeyImpactReport = serde_json::from_str(&json).expect("deserialization should succeed");
assert_eq!(decoded, report);
}
/// The wire shape is the contract callers build UIs on: a boolean that
/// reads as "this key is unused" must never appear in it, however the
/// report is populated.
#[test]
fn the_wire_shape_asserts_nothing_about_absence_of_use() {
let mut referenced = KeyImpactReport::configuration_layer("kms-key-1");
referenced.push_reference(bucket_reference("sse-bucket"));
for report in [KeyImpactReport::configuration_layer("kms-key-1"), referenced] {
let json = serde_json::to_value(&report).expect("serialization should succeed");
let mut fields: Vec<&str> = json
.as_object()
.expect("report is a JSON object")
.keys()
.map(String::as_str)
.collect();
fields.sort_unstable();
assert_eq!(fields, vec!["completeness", "coverage", "key_id", "references"]);
for forbidden in ["in_use", "unused", "unreferenced", "safe_to_delete", "deletable"] {
assert!(!json.to_string().contains(forbidden), "report must not carry a `{forbidden}` claim");
}
}
}
}
+5 -3
View File
@@ -73,6 +73,7 @@ pub mod config;
pub mod deletion_worker;
mod encryption;
mod error;
pub mod key_impact;
pub mod manager;
mod policy;
pub mod probe;
@@ -83,9 +84,9 @@ pub mod types;
// Re-export public API
pub use api_types::{
CacheSummary, ConfigureKmsRequest, ConfigureKmsResponse, ConfigureLocalKmsRequest, ConfigureStaticKmsRequest,
ConfigureVaultKmsRequest, ConfigureVaultTransitKmsRequest, KmsConfigSummary, KmsStatusResponse, StartKmsRequest,
StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
CacheSummary, ConfigureAwsKmsRequest, ConfigureKmsRequest, ConfigureKmsResponse, ConfigureLocalKmsRequest,
ConfigureStaticKmsRequest, ConfigureVaultKmsRequest, ConfigureVaultTransitKmsRequest, KmsConfigSummary, KmsStatusResponse,
StartKmsRequest, StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
};
pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
@@ -94,6 +95,7 @@ pub use config::*;
pub use deletion_worker::DeletionReferenceChecker;
pub use encryption::is_data_key_envelope;
pub use error::{KmsError, KmsUnavailableError, Result};
pub use key_impact::{KeyImpactReport, KeyReference, KeyReferenceKind, ReferenceCompleteness, ReferenceCoverage, ReferenceScope};
pub use manager::KmsManager;
pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
pub use service::{DataKey, ObjectEncryptionService};
+264 -2
View File
@@ -18,12 +18,15 @@ use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
use crate::backends::KmsBackend;
use crate::cache::{KmsCache, KmsCacheStats};
use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, KmsConfig};
use crate::deletion_worker::DeletionReferenceChecker;
use crate::error::{KmsError, Result};
use crate::types::{
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse,
DEFAULT_PENDING_DELETION_WINDOW_DAYS, DecryptRequest, DecryptResponse, DeleteKeyRequest, DeleteKeyResponse,
DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyRequest, GenerateDataKeyResponse,
ListKeysRequest, ListKeysResponse, MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext,
DescribeDataKeyWrappingRequest, DescribeDataKeyWrappingResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest,
EncryptResponse, GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse,
MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext, RewrapDataKeyRequest,
RewrapDataKeyResponse,
};
use std::collections::HashMap;
use std::sync::Arc;
@@ -41,6 +44,7 @@ pub struct KmsManager {
backend_kind: &'static str,
audit_sink: Option<Arc<dyn KmsAuditSink>>,
allow_immediate_deletion: bool,
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
}
impl KmsManager {
@@ -60,9 +64,21 @@ impl KmsManager {
backend_kind: config.backend.as_str(),
audit_sink: None,
allow_immediate_deletion: config.allow_immediate_deletion,
reference_checker: None,
}
}
/// Consult `checker` before immediate deletion destroys key material.
///
/// This is the same checker the deletion worker consults before it removes
/// an expired key; installing it here extends that gate to the one
/// deletion path that never reaches the worker. It can only add a refusal:
/// without a checker the manager behaves exactly as before.
pub fn with_deletion_reference_checker(mut self, checker: Option<Arc<dyn DeletionReferenceChecker>>) -> Self {
self.reference_checker = checker;
self
}
/// Send an audit record for every management operation to `sink`.
///
/// Without a sink the manager builds no records at all, so a deployment
@@ -153,6 +169,29 @@ impl KmsManager {
self.backend.generate_data_key(request).await
}
/// Re-wrap an existing data key envelope onto the master key's current
/// version, leaving the data key — and therefore every object body it
/// protects — untouched.
///
/// Backends without retained version history reject this with
/// [`KmsError::UnsupportedCapability`]; check
/// [`Self::backend_capabilities`] before offering it.
pub async fn rewrap_data_key(&self, request: RewrapDataKeyRequest) -> Result<RewrapDataKeyResponse> {
self.backend.rewrap_data_key(request).await
}
/// Report which master key version wraps an existing data key envelope.
///
/// The read-only side of [`Self::rewrap_data_key`], and the supported way to
/// ask that question: where the version is recorded differs per backend, so
/// callers must not inspect envelopes themselves.
pub async fn describe_data_key_wrapping(
&self,
request: DescribeDataKeyWrappingRequest,
) -> Result<DescribeDataKeyWrappingResponse> {
self.backend.describe_data_key_wrapping(request).await
}
/// Describe a key
///
/// Audited as an internal operation; callers serving an authenticated
@@ -262,6 +301,9 @@ impl KmsManager {
/// defensive assertion for callers that hold a backend handle directly.
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
self.check_deletion_request(&request)?;
if request.force_immediate.unwrap_or(false) {
self.refuse_referenced_immediate_deletion(&request.key_id).await?;
}
let response = self.backend.delete_key(request).await?;
@@ -312,6 +354,41 @@ impl KmsManager {
Ok(())
}
/// Refuse an immediate deletion while configuration still points at the
/// key.
///
/// A scheduled deletion is re-checked against these same references by the
/// deletion worker before it destroys anything, and stays cancellable
/// until then. Immediate deletion has neither property: it destroys
/// material on the spot and never reaches the worker, so the check has to
/// happen here or not at all.
///
/// Only ever a refusal. An empty reference set is not a clearance — it
/// means the sources consulted here raised no objection, while the caller
/// still had to pass the server-side opt-in and the key-id confirmation to
/// get this far. With no checker installed the manager has no
/// configuration source to consult and behaves as it did before, matching
/// the deletion worker, which also skips a checker it was not given.
async fn refuse_referenced_immediate_deletion(&self, key_id: &str) -> Result<()> {
let mut references = Vec::new();
if self.default_key_id.as_deref() == Some(key_id) {
references.push("kms-service-default-key".to_string());
}
if let Some(checker) = &self.reference_checker {
references.extend(checker.references(key_id).await);
}
if references.is_empty() {
return Ok(());
}
warn!(
key_id,
?references,
"immediate KMS key deletion refused; configuration still references the key"
);
Err(KmsError::key_still_referenced(key_id, references))
}
/// Cancel key deletion
///
/// Audited as an internal operation; callers serving an authenticated
@@ -1256,6 +1333,191 @@ mod tests {
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
}
/// Reference checker whose answer is fixed, standing in for the server's
/// bucket-configuration gate.
struct StaticReferences(Vec<String>);
#[async_trait]
impl DeletionReferenceChecker for StaticReferences {
async fn references(&self, _key_id: &str) -> Vec<String> {
self.0.clone()
}
}
#[tokio::test]
async fn immediate_deletion_is_refused_while_configuration_references_the_key() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, true)
.await
.with_deletion_reference_checker(Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))));
let key_id = create_named_key(&manager, "referenced-force-delete").await;
let probe = data_key_probe(&manager, &key_id).await;
// Server opt-in granted and the confirmation exact: the reference is
// the only thing left to refuse this.
let error = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
..Default::default()
})
.await
.expect_err("immediate deletion must be refused while configuration references the key");
match error {
KmsError::KeyStillReferenced {
key_id: refused,
references,
} => {
assert_eq!(refused, key_id);
assert_eq!(references, vec!["bucket:sse-bucket".to_string()], "the caller must learn what refused it");
}
other => panic!("expected KeyStillReferenced, got {other:?}"),
}
assert_key_material_intact(&manager, &key_id, &probe).await;
}
#[tokio::test]
async fn immediate_deletion_of_the_service_default_key_is_refused() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
config.allow_immediate_deletion = true;
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
let key_id = KmsManager::new(backend.clone(), config.clone())
.create_key(CreateKeyRequest {
key_name: Some("default-key-force-delete".to_string()),
key_usage: KeyUsage::EncryptDecrypt,
..Default::default()
})
.await
.expect("Failed to create key")
.key_id;
config.default_key_id = Some(key_id.clone());
let manager = KmsManager::new(backend, config);
let probe = data_key_probe(&manager, &key_id).await;
let error = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
..Default::default()
})
.await
.expect_err("the service default key must not be destroyed out from under the deployment");
assert!(
matches!(error, KmsError::KeyStillReferenced { .. }),
"expected KeyStillReferenced, got {error:?}"
);
assert_key_material_intact(&manager, &key_id, &probe).await;
}
/// A checker that reports nothing must not become a shortcut around the
/// gates that were already there: it is not a clearance, only the absence
/// of one more objection.
#[tokio::test]
async fn an_empty_reference_set_grants_no_deletion_on_its_own() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, false)
.await
.with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
let key_id = create_named_key(&manager, "unreferenced-force-delete").await;
let probe = data_key_probe(&manager, &key_id).await;
// Server opt-in withheld, then confirmation missing: both still refuse.
let error = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
..Default::default()
})
.await
.expect_err("an unreferenced key still needs the server-side opt-in");
assert!(
matches!(error, KmsError::InvalidOperation { .. }),
"expected InvalidOperation, got {error:?}"
);
let allowed = deletion_manager(&temp_dir, true)
.await
.with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
let error = allowed
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
..Default::default()
})
.await
.expect_err("an unreferenced key still needs the key-id confirmation");
assert!(
matches!(error, KmsError::InvalidOperation { .. }),
"expected InvalidOperation, got {error:?}"
);
assert_key_material_intact(&manager, &key_id, &probe).await;
}
/// The refusal is the only thing the checker adds: a fully authorized
/// immediate deletion of a key nothing points at still goes through.
#[tokio::test]
async fn immediate_deletion_still_succeeds_when_nothing_references_the_key() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, true)
.await
.with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
let key_id = create_named_key(&manager, "unreferenced-confirmed-force-delete").await;
manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
..Default::default()
})
.await
.expect("a confirmed immediate deletion must still be allowed when nothing references the key");
let error = manager
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
.await
.expect_err("an immediately deleted key must be gone");
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
}
/// Scheduling stays a schedule: it destroys nothing, stays cancellable,
/// and is re-checked against the same references by the deletion worker
/// before any material goes away. Turning references into an up-front
/// refusal here would let one unreadable bucket block routine operations.
#[tokio::test]
async fn scheduled_deletion_is_unaffected_by_references() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, false)
.await
.with_deletion_reference_checker(Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))));
let key_id = create_named_key(&manager, "referenced-schedule").await;
manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
..Default::default()
})
.await
.expect("a scheduled deletion must still be accepted while configuration references the key");
let state = manager
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
.await
.expect("a scheduled key must still be describable")
.key_metadata
.key_state;
assert_eq!(state, KeyState::PendingDeletion);
}
#[tokio::test]
async fn pending_window_outside_the_supported_range_is_refused() {
let temp_dir = tempdir().expect("Failed to create temp dir");
+43 -1
View File
@@ -634,7 +634,13 @@ impl KmsServiceManager {
};
// Create KMS manager
let mut kms_manager = KmsManager::new(backend, config.clone());
//
// The deletion reference checker is handed to the manager as well as to
// the worker: immediate deletion destroys material without ever
// reaching the worker, so that path has to consult the same gate
// itself.
let mut kms_manager =
KmsManager::new(backend, config.clone()).with_deletion_reference_checker(self.deletion_reference_checker());
if let Some(sink) = self.audit_sink() {
kms_manager = kms_manager.with_audit_sink(sink);
}
@@ -749,6 +755,42 @@ mod tests {
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
}
/// End-to-end wiring check for the AWS backend: an admin configure request
/// must select it, build a real client, and pass the startup health check.
///
/// `RUSTFS_KMS_AWS_REGION` names the region; the credential chain supplies
/// the rest. No key is created, so this test is not billable on its own.
#[tokio::test]
#[ignore] // Requires real AWS credentials
async fn aws_backend_configure_and_start_end_to_end() {
let region = std::env::var("RUSTFS_KMS_AWS_REGION").expect("RUSTFS_KMS_AWS_REGION must name the test region");
let config = crate::api_types::ConfigureAwsKmsRequest {
region,
endpoint_url: None,
default_key_id: std::env::var("RUSTFS_KMS_DEFAULT_KEY_ID").ok(),
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: None,
}
.to_kms_config();
let manager = KmsServiceManager::new();
manager.configure(config).await.expect("configure the AWS backend");
manager.start().await.expect("start the AWS backend");
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
let capabilities = manager
.get_manager()
.await
.expect("a running service exposes its manager")
.backend_capabilities();
assert!(capabilities.schedule_deletion);
assert!(!capabilities.physical_delete);
}
#[tokio::test]
async fn configure_rejects_insecure_development_defaults_before_state_update() {
let manager = KmsServiceManager::new();
+77
View File
@@ -959,3 +959,80 @@ impl Drop for DataKeyInfo {
self.clear_plaintext();
}
}
/// Request to rewrap an existing data key envelope under the current version of
/// the master key that already wraps it.
///
/// Rewrap changes only the wrapping. The data key inside is never replaced, so
/// object ciphertext, IV derivation and ETags are untouched — which is what
/// makes it possible to retire an old master key version without rewriting a
/// single object body.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RewrapDataKeyRequest {
/// The existing wrapped data key, exactly as it was persisted
pub ciphertext: Vec<u8>,
/// Encryption context the envelope was created with; the backend rejects
/// the request when the envelope's own context is not reproduced here, so a
/// caller cannot rewrap an envelope it could not have decrypted
pub encryption_context: HashMap<String, String>,
}
/// Request to report where an existing data key envelope sits relative to its
/// master key's current version.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DescribeDataKeyWrappingRequest {
/// The existing wrapped data key, exactly as it was persisted
pub ciphertext: Vec<u8>,
/// Encryption context the envelope was created with, checked exactly as
/// [`RewrapDataKeyRequest`] checks it
pub encryption_context: HashMap<String, String>,
}
/// Where an existing wrapped data key sits relative to its master key's current
/// version.
///
/// This is the only supported way to ask that question. The answer lives in a
/// different place for every backend that rotates — a JSON field on the envelope
/// for Vault KV2, the `vault:vN:` prefix of the ciphertext for Vault Transit,
/// nowhere at all for backends that do not rotate — and a caller that reached
/// into the envelope itself would be re-implementing that table, on the wrong
/// side of the KMS boundary, once per call site.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DescribeDataKeyWrappingResponse {
/// Master key wrapping the data key
pub key_id: String,
/// Version that wrapped the data key, when the backend can name it
pub key_version: Option<u32>,
/// The master key's current version, when the backend can name it
pub current_key_version: Option<u32>,
/// Whether the envelope is already wrapped by the current version, which is
/// exactly the condition under which a rewrap would be a no-op.
///
/// Callers must read this rather than compare the two version fields. A
/// backend may leave either version unset while still knowing the answer,
/// and the two must never disagree: this flag is what a retirement scan
/// counts, and a rewrap sweep that rewrote envelopes the scan already
/// considered done would never converge.
pub is_current: bool,
}
/// Result of [`RewrapDataKeyRequest`].
///
/// The plaintext data key is deliberately absent: rewrap exists so that the
/// data key can be re-wrapped without the caller ever holding it.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RewrapDataKeyResponse {
/// The wrapped data key to persist in place of the request's ciphertext.
/// Byte-identical to the request when `rewrapped` is false.
pub ciphertext: Vec<u8>,
/// Master key that wraps the data key; unchanged by a rewrap
pub key_id: String,
/// Master key version that wrapped the input, when the backend can name it
pub source_key_version: Option<u32>,
/// Master key version that wraps `ciphertext`, when the backend can name it
pub destination_key_version: Option<u32>,
/// Whether the wrapping actually changed. False means the input was already
/// wrapped by the current version and callers must not persist anything —
/// re-running a rewrap sweep has to converge without further writes.
pub rewrapped: bool,
}
@@ -17,8 +17,10 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD, BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD,
BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD, BUCKET_REPL_MRF_PENDING_COUNT_MD,
@@ -37,6 +39,7 @@ use std::borrow::Cow;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11;
const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4;
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationTargetStats {
@@ -99,6 +102,16 @@ pub(crate) struct BucketReplicationBacklogStats {
pub(crate) mrf_missed_count: u64,
pub(crate) mrf_flush_failures: u64,
pub(crate) mrf_last_flush_duration_millis: u64,
pub(crate) target_backlogs: Vec<BucketReplicationTargetBacklogStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationTargetBacklogStats {
pub(crate) target_arn: String,
pub(crate) current_backlog_count: u64,
pub(crate) current_backlog_bytes: u64,
pub(crate) durable_mrf_backlog_count: u64,
pub(crate) durable_mrf_backlog_bytes: u64,
}
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
@@ -290,7 +303,14 @@ pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicat
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
let metric_count = stats
.iter()
.map(|stat| {
BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET
+ stat.target_backlogs.len() * BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET
})
.sum();
let mut metrics = Vec::with_capacity(metric_count);
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
@@ -345,6 +365,42 @@ pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicat
)
.with_label(BUCKET_L, bucket_label),
);
for target in &stat.target_backlogs {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD,
target.current_backlog_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
target.current_backlog_bytes as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD,
target.durable_mrf_backlog_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
target.durable_mrf_backlog_bytes as f64,
)
.with_label(BUCKET_L, bucket_label)
.with_label(TARGET_ARN_L, target_label),
);
}
}
metrics
@@ -469,10 +525,17 @@ mod tests {
mrf_missed_count: 4,
mrf_flush_failures: 5,
mrf_last_flush_duration_millis: 6,
target_backlogs: vec![BucketReplicationTargetBacklogStats {
target_arn: "arn:rustfs:replication:target-a".to_string(),
current_backlog_count: 3,
current_backlog_bytes: 4096,
durable_mrf_backlog_count: 2,
durable_mrf_backlog_bytes: 2048,
}],
}];
let metrics = collect_bucket_replication_backlog_metrics(&stats);
assert_eq!(metrics.len(), 11);
assert_eq!(metrics.len(), 15);
let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
@@ -509,6 +572,50 @@ mod tests {
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let target_count_name = BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_count_name
&& metric.value == 3.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let target_bytes_name = BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_bytes_name
&& metric.value == 4096.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let durable_target_count_name = BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_target_count_name
&& metric.value == 2.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let durable_target_bytes_name = BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_target_bytes_name
&& metric.value == 2048.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let pending_count_name = BUCKET_REPL_MRF_PENDING_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == pending_count_name
@@ -574,6 +681,7 @@ mod tests {
mrf_missed_count: 4,
mrf_flush_failures: 5,
mrf_last_flush_duration_millis: 6,
target_backlogs: Vec::new(),
}];
let metrics = collect_bucket_replication_backlog_metrics(&stats);
+3 -1
View File
@@ -42,7 +42,9 @@ pub mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
pub(crate) use bucket_replication::{
BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics,
};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
+156 -2
View File
@@ -80,8 +80,10 @@ use crate::metrics::runtime_sources::bucket_monitor_available;
use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE_LENGTH_MD, AUDIT_TOTAL_MESSAGES_MD};
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD,
BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD, BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD,
BUCKET_REPL_MRF_PENDING_COUNT_MD, TARGET_ARN_L,
};
@@ -633,6 +635,17 @@ fn repl_backlog_live_keys(stats: &[BucketReplicationBacklogStats]) -> HashSet<Bu
stats.iter().map(|s| s.bucket.clone()).collect()
}
fn repl_backlog_target_live_keys(stats: &[BucketReplicationBacklogStats]) -> HashSet<ReplBwKey> {
stats
.iter()
.flat_map(|stat| {
stat.target_backlogs
.iter()
.map(|target| (stat.bucket.clone(), target.target_arn.clone()))
})
.collect()
}
fn update_series_zero_tombstones<T: Clone + Eq + std::hash::Hash>(
has_seen_valid_snapshot: &mut bool,
prev_live_keys: &mut HashSet<T>,
@@ -1060,6 +1073,40 @@ fn collect_repl_backlog_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketK
collect_bucket_replication_backlog_metrics(&stats)
}
fn collect_repl_backlog_target_zero_tombstone_metrics(zero_tombstones: &HashMap<ReplBwKey, u8>) -> Vec<PrometheusMetric> {
if zero_tombstones.is_empty() {
return Vec::new();
}
let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4);
for (bucket, target_arn) in zero_tombstones.keys() {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone());
let target_arn_label: Cow<'static, str> = Cow::Owned(target_arn.clone());
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, 0.0)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD, 0.0)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, 0.0)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD, 0.0)
.with_label(BUCKET_L, bucket_label)
.with_label(TARGET_ARN_L, target_arn_label),
);
}
zero_metrics
}
fn retire_repl_bw_metric_series(bucket: &str, target_arn: &str) -> usize {
let labels = [
(BUCKET_L, Cow::Owned(bucket.to_string())),
@@ -1089,6 +1136,17 @@ fn retire_repl_backlog_metric_series(bucket: &str) -> usize {
.sum()
}
fn retire_repl_backlog_target_metric_series(bucket: &str, target_arn: &str) -> usize {
let labels = [
(BUCKET_L, Cow::Owned(bucket.to_string())),
(TARGET_ARN_L, Cow::Owned(target_arn.to_string())),
];
retire_metric_series(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(), &labels)
}
fn expire_repl_bw_zero_tombstones(monitor_available: bool, zero_tombstones: &mut HashMap<ReplBwKey, u8>) -> Vec<ReplBwKey> {
if !monitor_available {
return Vec::new();
@@ -1105,6 +1163,17 @@ fn expire_repl_backlog_zero_tombstones(monitor_available: bool, zero_tombstones:
expire_series_zero_tombstones(zero_tombstones)
}
fn expire_repl_backlog_target_zero_tombstones(
target_metrics_available: bool,
zero_tombstones: &mut HashMap<ReplBwKey, u8>,
) -> Vec<ReplBwKey> {
if !target_metrics_available {
return Vec::new();
}
expire_series_zero_tombstones(zero_tombstones)
}
/// Initialize all metrics collectors.
///
/// This function spawns background tasks that periodically collect metrics
@@ -1399,6 +1468,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut prev_backlog_live_keys: HashSet<BucketKey> = HashSet::new();
let mut backlog_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
let mut has_seen_valid_backlog_snapshot = false;
let mut prev_backlog_target_live_keys: HashSet<ReplBwKey> = HashSet::new();
let mut backlog_target_zero_tombstones: HashMap<ReplBwKey, u8> = HashMap::new();
let mut has_seen_valid_backlog_target_snapshot = false;
loop {
tokio::select! {
_ = interval.tick() => {
@@ -1429,6 +1501,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
let durable_mrf_available = bucket_replication_backlog.iter().any(|stat| stat.durable_mrf_available);
let backlog_target_metrics_available =
durable_mrf_available || monitor_available || !bucket_replication_backlog.is_empty();
update_repl_backlog_zero_tombstones(
monitor_available,
&mut has_seen_valid_backlog_snapshot,
@@ -1437,9 +1512,19 @@ pub fn init_metrics_runtime(token: CancellationToken) {
repl_backlog_live_keys(&bucket_replication_backlog),
repl_bw_zero_tombstone_cycles,
);
if backlog_target_metrics_available {
update_series_zero_tombstones(
&mut has_seen_valid_backlog_target_snapshot,
&mut prev_backlog_target_live_keys,
&mut backlog_target_zero_tombstones,
repl_backlog_target_live_keys(&bucket_replication_backlog),
repl_bw_zero_tombstone_cycles,
);
}
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
metrics.extend(collect_repl_backlog_zero_tombstone_metrics(&backlog_zero_tombstones));
metrics.extend(collect_repl_backlog_target_zero_tombstone_metrics(&backlog_target_zero_tombstones));
let replication = collect_replication_stats().await;
metrics.extend(collect_replication_metrics(&replication));
report_metrics(&metrics);
@@ -1451,6 +1536,12 @@ pub fn init_metrics_runtime(token: CancellationToken) {
for bucket in expire_repl_backlog_zero_tombstones(monitor_available, &mut backlog_zero_tombstones) {
let _ = retire_repl_backlog_metric_series(&bucket);
}
for (bucket, target_arn) in expire_repl_backlog_target_zero_tombstones(
backlog_target_metrics_available,
&mut backlog_target_zero_tombstones,
) {
let _ = retire_repl_backlog_target_metric_series(&bucket, &target_arn);
}
},
).await;
}
@@ -2259,6 +2350,69 @@ mod tests {
assert_eq!(prev_live_keys, bucket_keys(&["photos"]));
}
#[test]
fn repl_backlog_target_tombstones_zero_removed_targets_then_expire() {
let mut has_seen_valid_snapshot = false;
let mut prev_live_keys = HashSet::new();
let mut zero_tombstones = HashMap::new();
let key = repl_bw_key("photos", "arn:rustfs:replication:target-a");
update_series_zero_tombstones(
&mut has_seen_valid_snapshot,
&mut prev_live_keys,
&mut zero_tombstones,
repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]),
2,
);
assert!(has_seen_valid_snapshot);
assert_eq!(prev_live_keys, repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]));
assert!(zero_tombstones.is_empty());
update_series_zero_tombstones(&mut has_seen_valid_snapshot, &mut prev_live_keys, &mut zero_tombstones, HashSet::new(), 2);
assert_eq!(zero_tombstones.get(&key), Some(&2));
let metrics = collect_repl_backlog_target_zero_tombstone_metrics(&zero_tombstones);
assert_eq!(metrics.len(), 4);
let expected_names = HashSet::from([
BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(),
BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(),
]);
let mut actual_names = HashSet::new();
for metric in metrics {
actual_names.insert(metric.name.to_string());
assert_eq!(metric.value, 0.0);
let labels = metric
.labels
.into_iter()
.map(|(key, value)| (key, value.to_string()))
.collect::<HashMap<_, _>>();
assert_eq!(labels.get(BUCKET_L).map(String::as_str), Some("photos"));
assert_eq!(labels.get(TARGET_ARN_L).map(String::as_str), Some("arn:rustfs:replication:target-a"));
}
assert_eq!(actual_names, expected_names);
let expired = expire_repl_backlog_target_zero_tombstones(true, &mut zero_tombstones);
assert!(expired.is_empty());
assert_eq!(zero_tombstones.get(&key), Some(&1));
let expired = expire_repl_backlog_target_zero_tombstones(true, &mut zero_tombstones);
assert_eq!(expired, vec![key]);
assert!(zero_tombstones.is_empty());
}
#[test]
fn repl_backlog_target_tombstones_do_not_advance_when_target_metrics_unavailable() {
let key = repl_bw_key("videos", "arn:rustfs:replication:target-b");
let mut zero_tombstones = HashMap::from([(key.clone(), 1)]);
let expired = expire_repl_backlog_target_zero_tombstones(false, &mut zero_tombstones);
assert!(expired.is_empty());
assert_eq!(zero_tombstones.get(&key), Some(&1));
}
#[test]
fn bucket_tombstones_zero_removed_buckets_then_expire() {
let mut has_seen_valid_snapshot = false;
@@ -35,9 +35,13 @@ const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
const CURRENT_TARGET_BACKLOG_COUNT: &str = "current_target_backlog_count";
const CURRENT_TARGET_BACKLOG_BYTES: &str = "current_target_backlog_bytes";
const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
const DURABLE_MRF_TARGET_BACKLOG_COUNT: &str = "durable_mrf_target_backlog_count";
const DURABLE_MRF_TARGET_BACKLOG_BYTES: &str = "durable_mrf_target_backlog_bytes";
const MRF_PENDING_COUNT: &str = "mrf_pending_count";
const MRF_PENDING_BYTES: &str = "mrf_pending_bytes";
const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
@@ -108,6 +112,24 @@ pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = La
)
});
pub static BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(CURRENT_TARGET_BACKLOG_COUNT),
"Current number of target-scoped operations admitted to in-memory replication worker queues for a bucket and target ARN",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(CURRENT_TARGET_BACKLOG_BYTES),
"Current bytes in target-scoped operations admitted to in-memory replication worker queues for a bucket and target ARN",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_AVAILABLE),
@@ -135,6 +157,24 @@ pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor>
)
});
pub static BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_TARGET_BACKLOG_COUNT),
"Current number of target-scoped operations in the durable MRF backlog file for a bucket and target ARN on this node",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_TARGET_BACKLOG_BYTES),
"Current bytes in target-scoped operations in the durable MRF backlog file for a bucket and target ARN on this node",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_PENDING_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(MRF_PENDING_COUNT),
+16 -5
View File
@@ -21,11 +21,11 @@
//! and convert them to the Stats structs used by collectors.
use crate::metrics::collectors::{
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
ScannerStats,
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetBacklogStats,
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats,
HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats,
ResourceStats, ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
@@ -212,6 +212,17 @@ async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>,
mrf_missed_count: stats.mrf_missed_count,
mrf_flush_failures: stats.mrf_flush_failures,
mrf_last_flush_duration_millis: stats.mrf_last_flush_duration_millis,
target_backlogs: stats
.target_backlogs
.iter()
.map(|target| BucketReplicationTargetBacklogStats {
target_arn: target.target_arn.clone(),
current_backlog_count: target.current_backlog_count,
current_backlog_bytes: target.current_backlog_bytes,
durable_mrf_backlog_count: target.durable_mrf_backlog_count,
durable_mrf_backlog_bytes: target.durable_mrf_backlog_bytes,
})
.collect(),
});
detail_stats.push(bucket_replication_detail_from_snapshot(stats));
}
+161 -50
View File
@@ -18,7 +18,8 @@ use std::time::Duration;
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
use rustfs_ecstore::api::bucket::replication::{
DurableMrfBucketBacklog, MrfBucketBacklogObservability, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog,
durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot, get_global_replication_stats,
mrf_backlog_observability_snapshot,
};
pub(crate) use rustfs_ecstore::api::capacity::{
@@ -44,6 +45,15 @@ pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
pub(crate) latency_ms: f64,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ObsBucketReplicationTargetBacklogSnapshot {
pub(crate) target_arn: String,
pub(crate) current_backlog_count: u64,
pub(crate) current_backlog_bytes: u64,
pub(crate) durable_mrf_backlog_count: u64,
pub(crate) durable_mrf_backlog_bytes: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ObsBucketReplicationStatsSnapshot {
pub(crate) bucket: String,
@@ -84,6 +94,7 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
pub(crate) mrf_flush_failures: u64,
pub(crate) mrf_last_flush_duration_millis: u64,
pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
pub(crate) target_backlogs: Vec<ObsBucketReplicationTargetBacklogSnapshot>,
}
#[derive(Debug, Clone, Default, PartialEq)]
@@ -122,6 +133,15 @@ struct ObsBucketReplicationProxySnapshot {
proxied_delete_tagging_requests_failures: u64,
}
#[derive(Debug, Clone, Default, PartialEq)]
struct ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: bool,
durable_bucket: DurableMrfBucketBacklog,
runtime_targets: Vec<RuntimeReplicationTargetBacklog>,
durable_targets: Vec<DurableMrfTargetBacklog>,
mrf_observability: MrfBucketBacklogObservability,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub(crate) struct ObsReplicationSiteStatsSnapshot {
pub(crate) average_active_workers: f64,
@@ -157,10 +177,40 @@ fn bucket_replication_stats_snapshot_from_parts(
bucket: String,
runtime: ObsBucketReplicationRuntimeSnapshot,
proxy: ObsBucketReplicationProxySnapshot,
durable_mrf_available: bool,
durable_bucket: DurableMrfBucketBacklog,
mrf_observability: MrfBucketBacklogObservability,
backlog: ObsBucketReplicationBacklogSnapshot,
) -> ObsBucketReplicationStatsSnapshot {
let mut target_backlogs = HashMap::with_capacity(backlog.runtime_targets.len().saturating_add(backlog.durable_targets.len()));
for target in backlog.runtime_targets {
let entry =
target_backlogs
.entry(target.target_arn.clone())
.or_insert_with(|| ObsBucketReplicationTargetBacklogSnapshot {
target_arn: target.target_arn,
current_backlog_count: 0,
current_backlog_bytes: 0,
durable_mrf_backlog_count: 0,
durable_mrf_backlog_bytes: 0,
});
entry.current_backlog_count = target.count;
entry.current_backlog_bytes = target.bytes;
}
for target in backlog.durable_targets {
let entry =
target_backlogs
.entry(target.target_arn.clone())
.or_insert_with(|| ObsBucketReplicationTargetBacklogSnapshot {
target_arn: target.target_arn,
current_backlog_count: 0,
current_backlog_bytes: 0,
durable_mrf_backlog_count: 0,
durable_mrf_backlog_bytes: 0,
});
entry.durable_mrf_backlog_count = target.count;
entry.durable_mrf_backlog_bytes = target.bytes;
}
let mut target_backlogs = target_backlogs.into_values().collect::<Vec<_>>();
target_backlogs.sort_by(|left, right| left.target_arn.cmp(&right.target_arn));
ObsBucketReplicationStatsSnapshot {
bucket,
total_failed_bytes: runtime.total_failed_bytes,
@@ -190,16 +240,17 @@ fn bucket_replication_stats_snapshot_from_parts(
resync_duration_ms: runtime.resync_duration_ms,
current_backlog_count: runtime.current_backlog_count,
current_backlog_bytes: runtime.current_backlog_bytes,
durable_mrf_available,
durable_mrf_backlog_count: durable_bucket.count,
durable_mrf_backlog_bytes: durable_bucket.bytes,
mrf_pending_count: mrf_observability.pending_count,
mrf_pending_bytes: mrf_observability.pending_bytes,
mrf_dropped_count: mrf_observability.dropped_count,
mrf_missed_count: mrf_observability.missed_count,
mrf_flush_failures: mrf_observability.flush_failure_count,
mrf_last_flush_duration_millis: mrf_observability.last_flush_duration_millis,
durable_mrf_available: backlog.durable_mrf_available,
durable_mrf_backlog_count: backlog.durable_bucket.count,
durable_mrf_backlog_bytes: backlog.durable_bucket.bytes,
mrf_pending_count: backlog.mrf_observability.pending_count,
mrf_pending_bytes: backlog.mrf_observability.pending_bytes,
mrf_dropped_count: backlog.mrf_observability.dropped_count,
mrf_missed_count: backlog.mrf_observability.missed_count,
mrf_flush_failures: backlog.mrf_observability.flush_failure_count,
mrf_last_flush_duration_millis: backlog.mrf_observability.last_flush_duration_millis,
targets: runtime.targets,
target_backlogs,
}
}
@@ -222,6 +273,27 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
.into_iter()
.map(|bucket| (bucket.bucket.clone(), bucket))
.collect::<HashMap<String, DurableMrfBucketBacklog>>();
let mut durable_targets_by_bucket: HashMap<String, Vec<DurableMrfTargetBacklog>> = HashMap::new();
let durable_mrf_targets = if replication_storage_available && durable_mrf_available {
durable_mrf_target_backlog_snapshot()
} else {
Vec::new()
};
for target in durable_mrf_targets {
durable_targets_by_bucket
.entry(target.bucket.clone())
.or_default()
.push(target);
}
let mut runtime_targets_by_bucket: HashMap<String, Vec<RuntimeReplicationTargetBacklog>> = HashMap::new();
if let Some(stats) = &stats {
for target in stats.runtime_target_backlog_snapshot() {
runtime_targets_by_bucket
.entry(target.bucket.clone())
.or_default()
.push(target);
}
}
let mrf_observability = if replication_storage_available {
mrf_backlog_observability_snapshot()
} else {
@@ -236,7 +308,8 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
all_bucket_stats
.len()
.saturating_add(durable_buckets.len())
.saturating_add(mrf_observability_buckets.len()),
.saturating_add(mrf_observability_buckets.len())
.saturating_add(runtime_targets_by_bucket.len()),
);
bucket_names.extend(all_bucket_stats.keys().cloned());
bucket_names.extend(
@@ -251,6 +324,16 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
.filter(|bucket| !all_bucket_stats.contains_key(*bucket) && !durable_buckets.contains_key(*bucket))
.cloned(),
);
bucket_names.extend(
runtime_targets_by_bucket
.keys()
.filter(|bucket| {
!all_bucket_stats.contains_key(*bucket)
&& !durable_buckets.contains_key(*bucket)
&& !mrf_observability_buckets.contains_key(*bucket)
})
.cloned(),
);
let mut buckets = Vec::with_capacity(bucket_names.len());
for bucket in bucket_names {
@@ -327,14 +410,20 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
let runtime_targets = runtime_targets_by_bucket.remove(&bucket).unwrap_or_default();
let durable_targets = durable_targets_by_bucket.remove(&bucket).unwrap_or_default();
let mrf_observability = mrf_observability_buckets.get(&bucket).cloned().unwrap_or_default();
buckets.push(bucket_replication_stats_snapshot_from_parts(
bucket,
runtime,
proxy,
durable_mrf_available,
durable_bucket,
mrf_observability,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available,
durable_bucket,
runtime_targets,
durable_targets,
mrf_observability,
},
));
}
@@ -426,20 +515,34 @@ mod tests {
proxied_get_requests_failures: 1,
..Default::default()
},
true,
DurableMrfBucketBacklog {
bucket: "runtime-bucket".to_string(),
count: 5,
bytes: 8192,
},
MrfBucketBacklogObservability {
bucket: "runtime-bucket".to_string(),
pending_count: 1,
pending_bytes: 512,
dropped_count: 2,
missed_count: 3,
flush_failure_count: 4,
last_flush_duration_millis: 5,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: true,
durable_bucket: DurableMrfBucketBacklog {
bucket: "runtime-bucket".to_string(),
count: 5,
bytes: 8192,
},
runtime_targets: vec![RuntimeReplicationTargetBacklog {
bucket: "runtime-bucket".to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
count: 3,
bytes: 4096,
}],
durable_targets: vec![DurableMrfTargetBacklog {
bucket: "runtime-bucket".to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
count: 2,
bytes: 4096,
}],
mrf_observability: MrfBucketBacklogObservability {
bucket: "runtime-bucket".to_string(),
pending_count: 1,
pending_bytes: 512,
dropped_count: 2,
missed_count: 3,
flush_failure_count: 4,
last_flush_duration_millis: 5,
},
},
);
@@ -449,6 +552,12 @@ mod tests {
assert!(snapshot.durable_mrf_available);
assert_eq!(snapshot.durable_mrf_backlog_count, 5);
assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
assert_eq!(snapshot.target_backlogs.len(), 1);
assert_eq!(snapshot.target_backlogs[0].target_arn, "arn:rustfs:replication:target-a");
assert_eq!(snapshot.target_backlogs[0].current_backlog_count, 3);
assert_eq!(snapshot.target_backlogs[0].current_backlog_bytes, 4096);
assert_eq!(snapshot.target_backlogs[0].durable_mrf_backlog_count, 2);
assert_eq!(snapshot.target_backlogs[0].durable_mrf_backlog_bytes, 4096);
assert_eq!(snapshot.mrf_pending_count, 1);
assert_eq!(snapshot.mrf_pending_bytes, 512);
assert_eq!(snapshot.mrf_dropped_count, 2);
@@ -466,13 +575,15 @@ mod tests {
"durable-only".to_string(),
ObsBucketReplicationRuntimeSnapshot::default(),
ObsBucketReplicationProxySnapshot::default(),
true,
DurableMrfBucketBacklog {
bucket: "durable-only".to_string(),
count: 11,
bytes: 2048,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: true,
durable_bucket: DurableMrfBucketBacklog {
bucket: "durable-only".to_string(),
count: 11,
bytes: 2048,
},
..Default::default()
},
MrfBucketBacklogObservability::default(),
);
assert_eq!(snapshot.bucket, "durable-only");
@@ -488,16 +599,18 @@ mod tests {
"mrf-observability-only".to_string(),
ObsBucketReplicationRuntimeSnapshot::default(),
ObsBucketReplicationProxySnapshot::default(),
true,
DurableMrfBucketBacklog::default(),
MrfBucketBacklogObservability {
bucket: "mrf-observability-only".to_string(),
pending_count: 13,
pending_bytes: 4096,
dropped_count: 1,
missed_count: 2,
flush_failure_count: 3,
last_flush_duration_millis: 4,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: true,
mrf_observability: MrfBucketBacklogObservability {
bucket: "mrf-observability-only".to_string(),
pending_count: 13,
pending_bytes: 4096,
dropped_count: 1,
missed_count: 2,
flush_failure_count: 3,
last_flush_duration_millis: 4,
},
..Default::default()
},
);
@@ -525,9 +638,7 @@ mod tests {
..Default::default()
},
ObsBucketReplicationProxySnapshot::default(),
false,
DurableMrfBucketBacklog::default(),
MrfBucketBacklogObservability::default(),
ObsBucketReplicationBacklogSnapshot::default(),
);
assert_eq!(snapshot.current_backlog_count, 1);
+281 -1
View File
@@ -365,8 +365,48 @@ pub mod default {
use super::Policy;
/// Name of the built-in policy granting KMS key lifecycle management.
pub const KMS_KEY_ADMINISTRATOR: &str = "KMSKeyAdministrator";
/// Name of the built-in policy granting cryptographic use of KMS keys.
pub const KMS_KEY_USER: &str = "KMSKeyUser";
/// Name of the built-in policy granting read-only visibility into KMS keys.
pub const KMS_AUDITOR: &str = "KMSAuditor";
/// Every KMS key, in the resource grammar identity policies use.
///
/// The built-in KMS policies ship unscoped so they behave like the other canned
/// policies; an operator narrows a copy to `arn:aws:kms:::key/<key_id>` per workload.
const ALL_KMS_KEYS: &str = "*";
/// A KMS statement allowing `actions` on every key.
fn kms_allow(actions: Vec<Action>) -> Statement {
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet(actions),
not_actions: ActionSet(Default::default()),
resources: ResourceSet(vec![Resource::Kms(ALL_KMS_KEYS.into())]),
conditions: Functions::default(),
..Default::default()
}
}
/// The `sts:AssumeRole` grant every canned policy carries so an STS session may
/// assume it.
fn assume_role_allow() -> Statement {
Statement {
sid: "".into(),
effect: Effect::Allow,
actions: ActionSet(vec![Action::StsAction(StsAction::AssumeRoleAction)]),
not_actions: ActionSet(Default::default()),
resources: ResourceSet(Default::default()),
conditions: Functions::default(),
..Default::default()
}
}
#[allow(clippy::incompatible_msrv)]
pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 5]> = LazyLock::new(|| {
pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 8]> = LazyLock::new(|| {
[
(
"readwrite",
@@ -534,6 +574,65 @@ pub mod default {
],
},
),
// KMS role templates. They deliberately carry no S3 or admin grants, so an
// operator combines one with a data-plane policy ("readwrite,KMSKeyUser").
//
// None of them grants kms:Configure, kms:ServiceControl, kms:ClearCache,
// kms:Backup or kms:Restore: those act on the KMS service or on the material
// of every key at once, which is a cluster-administration power rather than a
// key-management one, and they stay with consoleAdmin. Key creation currently
// shares kms:Configure with backend configuration, so it stays there too.
(
KMS_KEY_ADMINISTRATOR,
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![
// Separation of duties: a key administrator governs a key's
// lifecycle but is never able to encrypt or decrypt with it.
kms_allow(vec![
Action::KmsAction(KmsAction::DescribeKeyAction),
Action::KmsAction(KmsAction::ListKeysAction),
Action::KmsAction(KmsAction::EnableKeyAction),
Action::KmsAction(KmsAction::DisableKeyAction),
Action::KmsAction(KmsAction::RotateKeyAction),
Action::KmsAction(KmsAction::DeleteKeyAction),
]),
assume_role_allow(),
],
},
),
(
KMS_KEY_USER,
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![
// The two actions the SSE-KMS data path evaluates, plus the
// metadata read a client needs to tell which key it is using.
kms_allow(vec![
Action::KmsAction(KmsAction::GenerateDataKeyAction),
Action::KmsAction(KmsAction::DecryptAction),
Action::KmsAction(KmsAction::DescribeKeyAction),
]),
assume_role_allow(),
],
},
),
(
KMS_AUDITOR,
Policy {
id: "".into(),
version: DEFAULT_VERSION.into(),
statements: vec![
kms_allow(vec![
Action::KmsAction(KmsAction::DescribeKeyAction),
Action::KmsAction(KmsAction::ListKeysAction),
]),
assume_role_allow(),
],
},
),
]
});
}
@@ -542,6 +641,7 @@ pub mod default {
mod test {
use super::*;
use crate::error::Result;
use crate::policy::action::{AdminAction, KmsAction, S3Action};
#[tokio::test]
async fn test_parse_policy() -> Result<()> {
@@ -709,6 +809,186 @@ mod test {
}
}
// ------------------------------------------------------------------------
// Built-in KMS role templates
// ------------------------------------------------------------------------
fn default_policy(name: &str) -> &'static Policy {
default::DEFAULT_POLICIES
.iter()
.find_map(|(candidate, policy)| (*candidate == name).then_some(policy))
.unwrap_or_else(|| panic!("built-in policy {name} should exist"))
}
/// Evaluate `policy` for `account` against `action` on `key_id`.
///
/// Mirrors the admin and SSE call sites: the requested key identifier travels in
/// `object` with `bucket` left empty. An empty `key_id` is the unscoped call.
async fn kms_allows(policy: &Policy, account: &str, action: KmsAction, key_id: &str) -> bool {
let conditions = HashMap::new();
let claims = HashMap::new();
policy
.is_allowed(&Args {
account,
groups: &None,
action: Action::KmsAction(action),
bucket: "",
conditions: &conditions,
is_owner: false,
object: key_id,
claims: &claims,
deny_only: false,
})
.await
}
const KMS_LIFECYCLE_ACTIONS: [KmsAction; 4] = [
KmsAction::EnableKeyAction,
KmsAction::DisableKeyAction,
KmsAction::RotateKeyAction,
KmsAction::DeleteKeyAction,
];
const KMS_CRYPTO_ACTIONS: [KmsAction; 2] = [KmsAction::GenerateDataKeyAction, KmsAction::DecryptAction];
/// Actions that act on the service or on every key's material at once. No role
/// template may confer them.
const KMS_CLUSTER_ADMIN_ACTIONS: [KmsAction; 6] = [
KmsAction::AllActions,
KmsAction::ConfigureAction,
KmsAction::ServiceControlAction,
KmsAction::ClearCacheAction,
KmsAction::BackupAction,
KmsAction::RestoreAction,
];
const KMS_ROLE_TEMPLATES: [&str; 3] = [default::KMS_KEY_ADMINISTRATOR, default::KMS_KEY_USER, default::KMS_AUDITOR];
#[tokio::test]
async fn kms_key_administrator_manages_keys_but_cannot_use_them() {
let policy = default_policy(default::KMS_KEY_ADMINISTRATOR);
for action in KMS_LIFECYCLE_ACTIONS {
assert!(
kms_allows(policy, "keyadmin", action, "app-key").await,
"KMSKeyAdministrator should allow {action:?}"
);
}
assert!(kms_allows(policy, "keyadmin", KmsAction::DescribeKeyAction, "app-key").await);
assert!(kms_allows(policy, "keyadmin", KmsAction::ListKeysAction, "").await);
for action in KMS_CRYPTO_ACTIONS {
assert!(
!kms_allows(policy, "keyadmin", action, "app-key").await,
"KMSKeyAdministrator must not allow {action:?}"
);
}
}
#[tokio::test]
async fn kms_key_user_uses_keys_but_cannot_manage_them() {
let policy = default_policy(default::KMS_KEY_USER);
for action in KMS_CRYPTO_ACTIONS {
assert!(
kms_allows(policy, "appuser", action, "app-key").await,
"KMSKeyUser should allow {action:?}"
);
}
assert!(kms_allows(policy, "appuser", KmsAction::DescribeKeyAction, "app-key").await);
for action in KMS_LIFECYCLE_ACTIONS {
assert!(
!kms_allows(policy, "appuser", action, "app-key").await,
"KMSKeyUser must not allow {action:?}"
);
}
assert!(!kms_allows(policy, "appuser", KmsAction::ListKeysAction, "").await);
}
#[tokio::test]
async fn kms_auditor_only_reads_key_metadata() {
let policy = default_policy(default::KMS_AUDITOR);
assert!(kms_allows(policy, "auditor", KmsAction::DescribeKeyAction, "app-key").await);
assert!(kms_allows(policy, "auditor", KmsAction::ListKeysAction, "").await);
for action in KMS_LIFECYCLE_ACTIONS.iter().chain(KMS_CRYPTO_ACTIONS.iter()) {
assert!(
!kms_allows(policy, "auditor", *action, "app-key").await,
"KMSAuditor must not allow {action:?}"
);
}
}
#[tokio::test]
async fn kms_role_templates_withhold_service_and_bundle_actions() {
for name in KMS_ROLE_TEMPLATES {
let policy = default_policy(name);
for action in KMS_CLUSTER_ADMIN_ACTIONS {
assert!(
!kms_allows(policy, "someone", action, "app-key").await,
"{name} must not allow {action:?}"
);
}
}
}
#[tokio::test]
async fn kms_role_templates_grant_nothing_outside_kms() {
let conditions = HashMap::new();
let claims = HashMap::new();
let foreign_actions = [
Action::S3Action(S3Action::GetObjectAction),
Action::S3Action(S3Action::PutObjectAction),
Action::AdminAction(AdminAction::ServerInfoAdminAction),
];
for name in KMS_ROLE_TEMPLATES {
let policy = default_policy(name);
for action in &foreign_actions {
let allowed = policy
.is_allowed(&Args {
account: "someone",
groups: &None,
action: *action,
bucket: "any-bucket",
conditions: &conditions,
is_owner: false,
object: "any-object",
claims: &claims,
deny_only: false,
})
.await;
assert!(!allowed, "{name} must not allow {action:?}");
}
}
}
/// The narrowing an operator is told to apply must actually deny the other keys.
#[tokio::test]
async fn narrowed_kms_role_template_denies_other_keys() -> Result<()> {
let narrowed = Policy::parse_config(
br#"{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["kms:GenerateDataKey", "kms:Decrypt", "kms:DescribeKey"],
"Resource": ["arn:aws:kms:::key/reports-*"]
}
]
}"#,
)?;
assert!(kms_allows(&narrowed, "appuser", KmsAction::GenerateDataKeyAction, "reports-2026").await);
assert!(kms_allows(&narrowed, "appuser", KmsAction::DecryptAction, "reports-2026").await);
assert!(!kms_allows(&narrowed, "appuser", KmsAction::DecryptAction, "payroll-2026").await);
assert!(!kms_allows(&narrowed, "appuser", KmsAction::DisableKeyAction, "reports-2026").await);
Ok(())
}
#[tokio::test]
async fn test_deny_only_checks_only_deny_statements() -> Result<()> {
let data = r#"
+254 -72
View File
@@ -18,9 +18,12 @@ use crate::rule::ReplicationRuleExt as _;
use s3s::dto::DeleteMarkerReplicationStatus;
use s3s::dto::DeleteReplicationStatus;
use s3s::dto::Destination;
use s3s::dto::{ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRuleStatus, ReplicationRules};
use s3s::dto::{
ExistingObjectReplicationStatus, ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule,
ReplicationRuleStatus, ReplicationRules,
};
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use uuid::Uuid;
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -43,6 +46,44 @@ pub trait ReplicationConfigurationExt {
fn get_destination(&self) -> Destination;
fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
self.filter_target_arns(obj)
.into_iter()
.map(|arn| {
let mut target = obj.clone();
target.target_arn = arn.clone();
(arn, self.replicate(&target))
})
.collect()
}
}
fn rule_replicates(rule: &ReplicationRule, obj: &ObjectOpts) -> bool {
if let Some(status) = &rule.existing_object_replication
&& obj.existing_object
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
{
return false;
}
if obj.op_type != ReplicationType::Delete {
return rule.metadata_replicate(obj);
}
if !rule.metadata_replicate(obj) {
return false;
}
let version_purge = obj.version_id.is_some();
if version_purge {
rule.delete_replication
.as_ref()
.is_some_and(|delete| delete.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED))
} else {
rule.delete_marker_replication.as_ref().is_some_and(|delete_marker| {
delete_marker.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -73,6 +114,58 @@ pub fn unsupported_replication_config_field(config: &ReplicationConfiguration) -
None
}
pub fn invalid_replication_config_status_field(config: &ReplicationConfiguration) -> Option<&'static str> {
for rule in &config.rules {
if !matches!(rule.status.as_str(), ReplicationRuleStatus::ENABLED | ReplicationRuleStatus::DISABLED) {
return Some("Rule.Status");
}
if rule.existing_object_replication.as_ref().is_some_and(|existing| {
!matches!(
existing.status.as_str(),
ExistingObjectReplicationStatus::ENABLED | ExistingObjectReplicationStatus::DISABLED
)
}) {
return Some("Rule.ExistingObjectReplication.Status");
}
if rule.delete_replication.as_ref().is_some_and(|delete| {
!matches!(
delete.status.as_str(),
DeleteReplicationStatus::ENABLED | DeleteReplicationStatus::DISABLED
)
}) {
return Some("Rule.DeleteReplication.Status");
}
if rule
.delete_marker_replication
.as_ref()
.and_then(|delete| delete.status.as_ref())
.is_some_and(|status| {
!matches!(
status.as_str(),
DeleteMarkerReplicationStatus::ENABLED | DeleteMarkerReplicationStatus::DISABLED
)
})
{
return Some("Rule.DeleteMarkerReplication.Status");
}
if rule
.source_selection_criteria
.as_ref()
.and_then(|criteria| criteria.replica_modifications.as_ref())
.is_some_and(|modifications| {
!matches!(
modifications.status.as_str(),
ReplicaModificationsStatus::ENABLED | ReplicaModificationsStatus::DISABLED
)
})
{
return Some("Rule.SourceSelectionCriteria.ReplicaModifications.Status");
}
}
None
}
pub fn active_replication_rule_destination_arns(config: &ReplicationConfiguration) -> HashSet<String> {
let mut arns = HashSet::new();
@@ -220,40 +313,7 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
/// Determine whether an object should be replicated
fn replicate(&self, obj: &ObjectOpts) -> bool {
let rules = self.filter_actionable_rules(obj);
for rule in rules.iter() {
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
continue;
}
if let Some(status) = &rule.existing_object_replication
&& obj.existing_object
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
{
return false;
}
if obj.op_type == ReplicationType::Delete {
if !rule.metadata_replicate(obj) {
return false;
}
if obj.version_id.is_some() {
return rule
.delete_replication
.clone()
.is_some_and(|d| d.status == DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED));
} else {
return rule.delete_marker_replication.clone().is_some_and(|d| {
d.status == Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED))
});
}
}
// Regular object/metadata replication
return rule.metadata_replicate(obj);
}
false
rules.first().is_some_and(|rule| rule_replicates(rule, obj))
}
/// Check for an active rule
@@ -317,6 +377,41 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
}
arns
}
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
let rules = self.filter_actionable_rules(obj);
let role = self.role.trim();
if !role.is_empty() {
let mut selected = None;
for rule in &rules {
if selected.is_none_or(|current: &ReplicationRule| rule.priority > current.priority) {
selected = Some(rule);
}
}
return vec![(role.to_string(), selected.is_some_and(|rule| rule_replicates(rule, obj)))];
}
let mut target_indexes: HashMap<&str, usize> = HashMap::new();
let mut selected_rules: Vec<(&str, &ReplicationRule)> = Vec::new();
for rule in &rules {
let arn = rule.destination.bucket.trim();
if arn.is_empty() {
continue;
}
if let Some(index) = target_indexes.get(arn).copied() {
if rule.priority > selected_rules[index].1.priority {
selected_rules[index].1 = rule;
}
} else {
target_indexes.insert(arn, selected_rules.len());
selected_rules.push((arn, rule));
}
}
selected_rules
.into_iter()
.map(|(arn, rule)| (arn.to_string(), rule_replicates(rule, obj)))
.collect()
}
}
#[cfg(test)]
@@ -324,8 +419,8 @@ mod tests {
use super::*;
use s3s::dto::{
DeleteMarkerReplication, DeleteReplication, Destination, EncryptionConfiguration, ExistingObjectReplication, Metrics,
MetricsStatus, ReplicationRule, ReplicationTime, ReplicationTimeStatus, ReplicationTimeValue, SourceSelectionCriteria,
SseKmsEncryptedObjects, SseKmsEncryptedObjectsStatus,
MetricsStatus, ReplicaModifications, ReplicationRule, ReplicationTime, ReplicationTimeStatus, ReplicationTimeValue,
SourceSelectionCriteria, SseKmsEncryptedObjects, SseKmsEncryptedObjectsStatus,
};
fn replication_rule(id: &str, arn: &str) -> ReplicationRule {
@@ -627,61 +722,75 @@ mod tests {
);
}
#[test]
fn role_delete_decision_follows_highest_priority_rule() {
let role = "arn:rustfs:replication:us-east-1:role-target:bucket";
let destination = "arn:rustfs:replication:us-east-1:target:bucket";
let config = ReplicationConfiguration {
role: role.to_string(),
rules: vec![
delete_marker_rule("low-priority-enabled", destination, "logs/", 1, true),
delete_marker_rule("high-priority-disabled", destination, "logs/2026/", 5, false),
],
};
let opts = ObjectOpts {
name: "logs/2026/app.log".to_string(),
op_type: ReplicationType::Delete,
delete_marker: true,
..Default::default()
};
assert_eq!(
config.filter_target_replication_decisions(&opts),
vec![(role.to_string(), false)],
"the role target must use the highest-priority matching rule"
);
}
#[test]
fn version_purge_uses_delete_replication_for_object_and_marker_versions() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule("delete", arn);
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
});
let mut rule = delete_marker_rule("delete-switches", arn, "", 1, true);
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
let mut config = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
let version_id = Some(Uuid::new_v4());
for delete_marker in [false, true] {
assert!(config.replicate(&ObjectOpts {
name: "object".to_string(),
version_id,
delete_marker,
op_type: ReplicationType::Delete,
..Default::default()
}));
for version_id in [Some(Uuid::new_v4()), Some(Uuid::nil())] {
for delete_marker in [false, true] {
assert!(!config.replicate(&ObjectOpts {
name: "object".to_string(),
op_type: ReplicationType::Delete,
version_id,
delete_marker,
..Default::default()
}));
}
}
assert!(!config.replicate(&ObjectOpts {
let stored_marker = ObjectOpts {
name: "object".to_string(),
delete_marker: true,
op_type: ReplicationType::Delete,
delete_marker: true,
..Default::default()
}));
};
assert!(config.replicate(&stored_marker), "stored markers must use DeleteMarkerReplication");
assert_eq!(config.filter_target_replication_decisions(&stored_marker), vec![(arn.to_string(), true)]);
let rule = &mut config.rules[0];
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
for delete_marker in [false, true] {
assert!(!config.replicate(&ObjectOpts {
name: "object".to_string(),
version_id,
delete_marker,
op_type: ReplicationType::Delete,
..Default::default()
}));
}
assert!(config.replicate(&ObjectOpts {
name: "object".to_string(),
delete_marker: true,
op_type: ReplicationType::Delete,
version_id: Some(Uuid::nil()),
delete_marker: true,
..Default::default()
}));
assert!(config.replicate(&stored_marker));
}
#[test]
@@ -721,4 +830,77 @@ mod tests {
});
assert_eq!(unsupported_replication_config_field(&config), Some("Destination.ReplicationTime"));
}
#[test]
fn invalid_replication_status_fields_are_reported_before_persistence() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut config = ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule("invalid-status", arn)],
};
config.rules[0].status = ReplicationRuleStatus::from_static("Invalid");
assert_eq!(invalid_replication_config_status_field(&config), Some("Rule.Status"));
config.rules[0] = replication_rule("invalid-status", arn);
config.rules[0].existing_object_replication = Some(ExistingObjectReplication {
status: ExistingObjectReplicationStatus::from_static("Invalid"),
});
assert_eq!(
invalid_replication_config_status_field(&config),
Some("Rule.ExistingObjectReplication.Status")
);
config.rules[0] = replication_rule("invalid-status", arn);
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static("Invalid"),
});
assert_eq!(invalid_replication_config_status_field(&config), Some("Rule.DeleteReplication.Status"));
config.rules[0] = replication_rule("invalid-status", arn);
config.rules[0].delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static("Invalid")),
});
assert_eq!(
invalid_replication_config_status_field(&config),
Some("Rule.DeleteMarkerReplication.Status")
);
config.rules[0] = replication_rule("invalid-status", arn);
config.rules[0].source_selection_criteria = Some(SourceSelectionCriteria {
replica_modifications: Some(ReplicaModifications {
status: ReplicaModificationsStatus::from_static("Invalid"),
}),
sse_kms_encrypted_objects: None,
});
assert_eq!(
invalid_replication_config_status_field(&config),
Some("Rule.SourceSelectionCriteria.ReplicaModifications.Status")
);
}
#[test]
fn target_decisions_choose_highest_priority_rule_per_destination() {
let target_a = "arn:rustfs:replication:us-east-1:target:a";
let target_b = "arn:rustfs:replication:us-east-1:target:b";
let mut a_low = delete_marker_rule("a-low", target_a, "logs/", 1, true);
let b = delete_marker_rule("b", target_b, "logs/", 2, true);
let a_high = delete_marker_rule("a-high", target_a, "logs/2026/", 5, false);
a_low.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![a_low, b, a_high],
};
let decisions = config.filter_target_replication_decisions(&ObjectOpts {
name: "logs/2026/app.log".to_string(),
op_type: ReplicationType::Delete,
delete_marker: true,
..Default::default()
});
assert_eq!(decisions, vec![(target_a.to_string(), false), (target_b.to_string(), true)]);
}
}
+8
View File
@@ -49,6 +49,11 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
.delete_object
.delete_marker_mtime
.and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
target_arns: if self.target_arn.is_empty() {
Vec::new()
} else {
vec![self.target_arn.clone()]
},
}
}
@@ -111,6 +116,7 @@ mod tests {
delete_marker_mtime: Some(mtime),
..Default::default()
},
target_arn: "arn:target-a".to_string(),
..Default::default()
};
@@ -129,6 +135,7 @@ mod tests {
Some(mtime.unix_timestamp_nanos() as i64),
"delete-marker mtime must be persisted in the MRF entry"
);
assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
assert_eq!(info.get_object(), "object");
}
@@ -148,6 +155,7 @@ mod tests {
};
assert_eq!(info.to_mrf_entry().delete_marker_mtime, None);
assert!(info.to_mrf_entry().target_arns.is_empty());
}
#[test]
+73
View File
@@ -593,6 +593,9 @@ pub struct MrfReplicateEntry {
// to preserve pre-existing behaviour (backlog#867).
#[serde(rename = "deleteMarkerMtime", skip_serializing_if = "Option::is_none", default)]
pub delete_marker_mtime: Option<i64>,
#[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)]
pub target_arns: Vec<String>,
}
fn retry_count_to_mrf(retry_count: u32) -> i32 {
@@ -672,6 +675,18 @@ impl ReplicateDecision {
}
if result.is_empty() { None } else { Some(result) }
}
pub fn replicate_target_arns(&self) -> Vec<String> {
let mut arns = self
.targets_map
.values()
.filter(|target| target.replicate && !target.arn.is_empty())
.map(|target| target.arn.clone())
.collect::<Vec<_>>();
arns.sort();
arns.dedup();
arns
}
}
impl fmt::Display for ReplicateDecision {
@@ -799,6 +814,7 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: self.dsc.replicate_target_arns(),
}
}
@@ -853,6 +869,7 @@ impl ReplicateObjectInfo {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: self.dsc.replicate_target_arns(),
}
}
}
@@ -994,6 +1011,62 @@ impl Default for ResyncDecision {
mod tests {
use super::*;
#[test]
fn replicate_decision_returns_sorted_unique_replicating_target_arns() {
let mut decision = ReplicateDecision::new();
decision.set(ReplicateTargetDecision {
arn: "arn:target-b".to_string(),
replicate: true,
..Default::default()
});
decision.set(ReplicateTargetDecision {
arn: "arn:target-a".to_string(),
replicate: true,
..Default::default()
});
decision.set(ReplicateTargetDecision {
arn: "arn:target-c".to_string(),
replicate: false,
..Default::default()
});
decision.set(ReplicateTargetDecision {
arn: String::new(),
replicate: true,
..Default::default()
});
assert_eq!(
decision.replicate_target_arns(),
vec!["arn:target-a".to_string(), "arn:target-b".to_string()]
);
}
#[test]
fn replicate_object_info_mrf_entry_carries_replicating_targets() {
let mut decision = ReplicateDecision::new();
decision.set(ReplicateTargetDecision {
arn: "arn:target-a".to_string(),
replicate: true,
..Default::default()
});
decision.set(ReplicateTargetDecision {
arn: "arn:target-b".to_string(),
replicate: false,
..Default::default()
});
let info = ReplicateObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 42,
dsc: decision,
..Default::default()
};
let entry = info.to_mrf_entry();
assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
}
#[test]
fn target_state_reads_resync_timestamp_from_target_reset_header_key() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
+2 -2
View File
@@ -30,8 +30,8 @@ pub mod tagging;
pub use config::{
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, active_replication_rule_destination_arns,
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
validate_replication_config_target_arns,
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
unsupported_replication_config_field, validate_replication_config_target_arns,
};
pub use delete::{
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
+5
View File
@@ -71,6 +71,7 @@ mod tests {
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
},
MrfReplicateEntry {
bucket: "bucket-a".to_string(),
@@ -82,6 +83,7 @@ mod tests {
delete_marker_version_id: Some(del_vid),
delete_marker: true,
delete_marker_mtime: Some(1_705_312_200_123_456_789),
target_arns: vec!["arn:target-a".to_string()],
},
];
@@ -91,9 +93,11 @@ mod tests {
assert_eq!(decoded.len(), 2);
assert_eq!(decoded[0].version_id, Some(obj_vid));
assert_eq!(decoded[0].op, MrfOpKind::Object);
assert_eq!(decoded[0].target_arns, vec!["arn:target-a".to_string(), "arn:target-b".to_string()]);
assert_eq!(decoded[0].delete_marker_mtime, None);
assert_eq!(decoded[1].delete_marker_version_id, Some(del_vid));
assert_eq!(decoded[1].op, MrfOpKind::Delete);
assert_eq!(decoded[1].target_arns, vec!["arn:target-a".to_string()]);
assert!(decoded[1].delete_marker);
assert_eq!(
decoded[1].delete_marker_mtime,
@@ -129,6 +133,7 @@ mod tests {
assert_eq!(decoded[0].retry_count, 2);
assert_eq!(decoded[0].size, 100);
assert_eq!(decoded[0].op, MrfOpKind::Object);
assert!(decoded[0].target_arns.is_empty());
// Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the
// pre-#867 fallback to the current time.
assert_eq!(decoded[0].delete_marker_mtime, None);
+34 -26
View File
@@ -133,16 +133,13 @@ pub fn delete_replication_state_from_config(
replica: source.replica,
..Default::default()
};
let target_arns = config.filter_target_arns(&opts);
if target_arns.is_empty() {
let target_decisions = config.filter_target_replication_decisions(&opts);
if target_decisions.is_empty() {
return None;
}
let mut decision = ReplicateDecision::new();
for target_arn in target_arns {
let mut target_opts = opts.clone();
target_opts.target_arn = target_arn.clone();
let replicate = config.replicate(&target_opts);
for (target_arn, replicate) in target_decisions {
decision.set(ReplicateTargetDecision::new(target_arn, replicate, false));
}
if !decision.replicate_any() {
@@ -174,20 +171,13 @@ pub struct ReplicationDeleteScheduleInput<'a> {
pub deleted_delete_marker_version: bool,
}
fn delete_version_purge_source_status(status: &ReplicationStatusType) -> bool {
status == &ReplicationStatusType::Replica
|| status == &ReplicationStatusType::Pending
|| status == &ReplicationStatusType::Completed
|| status == &ReplicationStatusType::Failed
}
pub fn should_schedule_delete_replication(input: ReplicationDeleteScheduleInput<'_>) -> bool {
if input.replication_request {
return false;
}
if input.version_id_requested && !input.deleted_delete_marker_version && !input.source_delete_marker {
return delete_version_purge_source_status(input.source_replication_status);
if input.version_id_requested {
return input.source_version_purge_status == &VersionPurgeStatusType::Pending;
}
input.source_replication_status == &ReplicationStatusType::Replica
@@ -433,9 +423,7 @@ mod tests {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
}),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
delete_replication: None,
destination: Destination {
bucket: arn.to_string(),
..Default::default()
@@ -500,11 +488,15 @@ mod tests {
}
#[test]
fn delete_replication_state_tracks_delete_marker_version_purges() {
fn delete_replication_state_uses_delete_switch_for_marker_version_purges() {
let arn = "arn:aws:s3:::target-bucket";
let config = ReplicationConfiguration {
let mut rule = delete_replication_rule(arn, false);
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
});
let mut config = ReplicationConfiguration {
role: arn.to_string(),
rules: vec![delete_replication_rule(arn, false)],
rules: vec![rule],
};
let source = ReplicationDeleteStateSource {
name: "test/object.txt".to_string(),
@@ -514,8 +506,16 @@ mod tests {
replica: false,
};
assert!(
delete_replication_state_from_config(&config, &source).is_none(),
"delete-marker version purge must not use the enabled marker-creation switch"
);
config.rules[0].delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
let state = delete_replication_state_from_config(&config, &source)
.expect("delete-marker version purge should honor delete replication rules");
.expect("delete-marker version purge should honor the enabled permanent-delete switch");
let pending = format!("{arn}=PENDING;");
assert_eq!(state.version_purge_status_internal.as_deref(), Some(pending.as_str()));
@@ -536,8 +536,8 @@ mod tests {
}
#[test]
fn delete_replication_schedule_keeps_marker_and_version_purges() {
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
fn delete_replication_schedule_requires_current_admission_for_version_purges() {
assert!(!should_schedule_delete_replication(ReplicationDeleteScheduleInput {
replication_request: false,
version_id_requested: true,
source_delete_marker: true,
@@ -549,8 +549,16 @@ mod tests {
replication_request: false,
version_id_requested: true,
source_delete_marker: false,
source_replication_status: &ReplicationStatusType::Completed,
source_version_purge_status: &VersionPurgeStatusType::Empty,
source_replication_status: &ReplicationStatusType::Empty,
source_version_purge_status: &VersionPurgeStatusType::Pending,
deleted_delete_marker_version: true,
}));
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
replication_request: false,
version_id_requested: true,
source_delete_marker: false,
source_replication_status: &ReplicationStatusType::Empty,
source_version_purge_status: &VersionPurgeStatusType::Pending,
deleted_delete_marker_version: false,
}));
assert!(should_schedule_delete_replication(ReplicationDeleteScheduleInput {
+15
View File
@@ -410,6 +410,21 @@ mod tests {
assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
}
#[test]
fn heal_queue_action_preserves_pending_null_version_purge() {
let mut roi = replicate_object_info(ReplicationStatusType::Completed);
roi.version_id = Some(Uuid::nil());
roi.version_purge_status = VersionPurgeStatusType::Pending;
let action = replication_heal_queue_action(&mut roi);
let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
panic!("expected null-version purge delete queue action");
};
assert_eq!(delete_info.delete_object.version_id, Some(Uuid::nil()));
assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
}
#[test]
fn replication_priority_parses_known_values_and_defaults_unknown() {
assert_eq!(ReplicationPriority::from_str("fast"), Ok(ReplicationPriority::Fast));
+497 -11
View File
@@ -52,9 +52,9 @@ use tracing::{debug, error, warn};
use crate::{
BucketVersioningSys, Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts,
ReplicationConfig, ReplicationHealObject, ReplicationQueueAdmission, ReplicationStatusType, ScannerDiskExt as _,
ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule, apply_transition_rule,
enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object,
ReplicationConfig, ReplicationHealObject, ReplicationQueueAdmission, ReplicationStatusType, STORAGE_FORMAT_FILE,
ScannerDiskExt as _, ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule,
apply_transition_rule, enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object,
path2_bucket_object_with_base_path, queue_replication_heal, scanner_is_erasure,
scanner_replication_config_for_lifecycle_eval,
};
@@ -78,6 +78,8 @@ const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
// Erasure data directories contain direct part.N files; keep namespace probes bounded.
const ERASURE_DATA_DIR_PROBE_ENTRY_LIMIT: usize = 64;
const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
const ENV_DATA_USAGE_UPDATE_DIR_CYCLES: &str = "RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES";
const ENV_HEAL_OBJECT_SELECT_PROB: &str = "RUSTFS_HEAL_OBJECT_SELECT_PROB";
@@ -1038,7 +1040,7 @@ impl ScannerItem {
}
async fn heal_replication(&mut self, oi: &ObjectInfo, size_summary: &mut SizeSummary) {
if oi.version_id.is_none_or(|v| v.is_nil()) {
if oi.version_id.is_none_or(|version| version.is_nil()) && !oi.delete_marker && oi.version_purge_status.is_empty() {
return;
}
@@ -1254,6 +1256,43 @@ fn classify_get_size_failure(item: &ScannerItem, err: &StorageError) -> GetSizeF
GetSizeFailureAction::RecordFailed
}
async fn contains_erasure_part_file(path: &str) -> Result<bool, ScannerError> {
let mut entries = match tokio::fs::read_dir(path).await {
Ok(entries) => entries,
Err(err) if matches!(err.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => return Ok(false),
Err(err) => return Err(ScannerError::Io(err)),
};
for _ in 0..ERASURE_DATA_DIR_PROBE_ENTRY_LIMIT {
let entry = match entries.next_entry().await {
Ok(Some(entry)) => entry,
Ok(None) => return Ok(false),
Err(err) if matches!(err.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => return Ok(false),
Err(err) => return Err(ScannerError::Io(err)),
};
let file_name = entry.file_name();
let Some(part_number) = file_name
.to_str()
.and_then(|name| name.strip_prefix("part."))
.and_then(|number| number.parse::<u32>().ok())
else {
continue;
};
if part_number == 0 {
continue;
}
match entry.file_type().await {
Ok(file_type) if file_type.is_file() => return Ok(true),
Ok(_) => {}
Err(err) if matches!(err.kind(), ErrorKind::NotFound | ErrorKind::TooManyLinks) => {}
Err(err) => return Err(ScannerError::Io(err)),
}
}
Ok(false)
}
fn data_usage_root_has_progress(root: &DataUsageEntry) -> bool {
!root.children.is_empty()
|| root.size > 0
@@ -1278,6 +1317,7 @@ pub struct FolderScanner {
data_usage_scanner_debug: bool,
heal_object_select: u32,
scan_mode: HealScanMode,
is_erasure_mode: bool,
failed_object_ttl_secs: u64,
failed_objects_max: usize,
@@ -1852,7 +1892,8 @@ impl FolderScanner {
let mut existing_folders: Vec<CachedFolder> = Vec::new();
let mut new_folders: Vec<CachedFolder> = Vec::new();
let mut found_objects = false;
let mut found_object_metadata = false;
let mut erasure_data_directory_candidates: Vec<(CachedFolder, bool, String)> = Vec::new();
let mut object_count: u64 = 0;
let yield_every_objects = scanner_yield_every_n_objects();
@@ -1919,6 +1960,7 @@ impl FolderScanner {
if file_name.is_empty() || file_name == "." || file_name == ".." {
continue;
}
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
let file_path = entry.path().to_string_lossy().to_string();
@@ -1963,6 +2005,14 @@ impl FolderScanner {
Err(e) => return Err(ScannerError::Io(e)),
};
// Metadata presence establishes an erasure object boundary;
// parsing failures still belong to accounting and healing. A
// directory named `xl.meta` remains a valid namespace prefix,
// and symlinks are classified after resolving their target.
if is_storage_format_entry && !entry_type.is_dir() && !entry_type.is_symlink() {
found_object_metadata = true;
}
if entry_type.is_symlink() {
let metadata = match tokio::fs::metadata(&file_path).await {
Ok(metadata) => metadata,
@@ -2009,6 +2059,9 @@ impl FolderScanner {
}
entry_type = metadata.file_type();
if is_storage_format_entry {
found_object_metadata = true;
}
}
// ok
@@ -2041,6 +2094,11 @@ impl FolderScanner {
object_heal_prob_div: folder.object_heal_prob_div,
};
if self.is_erasure_mode && uuid::Uuid::parse_str(&file_name).is_ok_and(|data_dir_id| !data_dir_id.is_nil()) {
erasure_data_directory_candidates.push((this, exists, file_path));
continue;
}
abandoned_children.remove(&h.key());
if exists {
@@ -2130,7 +2188,7 @@ impl FolderScanner {
}
};
found_objects = true;
found_object_metadata = true;
item.transform_meta_dir();
@@ -2156,11 +2214,70 @@ impl FolderScanner {
}
self.budget.record_entries_visited(pending_entry_progress);
let mut found_erasure_data_directory = false;
if self.is_erasure_mode && !found_object_metadata {
for (_, _, path) in &erasure_data_directory_candidates {
if contains_erasure_part_file(path).await? {
found_erasure_data_directory = true;
break;
}
}
}
if !found_object_metadata && !found_erasure_data_directory {
for (candidate, exists, _) in erasure_data_directory_candidates {
let h = hash_path(&candidate.name);
abandoned_children.remove(&h.key());
if exists {
self.update_cache.copy_with_children(&self.old_cache, &h, &candidate.parent);
existing_folders.push(candidate);
} else {
new_folders.push(candidate);
}
}
}
if self.is_erasure_mode && found_erasure_data_directory && !found_object_metadata {
found_object_metadata = true;
let metadata_path = path_join_buf(&[&dir_path, STORAGE_FORMAT_FILE]);
if !self.should_skip_failed(&metadata_path) {
into.failed_objects = into.failed_objects.saturating_add(1);
self.record_failed(&metadata_path);
let failed_cache_entries = self.new_cache.info.failed_objects.len();
if failed_cache_entries > 0 && should_log_failed_object(failed_cache_entries) {
warn!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
path = %metadata_path,
failed_objects = failed_cache_entries,
state = "object_metadata_missing",
"Scanner found erasure object data without metadata"
);
}
let (bucket, object) = path2_bucket_object_with_base_path(&self.root, &folder.name);
if !bucket.is_empty() && !object.is_empty() {
self.send_required_scanner_heal_request(
PendingScannerHealKind::Object,
bucket.clone(),
Some(object.clone()),
None,
build_object_heal_request(bucket, object, None, self.scan_mode, HealChannelPriority::High),
)
.await?;
}
}
}
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
if found_objects && scanner_is_erasure().await {
if found_object_metadata && self.is_erasure_mode {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
debug!(
target: "rustfs::scanner::folder",
@@ -2829,6 +2946,7 @@ pub async fn scan_data_folder(
data_usage_scanner_debug: false,
heal_object_select,
scan_mode,
is_erasure_mode,
failed_object_ttl_secs: failed_object_ttl,
failed_objects_max,
sleeper,
@@ -3037,6 +3155,7 @@ mod tests {
data_usage_scanner_debug: false,
heal_object_select: 0,
scan_mode: HealScanMode::Normal,
is_erasure_mode: false,
failed_object_ttl_secs: u64::MAX,
failed_objects_max: usize::MAX,
sleeper: SCANNER_SLEEPER.clone(),
@@ -3188,6 +3307,61 @@ mod tests {
assert!(!ScannerItem::should_account_replication_stats(&purge_version));
}
#[tokio::test]
#[serial]
async fn test_heal_replication_only_queues_pending_null_deletes() {
async fn replication_skipped_count() -> u64 {
global_metrics()
.report()
.await
.source_work
.iter()
.find(|work| work.source == ScannerWorkSource::BucketReplication.as_str())
.map(|work| work.skipped)
.unwrap_or_default()
}
let temp_dir = std::env::temp_dir();
let file_type = std::fs::metadata(&temp_dir)
.expect("temp dir metadata should be readable")
.file_type();
let mut item = ScannerItem {
path: temp_dir.join("object").to_string_lossy().to_string(),
bucket: "bucket".to_string(),
prefix: String::new(),
object_name: "object".to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: Some(Arc::new(ReplicationConfig::new(None, None))),
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let null_object = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::nil()),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
let mut size_summary = SizeSummary::default();
let before = replication_skipped_count().await;
item.heal_replication(&null_object, &mut size_summary).await;
assert_eq!(replication_skipped_count().await, before);
item.heal_replication(
&ObjectInfo {
version_purge_status: VersionPurgeStatusType::Pending,
..null_object
},
&mut size_summary,
)
.await;
assert_eq!(replication_skipped_count().await, before + 1);
}
#[tokio::test]
async fn test_scanner_ilm_action_accounting_requires_enqueue_success() {
let metrics = Metrics::new();
@@ -4272,18 +4446,19 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_scan_folder_directory_budget_cancels_after_limit() {
async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let bucket_dir = temp_dir.join("bucket");
tokio::fs::create_dir_all(bucket_dir.join("child"))
tokio::fs::create_dir_all(bucket_dir.join(STORAGE_FORMAT_FILE))
.await
.expect("failed to create child directory");
.expect("failed to create xl.meta namespace directory");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.is_erasure_mode = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
@@ -4305,13 +4480,324 @@ mod tests {
let mut into = DataUsageEntry::default();
let result = scanner.scan_folder(ctx, folder, &mut into).await;
assert!(result.is_err(), "directory budget cancellation should make the scan partial");
assert!(
result.is_err(),
"an xl.meta namespace directory must be traversed instead of treated as object metadata"
);
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
assert!(budget.token().is_cancelled());
assert!(budget.entries_visited() >= 1);
}
#[tokio::test]
#[serial]
async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let object_dir = temp_dir.join("bucket").join("object");
let data_dir = object_dir.join(Uuid::new_v4().to_string());
tokio::fs::create_dir_all(&data_dir)
.await
.expect("failed to create erasure data directory");
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
tokio::fs::write(&metadata_path, b"")
.await
.expect("failed to create corrupt object metadata");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.is_erasure_mode = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = budget.clone();
let folder = CachedFolder {
name: "bucket/object".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut into = DataUsageEntry::default();
scanner
.scan_folder(budget.token(), folder, &mut into)
.await
.expect("failed metadata must not make scanner descend into erasure data directories");
assert_eq!(into.failed_objects, 1);
assert!(
scanner
.new_cache
.info
.failed_objects
.contains_key(metadata_path.to_string_lossy().as_ref())
);
assert!(!budget.budget_elapsed());
assert_eq!(budget.reason(), None);
let retry_budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = retry_budget.clone();
let retry_folder = CachedFolder {
name: "bucket/object".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut retry_into = DataUsageEntry::default();
scanner
.scan_folder(retry_budget.token(), retry_folder, &mut retry_into)
.await
.expect("cached metadata failure must still stop erasure data directory descent");
assert_eq!(retry_into.failed_objects, 0, "cached failure should not be counted twice");
assert!(!retry_budget.budget_elapsed());
assert_eq!(retry_budget.reason(), None);
#[cfg(unix)]
{
tokio::fs::remove_file(&metadata_path)
.await
.expect("failed to remove corrupt object metadata");
tokio::fs::write(data_dir.join("part.1"), b"shard")
.await
.expect("failed to create erasure shard");
scanner.new_cache.info.failed_objects.clear();
let metadata_target = temp_dir.join("metadata-target");
tokio::fs::create_dir(&metadata_target)
.await
.expect("failed to create metadata symlink target");
std::os::unix::fs::symlink(&metadata_target, &metadata_path).expect("failed to create metadata directory symlink");
let symlink_budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = symlink_budget.clone();
let symlink_folder = CachedFolder {
name: "bucket/object".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut symlink_into = DataUsageEntry::default();
scanner
.scan_folder(symlink_budget.token(), symlink_folder, &mut symlink_into)
.await
.expect("metadata symlink must still stop erasure data directory descent");
assert_eq!(symlink_into.failed_objects, 1);
assert!(
scanner
.new_cache
.info
.failed_objects
.contains_key(metadata_path.to_string_lossy().as_ref())
);
assert!(!symlink_budget.budget_elapsed());
assert_eq!(symlink_budget.reason(), None);
}
}
#[tokio::test]
#[serial]
async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let object_dir = temp_dir.join("bucket").join("object");
let data_dir = object_dir.join(Uuid::new_v4().to_string());
tokio::fs::create_dir_all(&data_dir)
.await
.expect("failed to create erasure data directory");
tokio::fs::write(data_dir.join("part.1"), b"shard")
.await
.expect("failed to create erasure shard");
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.is_erasure_mode = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = budget.clone();
let folder = CachedFolder {
name: "bucket/object".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut into = DataUsageEntry::default();
scanner
.scan_folder(budget.token(), folder, &mut into)
.await
.expect("missing metadata must not make scanner descend into erasure data directories");
assert_eq!(into.failed_objects, 1);
assert!(
scanner
.new_cache
.info
.failed_objects
.contains_key(metadata_path.to_string_lossy().as_ref())
);
assert!(!budget.budget_elapsed());
assert_eq!(budget.reason(), None);
let retry_budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = retry_budget.clone();
let retry_folder = CachedFolder {
name: "bucket/object".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut retry_into = DataUsageEntry::default();
scanner
.scan_folder(retry_budget.token(), retry_folder, &mut retry_into)
.await
.expect("cached missing metadata must still stop erasure data directory descent");
assert_eq!(retry_into.failed_objects, 0, "cached failure should not be counted twice");
assert!(!retry_budget.budget_elapsed());
assert_eq!(retry_budget.reason(), None);
}
#[tokio::test]
#[serial]
async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let namespace_name = Uuid::new_v4().to_string();
let namespace = temp_dir.join("bucket").join(&namespace_name);
tokio::fs::create_dir_all(namespace.join("part.1"))
.await
.expect("failed to create UUID namespace with part-like child directory");
let nil_uuid_namespace = temp_dir.join("bucket").join(Uuid::nil().to_string());
tokio::fs::create_dir_all(&nil_uuid_namespace)
.await
.expect("failed to create nil UUID namespace");
tokio::fs::write(nil_uuid_namespace.join("part.1"), b"namespace object")
.await
.expect("failed to create object in nil UUID namespace");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.is_erasure_mode = true;
let namespace_hash = hash_path(&format!("bucket/{namespace_name}"));
scanner.old_cache.replace_hashed(
&namespace_hash,
&Some(hash_path("bucket")),
&DataUsageEntry {
objects: 1,
size: 16,
..Default::default()
},
);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = budget.clone();
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut into = DataUsageEntry::default();
let result = scanner.scan_folder(budget.token(), folder, &mut into).await;
assert!(result.is_err(), "part.N directories and nil UUID namespaces must remain traversable");
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
assert!(scanner.new_cache.info.failed_objects.is_empty());
assert!(
scanner.update_cache.find(&namespace_hash.key()).is_some(),
"an existing UUID namespace must retain its cached subtree"
);
}
#[tokio::test]
#[serial]
async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let folder_path = temp_dir.join("bucket").join("object");
tokio::fs::create_dir_all(folder_path.join("child"))
.await
.expect("failed to create child namespace");
tokio::fs::write(folder_path.join(STORAGE_FORMAT_FILE), b"")
.await
.expect("failed to create metadata-shaped file");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.is_erasure_mode = false;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_directories: Some(1),
..Default::default()
},
);
scanner.budget = budget.clone();
let folder = CachedFolder {
name: "bucket/object".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut into = DataUsageEntry::default();
let result = scanner.scan_folder(budget.token(), folder, &mut into).await;
assert!(result.is_err(), "non-erasure scans must not stop at metadata-shaped files");
assert!(budget.budget_elapsed());
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn test_scan_folder_compacted_parent_sends_partial_update() {
@@ -63,6 +63,14 @@ pub enum RouteRiskLevel {
Normal,
Sensitive,
High,
/// A single authorized request can destroy stored data beyond every
/// recovery path the server offers — no undo, no waiting window, no
/// backup taken on the caller's behalf.
///
/// This is deliberately narrower than [`Self::High`], which covers routes
/// that change state an operator can put back. Reserve it for routes whose
/// worst case is permanent loss of user data.
Critical,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+1
View File
@@ -183,6 +183,7 @@ pub enum WalkVersionsSortOrder {
pub struct ObjectToDelete {
pub object_name: String,
pub version_id: Option<Uuid>,
pub synthetic_version_id: bool,
pub delete_marker_replication_status: Option<String>,
pub version_purge_status: Option<VersionPurgeStatusType>,
pub version_purge_statuses: Option<String>,