From dd11145a260225ba77e949275fabaf61ef0a4592 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 31 Jul 2026 10:56:44 +0800 Subject: [PATCH 1/8] feat(kms): record operation metrics in the retry policy engine (#5500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): record operation metrics in the retry policy engine Instrument policy::execute — the single choke point every outbound Vault call and credential exchange already flows through — so no call site needs its own instrumentation: - rustfs_kms_backend_operations_total (counter): operation, op_class, outcome (success / fatal / budget_exhausted / deadline_exceeded / cancelled) - rustfs_kms_backend_attempt_failures_total (counter): operation, error_class (retryable_conn / retryable_status / fatal / attempt_timeout) - rustfs_kms_backend_operation_duration_seconds (histogram): wall-clock duration including retries and backoff - rustfs_kms_backend_operation_attempts (histogram): attempts used Metric labels carry only static enum values (operation names, classes, outcomes) — never key identifiers, key material, ciphertext, or tokens. Emission goes through the process-global metrics facade recorder, the same pattern the rest of the workspace uses, so no new wiring is needed in rustfs/src. Tests drive a paused-clock runtime under a thread-local debugging recorder, so counts, attempts, and even the recorded (virtual-clock) durations are asserted deterministically with zero real sleeps. Refs rustfs/backlog#1569 (part of rustfs/backlog#1562) * test(kms): add Vault fault-injection matrix Offline cases inject transport faults locally and are fully deterministic: a refused connection is retried up to the configured budget, and a stalled connection is cut off by the per-attempt timeout instead of hanging. Ignored cases run against a real dev Vault (RUSTFS_KMS_VAULT_ADDR) and pin the fail-closed auth behavior: an invalid token and a missing key each resolve in exactly one attempt. Every case asserts through the policy metrics recorded by a thread-local debugging recorder, which doubles as the request-count assertion even against a real server. Throttling and recoverable 5xx responses cannot be forced on a stock dev Vault; those paths stay pinned by the scripted-Vault wiring tests and the engine tests. Refs rustfs/backlog#1569 (part of rustfs/backlog#1562) --- Cargo.lock | 2 + crates/kms/Cargo.toml | 4 + crates/kms/src/policy.rs | 530 +++++++++++++++++++++- crates/kms/tests/vault_fault_injection.rs | 255 +++++++++++ 4 files changed, 769 insertions(+), 22 deletions(-) create mode 100644 crates/kms/tests/vault_fault_injection.rs diff --git a/Cargo.lock b/Cargo.lock index 97584df3a..0d52e411b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9502,6 +9502,8 @@ dependencies = [ "insta", "jiff", "md-5 0.11.0", + "metrics", + "metrics-util", "moka", "rand 0.10.2", "reqwest", diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 3c46a63a3..33fa8e3bd 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -37,6 +37,8 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } tracing = { workspace = true } thiserror = { workspace = true } +# Operation metrics emitted by the retry policy engine (crate::policy). +metrics = { workspace = true } # Cryptography aes-gcm = { workspace = true, features = ["rand_core"] } @@ -72,6 +74,8 @@ tokio-util = { workspace = true } [dev-dependencies] anyhow = { workspace = true } +# Debugging recorder for asserting emitted metrics in tests. +metrics-util = { version = "0.20", features = ["debugging"] } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } diff --git a/crates/kms/src/policy.rs b/crates/kms/src/policy.rs index 58411a51b..bc7fcc664 100644 --- a/crates/kms/src/policy.rs +++ b/crates/kms/src/policy.rs @@ -27,6 +27,12 @@ //! retried automatically: a response lost after the server applied the write //! would otherwise be replayed into duplicate side effects (extra key versions, //! repeated deletes). +//! +//! Every execution also records operation metrics (attempt failures by retry +//! class, terminal outcome, attempts used, wall-clock duration) through the +//! process-global `metrics` recorder. Metric labels carry only static enum +//! values — operation names, classes, outcomes — never key identifiers, key +//! material, ciphertext, or tokens. use std::future::Future; use std::time::Duration; @@ -200,6 +206,130 @@ fn equal_jitter(rng: &mut impl RngExt, cap: Duration) -> Duration { half + Duration::from_nanos(rng.random_range(0..=spread)) } +// --------------------------------------------------------------------------- +// Metrics +// +// Every execution is recorded here, at the single choke point all backend +// calls flow through, so instrumenting a new call site costs nothing beyond +// naming its operation. Label values are exclusively static enum strings +// (operation names, classes, outcomes) — key identifiers, key material, +// ciphertext, and tokens must never reach a metric label. +// --------------------------------------------------------------------------- + +/// Counter: operations executed, by `operation`, `op_class`, and `outcome`. +const METRIC_OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; +/// Counter: failed attempts, by `operation` and `error_class` (including +/// `attempt_timeout` for attempts cut off by the per-attempt timeout). +const METRIC_ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; +/// Histogram: wall-clock duration of a whole operation (attempts plus +/// backoff), in seconds, by `operation` and `outcome`. +const METRIC_OPERATION_DURATION_SECONDS: &str = "rustfs_kms_backend_operation_duration_seconds"; +/// Histogram: attempts one operation used before completing, by `operation` +/// and `outcome`. +const METRIC_OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts"; + +impl OpClass { + fn as_label(self) -> &'static str { + match self { + OpClass::ReadIdempotent => "read_idempotent", + OpClass::MutatingNonIdempotent => "mutating_non_idempotent", + OpClass::Auth => "auth", + } + } +} + +impl ErrorClass { + fn as_label(self) -> &'static str { + match self { + ErrorClass::RetryableConn => "retryable_conn", + ErrorClass::RetryableStatus => "retryable_status", + ErrorClass::Fatal => "fatal", + } + } +} + +/// How one policy execution terminated, for the `outcome` metric label. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Outcome { + Success, + /// A fatal-classified failure ended the operation on its first observation. + Fatal, + /// The attempt budget ran out; the last failure was retryable (including a + /// timed-out final attempt). + BudgetExhausted, + /// The operation deadline ran out before another attempt could complete. + DeadlineExceeded, + Cancelled, +} + +impl Outcome { + fn as_label(self) -> &'static str { + match self { + Outcome::Success => "success", + Outcome::Fatal => "fatal", + Outcome::BudgetExhausted => "budget_exhausted", + Outcome::DeadlineExceeded => "deadline_exceeded", + Outcome::Cancelled => "cancelled", + } + } +} + +/// Register metric descriptions once per process. +fn describe_metrics() { + static DESCRIBE: std::sync::Once = std::sync::Once::new(); + DESCRIBE.call_once(|| { + metrics::describe_counter!( + METRIC_OPERATIONS_TOTAL, + "Total KMS backend operations executed under the operation policy, by operation, operation class, and outcome" + ); + metrics::describe_counter!( + METRIC_ATTEMPT_FAILURES_TOTAL, + "Total failed KMS backend attempts, by operation and retry classification" + ); + metrics::describe_histogram!( + METRIC_OPERATION_DURATION_SECONDS, + "Wall-clock duration of KMS backend operations including retries and backoff, in seconds" + ); + metrics::describe_histogram!( + METRIC_OPERATION_ATTEMPTS, + "Number of attempts a KMS backend operation used before completing" + ); + }); +} + +/// Record one failed attempt with its retry classification. +fn record_attempt_failure(operation: &'static str, error_class: &'static str) { + metrics::counter!( + METRIC_ATTEMPT_FAILURES_TOTAL, + "operation" => operation, + "error_class" => error_class + ) + .increment(1); +} + +/// Record the terminal outcome of one policy execution. +fn record_operation(operation: &'static str, class: OpClass, outcome: Outcome, attempts: u32, elapsed: Duration) { + metrics::counter!( + METRIC_OPERATIONS_TOTAL, + "operation" => operation, + "op_class" => class.as_label(), + "outcome" => outcome.as_label() + ) + .increment(1); + metrics::histogram!( + METRIC_OPERATION_DURATION_SECONDS, + "operation" => operation, + "outcome" => outcome.as_label() + ) + .record(elapsed.as_secs_f64()); + metrics::histogram!( + METRIC_OPERATION_ATTEMPTS, + "operation" => operation, + "outcome" => outcome.as_label() + ) + .record(f64::from(attempts)); +} + /// Run `attempt` under the policy. /// /// Each attempt is bounded by `attempt_timeout` (further capped by whatever is @@ -230,13 +360,37 @@ where /// [`execute`] with an injectable jitter source so tests can pin deterministic /// backoff durations instead of asserting around random sleeps. pub(crate) async fn execute_with_jitter( + operation: &'static str, + class: OpClass, + policy: &RetryPolicy, + cancel: &CancellationToken, + jitter: J, + attempt: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, + J: FnMut(Duration) -> Duration, +{ + describe_metrics(); + let started = Instant::now(); + let mut attempts_made = 0u32; + let (outcome, result) = drive_attempts(operation, class, policy, cancel, jitter, attempt, &mut attempts_made).await; + record_operation(operation, class, outcome, attempts_made, started.elapsed()); + result +} + +/// The attempt loop behind [`execute_with_jitter`], returning the terminal +/// outcome alongside the result so the caller can record it exactly once. +async fn drive_attempts( operation: &'static str, class: OpClass, policy: &RetryPolicy, cancel: &CancellationToken, mut jitter: J, mut attempt: F, -) -> Result + attempts_made: &mut u32, +) -> (Outcome, Result) where F: FnMut() -> Fut, Fut: Future>, @@ -247,48 +401,68 @@ where let mut attempt_no = 0u32; loop { - attempt_no += 1; if cancel.is_cancelled() { - return Err(KmsError::operation_cancelled(format!( - "{operation} cancelled before attempt {attempt_no}" - ))); + return ( + Outcome::Cancelled, + Err(KmsError::operation_cancelled(format!( + "{operation} cancelled before attempt {}", + attempt_no + 1 + ))), + ); } let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { - return Err(KmsError::operation_timed_out(format!( - "{operation} exceeded operation deadline of {:?}", - policy.op_deadline - ))); + return ( + Outcome::DeadlineExceeded, + Err(KmsError::operation_timed_out(format!( + "{operation} exceeded operation deadline of {:?}", + policy.op_deadline + ))), + ); } + attempt_no += 1; + *attempts_made = attempt_no; let attempt_budget = policy.attempt_timeout.min(remaining); let outcome = tokio::select! { biased; _ = cancel.cancelled() => { - return Err(KmsError::operation_cancelled(format!("{operation} cancelled during attempt {attempt_no}"))); + return ( + Outcome::Cancelled, + Err(KmsError::operation_cancelled(format!("{operation} cancelled during attempt {attempt_no}"))), + ); } outcome = tokio::time::timeout(attempt_budget, attempt()) => outcome, }; let failure = match outcome { - Ok(Ok(value)) => return Ok(value), - Ok(Err(failure)) => failure, - Err(_) => AttemptError { - class: ErrorClass::RetryableConn, - error: KmsError::operation_timed_out(format!( - "{operation} attempt {attempt_no} timed out after {attempt_budget:?}" - )), - }, + Ok(Ok(value)) => return (Outcome::Success, Ok(value)), + Ok(Err(failure)) => { + record_attempt_failure(operation, failure.class.as_label()); + failure + } + Err(_) => { + record_attempt_failure(operation, "attempt_timeout"); + AttemptError { + class: ErrorClass::RetryableConn, + error: KmsError::operation_timed_out(format!( + "{operation} attempt {attempt_no} timed out after {attempt_budget:?}" + )), + } + } }; - if failure.class == ErrorClass::Fatal || attempt_no >= max_attempts { - return Err(failure.error); + if failure.class == ErrorClass::Fatal { + return (Outcome::Fatal, Err(failure.error)); + } + if attempt_no >= max_attempts { + return (Outcome::BudgetExhausted, Err(failure.error)); } let backoff = jitter(backoff_cap(policy, attempt_no)); if backoff >= deadline.saturating_duration_since(Instant::now()) { // Not enough deadline budget left for another attempt. - return Err(failure.error); + return (Outcome::DeadlineExceeded, Err(failure.error)); } tracing::warn!( operation, @@ -300,7 +474,10 @@ where tokio::select! { biased; _ = cancel.cancelled() => { - return Err(KmsError::operation_cancelled(format!("{operation} cancelled during retry backoff"))); + return ( + Outcome::Cancelled, + Err(KmsError::operation_cancelled(format!("{operation} cancelled during retry backoff"))), + ); } _ = tokio::time::sleep(backoff) => {} } @@ -628,4 +805,313 @@ mod tests { assert_eq!(classify_vaultrs(&ClientError::ResponseEmptyError), ErrorClass::Fatal); assert_eq!(classify_vaultrs(&ClientError::InvalidLoginMethodError), ErrorClass::Fatal); } + + // -- Metric emission ---------------------------------------------------- + // + // Each test installs a thread-local debugging recorder and drives a + // paused-clock current-thread runtime inside it, so the emitted metrics + // (including virtual-clock durations) are fully deterministic. + + use metrics_util::MetricKind; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + type MetricEntry = ( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + ); + + /// Run `test` on a paused current-thread runtime under a debugging + /// recorder and return one snapshot of everything it emitted. + /// + /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains + /// the recorded state, so taking it per assertion would only show the + /// first assertion any data. + fn record_metrics(test: impl FnOnce() -> std::pin::Pin>>) -> (Vec, Out) { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let out = metrics::with_local_recorder(&recorder, || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .start_paused(true) + .build() + .expect("current-thread runtime must build"); + runtime.block_on(test()) + }); + (snapshotter.snapshot().into_vec(), out) + } + + fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool { + labels + .iter() + .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected)) + } + + fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 { + snapshot + .iter() + .filter_map(|(composite, _unit, _description, value)| { + let matches = composite.kind() == MetricKind::Counter + && composite.key().name() == name + && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Counter(count)) => Some(*count), + _ => None, + } + }) + .sum() + } + + fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec { + snapshot + .iter() + .filter_map(|(composite, _unit, _description, value)| { + let matches = composite.kind() == MetricKind::Histogram + && composite.key().name() == name + && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Histogram(values)) => Some(values), + _ => None, + } + }) + .flatten() + .map(|value| value.into_inner()) + .collect() + } + + #[test] + fn metrics_record_retried_success_with_attempts_and_duration() { + let calls_in_test = Arc::new(AtomicU32::new(0)); + let (snapshot, ()) = record_metrics(move || { + Box::pin(async move { + let policy = policy_of(1_000, 60_000, 3, 100, 2_000); + let cancel = CancellationToken::new(); + let calls_in_attempt = calls_in_test.clone(); + execute_with_jitter("metrics_read", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + if calls.fetch_add(1, Ordering::SeqCst) < 2 { + Err(AttemptError { + class: ErrorClass::RetryableStatus, + error: KmsError::backend_error("throttled (429)"), + }) + } else { + Ok(()) + } + } + }) + .await + .expect("retries within budget must succeed"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[ + ("operation", "metrics_read"), + ("op_class", "read_idempotent"), + ("outcome", "success") + ] + ), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + METRIC_ATTEMPT_FAILURES_TOTAL, + &[("operation", "metrics_read"), ("error_class", "retryable_status")] + ), + 2 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_read"), ("outcome", "success")] + ), + vec![3.0] + ); + // Full-cap backoffs of 100ms and 200ms on the paused clock. + let durations = histogram_values( + &snapshot, + METRIC_OPERATION_DURATION_SECONDS, + &[("operation", "metrics_read"), ("outcome", "success")], + ); + assert_eq!(durations.len(), 1); + assert!((durations[0] - 0.3).abs() < 1e-9, "expected 0.3s of virtual backoff, got {durations:?}"); + } + + #[test] + fn metrics_record_fatal_outcome_with_single_attempt() { + let (snapshot, ()) = record_metrics(|| { + Box::pin(async { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let result: Result<()> = + execute_with_jitter("metrics_fatal", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, || async { + Err(AttemptError { + class: ErrorClass::Fatal, + error: KmsError::access_denied("permission denied (403)"), + }) + }) + .await; + result.expect_err("a fatal failure must end the operation"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[("operation", "metrics_fatal"), ("outcome", "fatal")] + ), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + METRIC_ATTEMPT_FAILURES_TOTAL, + &[("operation", "metrics_fatal"), ("error_class", "fatal")] + ), + 1 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_fatal"), ("outcome", "fatal")] + ), + vec![1.0] + ); + } + + #[test] + fn metrics_record_mutating_budget_exhausted_after_one_attempt() { + let (snapshot, ()) = record_metrics(|| { + Box::pin(async { + let policy = policy_of(1_000, 60_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let result: Result<()> = execute_with_jitter( + "metrics_rotate", + OpClass::MutatingNonIdempotent, + &policy, + &cancel, + full_jitter, + || async { Err(retryable_conn_error()) }, + ) + .await; + result.expect_err("a mutating operation must not retry a retryable failure"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[ + ("operation", "metrics_rotate"), + ("op_class", "mutating_non_idempotent"), + ("outcome", "budget_exhausted") + ] + ), + 1 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_rotate"), ("outcome", "budget_exhausted")] + ), + vec![1.0] + ); + } + + #[test] + fn metrics_record_timeouts_and_deadline_outcome() { + let (snapshot, ()) = record_metrics(|| { + Box::pin(async { + // Hung attempts: 10s each against a 25s deadline (see + // total_duration_never_exceeds_deadline for the timeline). + let policy = policy_of(10_000, 25_000, 5, 100, 2_000); + let cancel = CancellationToken::new(); + let result: Result<()> = + execute_with_jitter("metrics_hung", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, || { + std::future::pending::>() + }) + .await; + result.expect_err("hung attempts must exhaust the deadline"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[("operation", "metrics_hung"), ("outcome", "deadline_exceeded")] + ), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + METRIC_ATTEMPT_FAILURES_TOTAL, + &[("operation", "metrics_hung"), ("error_class", "attempt_timeout")] + ), + 3 + ); + let durations = histogram_values( + &snapshot, + METRIC_OPERATION_DURATION_SECONDS, + &[("operation", "metrics_hung"), ("outcome", "deadline_exceeded")], + ); + assert_eq!(durations.len(), 1); + assert!((durations[0] - 25.0).abs() < 1e-9, "expected the full 25s deadline, got {durations:?}"); + } + + #[test] + fn metrics_record_cancelled_outcome() { + let calls_in_test = Arc::new(AtomicU32::new(0)); + let (snapshot, ()) = record_metrics(move || { + Box::pin(async move { + let policy = policy_of(1_000, 600_000, 5, 10_000, 10_000); + let cancel = CancellationToken::new(); + let canceller = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(500)).await; + canceller.cancel(); + }); + let calls_in_attempt = calls_in_test.clone(); + let result: Result<()> = + execute_with_jitter("metrics_cancel", OpClass::ReadIdempotent, &policy, &cancel, full_jitter, move || { + let calls = calls_in_attempt.clone(); + async move { + calls.fetch_add(1, Ordering::SeqCst); + Err(retryable_conn_error()) + } + }) + .await; + result.expect_err("cancellation must abort the backoff"); + }) + }); + + assert_eq!( + counter_value( + &snapshot, + METRIC_OPERATIONS_TOTAL, + &[("operation", "metrics_cancel"), ("outcome", "cancelled")] + ), + 1 + ); + assert_eq!( + histogram_values( + &snapshot, + METRIC_OPERATION_ATTEMPTS, + &[("operation", "metrics_cancel"), ("outcome", "cancelled")] + ), + vec![1.0] + ); + } } diff --git a/crates/kms/tests/vault_fault_injection.rs b/crates/kms/tests/vault_fault_injection.rs new file mode 100644 index 000000000..14176fdb9 --- /dev/null +++ b/crates/kms/tests/vault_fault_injection.rs @@ -0,0 +1,255 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Fault-injection matrix for the Vault backend operation policy. +//! +//! Offline cases run against locally injected transport faults (a closed +//! port, a listener that never responds) — deterministic, no external +//! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault +//! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`). +//! +//! Throttling (429) and recoverable 5xx responses cannot be forced on a stock +//! dev Vault, so their retry and metric behavior is pinned deterministically +//! by the scripted-Vault wiring tests in `backends::vault` and the engine +//! tests in `policy.rs`. Pointing `RUSTFS_KMS_VAULT_ADDR` at a +//! fault-injecting proxy reuses the ignored cases here unchanged. +//! +//! Every case installs a thread-local debugging metrics recorder and drives a +//! current-thread runtime inside it, so the policy metrics double as the +//! request-count assertion even against a real server. + +use std::time::Duration; + +use metrics_util::MetricKind; +use metrics_util::debugging::{DebugValue, DebuggingRecorder}; +use rustfs_kms::backends::KmsClient; +use rustfs_kms::backends::vault::VaultKmsClient; +use rustfs_kms::{KmsConfig, KmsError, VaultAuthMethod, VaultConfig}; + +const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; +const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; + +fn vault_config(address: &str, token: &str) -> VaultConfig { + VaultConfig { + address: address.to_string(), + auth_method: VaultAuthMethod::Token { + token: token.to_string(), + }, + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/fault-injection".to_string(), + tls: None, + } +} + +fn kms_config(attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig { + KmsConfig { + timeout: attempt_timeout, + retry_attempts, + ..KmsConfig::default() + } +} + +type MetricEntry = ( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, +); + +/// Run `test` on a current-thread runtime under a debugging metrics recorder +/// and return one snapshot of everything it emitted. +/// +/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains the +/// recorded state, so taking it per assertion would only show the first +/// assertion any data. +fn record_metrics(test: impl FnOnce() -> std::pin::Pin>>) -> Vec { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime must build"); + runtime.block_on(test()); + }); + snapshotter.snapshot().into_vec() +} + +/// Sum of counters with `name` whose labels include every `(key, value)` pair. +fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 { + snapshot + .iter() + .filter_map(|(composite, _unit, _description, value)| { + let key = composite.key(); + let matches = composite.kind() == MetricKind::Counter + && key.name() == name + && labels + .iter() + .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected)); + match (matches, value) { + (true, DebugValue::Counter(count)) => Some(*count), + _ => None, + } + }) + .sum() +} + +/// Connection refused: connection-class failures are retried up to the +/// configured budget, then surface as a backend error. +#[test] +fn connection_refused_is_retried_within_budget() { + // Reserve a loopback port and release it so nothing is listening there. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve a loopback port"); + let address = format!("http://{}", listener.local_addr().expect("reserved port addr")); + drop(listener); + + let snapshot = record_metrics(|| { + Box::pin(async move { + let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_secs(2), 2)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-refused", None) + .await + .expect_err("a refused connection must fail the operation"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + }) + }); + + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]), + 2, + "both budgeted attempts must observe the refused connection" + ); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1); + // The static-token login records its own success; the Vault read must not. + assert_eq!( + counter_value( + &snapshot, + OPERATIONS_TOTAL, + &[("operation", "vault_kv2_read_key"), ("outcome", "success")] + ), + 0 + ); +} + +/// Stalled connection: a server that accepts but never responds is cut off by +/// the per-attempt timeout (either the policy timer or the equally sized HTTP +/// client timeout, whichever fires first) instead of hanging forever. +#[test] +fn stalled_connection_is_cut_off_by_the_attempt_timeout() { + let snapshot = record_metrics(|| { + Box::pin(async move { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind stall listener"); + let address = format!("http://{}", listener.local_addr().expect("stall listener addr")); + // Accept and park every connection without ever responding. + tokio::spawn(async move { + let mut parked = Vec::new(); + loop { + let Ok((socket, _)) = listener.accept().await else { return }; + parked.push(socket); + } + }); + + let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_millis(250), 1)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-stalled", None) + .await + .expect_err("a stalled request must be cut off by the attempt timeout"); + assert!( + matches!(error, KmsError::OperationTimedOut { .. } | KmsError::BackendError { .. }), + "got {error:?}" + ); + }) + }); + + // The policy timer reports attempt_timeout; the client-level HTTP timeout + // surfaces as a connection-class failure. Either way it is exactly one + // attempt that was cut off. + let cut_off = counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "attempt_timeout")]) + + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]); + assert_eq!(cut_off, 1, "the single budgeted attempt must be cut off by a timeout"); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1); +} + +fn real_vault_address() -> String { + std::env::var("RUSTFS_KMS_VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()) +} + +/// Invalid token against a real Vault: the 403 is fatal — exactly one +/// attempt, no retry, and the operation fails closed. +#[test] +#[ignore] // Requires a running Vault dev server +fn real_vault_invalid_token_is_fatal_and_never_retried() { + let snapshot = record_metrics(|| { + Box::pin(async { + let config = vault_config(&real_vault_address(), "fault-injection-invalid-token"); + let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-forbidden", None) + .await + .expect_err("an invalid token must be rejected"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + }) + }); + + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]), + 1, + "a 403 must be observed by exactly one attempt" + ); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1); + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_status")]), + 0, + "an auth failure must never be classified as retryable" + ); +} + +/// Healthy read against a real Vault: a missing key resolves in one attempt +/// (404 is fatal for retry purposes) and records a fatal outcome rather than +/// burning the retry budget. +#[test] +#[ignore] // Requires a running Vault dev server +fn real_vault_missing_key_is_resolved_in_one_attempt() { + let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-only-token".to_string()); + let snapshot = record_metrics(|| { + Box::pin(async move { + let config = vault_config(&real_vault_address(), &token); + let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + .await + .expect("client construction performs no network calls"); + let error = client + .describe_key("fault-injection-definitely-missing", None) + .await + .expect_err("a missing key must resolve to key-not-found"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}"); + }) + }); + + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]), + 1, + "a 404 must be observed by exactly one attempt" + ); + assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1); +} From 78d6918c526c031c85935741c77a360a97e9762f Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 31 Jul 2026 13:42:19 +0800 Subject: [PATCH 2/8] feat: extend hotpath coverage across crates (#5505) Add opt-in hotpath feature surfaces to every workspace crate and wire the root rustfs feature passthrough for function, allocation, and CPU profiling. Add a focused set of function-level measurements for scanner, heal, lock, target replay, IAM, KMS, Keystone, trusted proxy, and capacity paths without adding request-scoped primitive wrappers. Co-authored-by: heihutu --- Cargo.lock | 42 +++++++ crates/audit/Cargo.toml | 26 ++++ crates/checksums/Cargo.toml | 7 ++ crates/common/Cargo.toml | 7 ++ crates/concurrency/Cargo.toml | 7 ++ crates/config/Cargo.toml | 4 + crates/credentials/Cargo.toml | 7 ++ crates/crypto/Cargo.toml | 4 + crates/data-usage/Cargo.toml | 7 ++ crates/e2e_test/Cargo.toml | 48 ++++++++ crates/ecstore/Cargo.toml | 64 ++++++++++ crates/ecstore/src/lib.rs | 2 + crates/extension-schema/Cargo.toml | 7 ++ crates/filemeta/Cargo.toml | 6 +- crates/heal/Cargo.toml | 41 +++++++ crates/heal/src/heal/erasure_healer.rs | 1 + crates/heal/src/heal/task.rs | 3 + crates/iam/Cargo.toml | 48 ++++++++ crates/iam/src/store/object.rs | 2 + crates/io-core/Cargo.toml | 7 ++ crates/io-metrics/Cargo.toml | 25 ++++ crates/keystone/Cargo.toml | 27 +++++ crates/keystone/src/client.rs | 2 + crates/kms/Cargo.toml | 15 +++ crates/kms/src/manager.rs | 3 + crates/lifecycle/Cargo.toml | 28 +++++ crates/lock/Cargo.toml | 14 +++ crates/lock/src/distributed_lock.rs | 4 + crates/log-analyzer/Cargo.toml | 7 ++ crates/madmin/Cargo.toml | 7 ++ crates/notify/Cargo.toml | 31 +++++ crates/object-capacity/Cargo.toml | 26 ++++ .../object-capacity/src/capacity_manager.rs | 2 + crates/object-capacity/src/scan.rs | 1 + crates/object-data-cache/Cargo.toml | 7 ++ crates/obs/Cargo.toml | 45 +++++++ crates/policy/Cargo.toml | 24 +++- crates/protocols/Cargo.toml | 50 ++++++++ crates/protos/Cargo.toml | 31 +++++ crates/replication/Cargo.toml | 7 ++ crates/rio-v2/Cargo.toml | 25 ++++ crates/rio/Cargo.toml | 29 ++++- crates/s3-ops/Cargo.toml | 7 ++ crates/s3-types/Cargo.toml | 7 ++ crates/s3select-api/Cargo.toml | 30 +++++ crates/s3select-query/Cargo.toml | 13 ++ crates/scanner/Cargo.toml | 41 +++++++ crates/scanner/src/scanner.rs | 1 + crates/security-governance/Cargo.toml | 7 ++ crates/signer/Cargo.toml | 7 ++ crates/storage-api/Cargo.toml | 7 ++ crates/targets/Cargo.toml | 34 ++++++ crates/targets/src/runtime/mod.rs | 5 + crates/targets/src/target/webhook.rs | 2 + crates/test-utils/Cargo.toml | 25 ++++ crates/tls-runtime/Cargo.toml | 7 ++ crates/trusted-proxies/Cargo.toml | 13 ++ crates/trusted-proxies/src/cloud/ranges.rs | 1 + crates/utils/Cargo.toml | 4 + crates/zip/Cargo.toml | 7 ++ rustfs/Cargo.toml | 112 ++++++++++++++++++ 61 files changed, 1071 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d52e411b..416cbd5c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3672,6 +3672,7 @@ dependencies = [ "flate2", "futures", "hex", + "hotpath", "http 1.5.0", "http-body-util", "hyper", @@ -9039,6 +9040,7 @@ dependencies = [ "const-str", "futures", "hashbrown 0.17.1", + "hotpath", "metrics", "rustfs-config", "rustfs-s3-types", @@ -9059,6 +9061,7 @@ dependencies = [ "base64-simd", "bytes", "crc-fast", + "hotpath", "http 1.5.0", "md-5 0.11.0", "pretty_assertions", @@ -9072,6 +9075,7 @@ name = "rustfs-common" version = "1.0.0-beta.12" dependencies = [ "chrono", + "hotpath", "metrics", "rmp-serde", "s3s", @@ -9086,6 +9090,7 @@ dependencies = [ name = "rustfs-concurrency" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "insta", "rustfs-io-core", "serde", @@ -9099,6 +9104,7 @@ name = "rustfs-config" version = "1.0.0-beta.12" dependencies = [ "const-str", + "hotpath", "serde", "serde_json", ] @@ -9109,6 +9115,7 @@ version = "1.0.0-beta.12" dependencies = [ "base64-simd", "hmac 0.13.0", + "hotpath", "rand 0.10.2", "serde", "serde_json", @@ -9124,6 +9131,7 @@ dependencies = [ "argon2", "base64-simd", "chacha20poly1305", + "hotpath", "jsonwebtoken 11.0.0", "pbkdf2 0.13.0", "rand 0.10.2", @@ -9141,6 +9149,7 @@ name = "rustfs-data-usage" version = "1.0.0-beta.12" dependencies = [ "async-trait", + "hotpath", "rmp-serde", "rustfs-filemeta", "serde", @@ -9286,6 +9295,7 @@ dependencies = [ name = "rustfs-extension-schema" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "serde", "serde_json", "thiserror 2.0.19", @@ -9324,6 +9334,7 @@ dependencies = [ "async-trait", "base64 0.23.0", "futures", + "hotpath", "http 1.5.0", "metrics", "rustfs-common", @@ -9355,6 +9366,7 @@ dependencies = [ "async-trait", "base64-simd", "futures", + "hotpath", "http 1.5.0", "jsonwebtoken 11.0.0", "moka", @@ -9389,6 +9401,7 @@ name = "rustfs-io-core" version = "1.0.0-beta.12" dependencies = [ "bytes", + "hotpath", "memmap2", "rustfs-io-metrics", "thiserror 2.0.19", @@ -9401,6 +9414,7 @@ name = "rustfs-io-metrics" version = "1.0.0-beta.12" dependencies = [ "criterion", + "hotpath", "metrics", "metrics-util", "num_cpus", @@ -9467,6 +9481,7 @@ version = "1.0.0-beta.12" dependencies = [ "bytes", "futures", + "hotpath", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -9499,6 +9514,7 @@ dependencies = [ "base64 0.23.0", "chacha20poly1305", "hex", + "hotpath", "insta", "jiff", "md-5 0.11.0", @@ -9531,6 +9547,7 @@ name = "rustfs-lifecycle" version = "1.0.0-beta.12" dependencies = [ "async-trait", + "hotpath", "metrics", "metrics-util", "proptest", @@ -9555,6 +9572,7 @@ dependencies = [ "async-trait", "crossbeam-queue", "futures", + "hotpath", "parking_lot", "rand 0.10.2", "rustfs-io-metrics", @@ -9576,6 +9594,7 @@ version = "1.0.0-beta.12" dependencies = [ "chrono", "flate2", + "hotpath", "regex", "serde", "serde_json", @@ -9593,6 +9612,7 @@ name = "rustfs-madmin" version = "1.0.0-beta.12" dependencies = [ "chrono", + "hotpath", "humantime", "hyper", "rmp-serde", @@ -9613,6 +9633,7 @@ dependencies = [ "criterion", "form_urlencoded", "hashbrown 0.17.1", + "hotpath", "metrics", "percent-encoding", "quick-xml", @@ -9642,6 +9663,7 @@ version = "1.0.0-beta.12" dependencies = [ "criterion", "futures", + "hotpath", "rustfs-config", "rustfs-io-metrics", "rustfs-utils", @@ -9660,6 +9682,7 @@ version = "1.0.0-beta.12" dependencies = [ "bytes", "criterion", + "hotpath", "metrics", "metrics-util", "moka", @@ -9682,6 +9705,7 @@ dependencies = [ "flate2", "futures-util", "glob", + "hotpath", "jiff", "libc", "metrics", @@ -9767,6 +9791,7 @@ dependencies = [ "futures-util", "hex", "hmac 0.13.0", + "hotpath", "http 1.5.0", "http-body-util", "hyper", @@ -9819,6 +9844,7 @@ name = "rustfs-protos" version = "1.0.0-beta.12" dependencies = [ "flatbuffers", + "hotpath", "prost 0.14.4", "rmp-serde", "rustfs-common", @@ -9843,6 +9869,7 @@ version = "1.0.0-beta.12" dependencies = [ "byteorder", "bytes", + "hotpath", "regex", "rmp", "rmp-serde", @@ -9901,6 +9928,7 @@ dependencies = [ "chacha20poly1305", "hex", "hmac 0.13.0", + "hotpath", "minlz", "pin-project-lite", "rand 0.10.2", @@ -9918,6 +9946,7 @@ dependencies = [ name = "rustfs-s3-ops" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "rustfs-s3-types", ] @@ -9925,6 +9954,7 @@ dependencies = [ name = "rustfs-s3-types" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "serde", "serde_json", ] @@ -9939,6 +9969,7 @@ dependencies = [ "datafusion", "futures", "futures-core", + "hotpath", "http 1.5.0", "metrics", "parking_lot", @@ -9967,6 +9998,7 @@ dependencies = [ "datafusion", "derive_builder", "futures", + "hotpath", "parking_lot", "rustfs-s3select-api", "s3s", @@ -9984,6 +10016,7 @@ dependencies = [ "futures", "hex-simd", "hmac 0.13.0", + "hotpath", "http 1.5.0", "metrics", "rand 0.10.2", @@ -10016,6 +10049,7 @@ dependencies = [ name = "rustfs-security-governance" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "thiserror 2.0.19", ] @@ -10025,6 +10059,7 @@ version = "1.0.0-beta.12" dependencies = [ "base64-simd", "bytes", + "hotpath", "http 1.5.0", "hyper", "rustfs-utils", @@ -10041,6 +10076,7 @@ name = "rustfs-storage-api" version = "1.0.0-beta.12" dependencies = [ "async-trait", + "hotpath", "insta", "rustfs-filemeta", "serde", @@ -10062,6 +10098,7 @@ dependencies = [ "deadpool-postgres", "futures-util", "hashbrown 0.17.1", + "hotpath", "hyper", "hyper-rustls", "lapin", @@ -10107,6 +10144,7 @@ dependencies = [ name = "rustfs-test-utils" version = "1.0.0-beta.12" dependencies = [ + "hotpath", "rustfs-data-usage", "rustfs-ecstore", "rustfs-storage-api", @@ -10123,6 +10161,7 @@ name = "rustfs-tls-runtime" version = "1.0.0-beta.12" dependencies = [ "arc-swap", + "hotpath", "metrics", "rcgen", "rustfs-common", @@ -10144,6 +10183,7 @@ version = "1.0.0-beta.12" dependencies = [ "async-trait", "axum", + "hotpath", "http 1.5.0", "ipnetwork", "metrics", @@ -10190,6 +10230,7 @@ dependencies = [ "hex-simd", "highway", "hmac 0.13.0", + "hotpath", "http 1.5.0", "hyper", "local-ip-address", @@ -10222,6 +10263,7 @@ dependencies = [ "astral-tokio-tar", "async-compression", "criterion", + "hotpath", "tempfile", "thiserror 2.0.19", "tokio", diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml index d598bc281..421b2b3a2 100644 --- a/crates/audit/Cargo.toml +++ b/crates/audit/Cargo.toml @@ -25,7 +25,33 @@ documentation = "https://docs.rs/rustfs-audit/latest/rustfs_audit/" keywords = ["audit", "target", "management", "fan-out", "RustFS"] categories = ["web-programming", "development-tools", "asynchronous", "api-bindings"] +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-config/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-targets/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-targets/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-targets/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-targets = { workspace = true } rustfs-config = { workspace = true, features = ["audit", "server-config-model"] } rustfs-s3-types = { workspace = true } diff --git a/crates/checksums/Cargo.toml b/crates/checksums/Cargo.toml index 6af2106b1..0229dd3a8 100644 --- a/crates/checksums/Cargo.toml +++ b/crates/checksums/Cargo.toml @@ -28,7 +28,14 @@ documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/" [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } crc-fast = { workspace = true } http = { workspace = true } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 93fe21df7..df98511d6 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools", "data-structures"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } tonic = { workspace = true, features = ["gzip", "deflate"] } uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] } diff --git a/crates/concurrency/Cargo.toml b/crates/concurrency/Cargo.toml index 55eb20eb9..ebc7ae095 100644 --- a/crates/concurrency/Cargo.toml +++ b/crates/concurrency/Cargo.toml @@ -13,7 +13,14 @@ categories = ["concurrency", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"] + [dependencies] +hotpath.workspace = true # Internal crates rustfs-io-core = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 9dcd0ec2f..7b61b1c5f 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -25,6 +25,7 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "config"] [dependencies] +hotpath.workspace = true const-str = { workspace = true, optional = true, features = ["std", "proc"] } serde = { workspace = true, optional = true, features = ["derive"] } serde_json = { workspace = true, optional = true, features = ["raw_value"] } @@ -34,6 +35,9 @@ workspace = true [features] default = ["constants"] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] audit = ["dep:const-str", "constants"] constants = ["dep:const-str"] notify = ["dep:const-str", "constants"] diff --git a/crates/credentials/Cargo.toml b/crates/credentials/Cargo.toml index bc2eca866..605d257d5 100644 --- a/crates/credentials/Cargo.toml +++ b/crates/credentials/Cargo.toml @@ -24,7 +24,14 @@ description = "Credentials management utilities for RustFS, enabling secure hand keywords = ["rustfs", "Minio", "credentials", "authentication", "authorization"] categories = ["web-programming", "development-tools", "data-structures", "security"] +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true base64-simd = { workspace = true } hmac = { workspace = true } rand = { workspace = true, features = ["serde"] } diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index e7cdf3357..080db7364 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -29,6 +29,7 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/" workspace = true [dependencies] +hotpath.workspace = true aes-gcm = { workspace = true, optional = true, features = ["rand_core"] } argon2 = { workspace = true, optional = true } chacha20poly1305 = { workspace = true, optional = true } @@ -49,6 +50,9 @@ time = { workspace = true, features = ["parsing", "formatting", "macros", "serde [features] default = ["crypto", "fips"] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] fips = [] crypto = [ "dep:aes-gcm", diff --git a/crates/data-usage/Cargo.toml b/crates/data-usage/Cargo.toml index b239e3dd6..3e33b530e 100644 --- a/crates/data-usage/Cargo.toml +++ b/crates/data-usage/Cargo.toml @@ -27,7 +27,14 @@ categories = ["data-structures", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "rustfs-filemeta/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-filemeta/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"] + [dependencies] +hotpath.workspace = true serde = { workspace = true, features = ["derive"] } rmp-serde = { workspace = true } async-trait = { workspace = true } diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index 00ddbc47a..b0bc6ed2a 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -25,10 +25,58 @@ workspace = true [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-data-usage/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-filemeta/hotpath", + "rustfs-lock/hotpath", + "rustfs-madmin/hotpath", + "rustfs-protos/hotpath", + "rustfs-rio/hotpath", + "rustfs-signer/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", + "rustfs-lock/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-protos/hotpath-alloc", + "rustfs-rio/hotpath-alloc", + "rustfs-signer/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", + "rustfs-lock/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-protos/hotpath-cpu", + "rustfs-rio/hotpath-cpu", + "rustfs-signer/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] ftps = [] sftp = [] [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["constants"] } rustfs-credentials.workspace = true rustfs-ecstore.workspace = true diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 726c5f2a2..84ad18647 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -40,19 +40,83 @@ hotpath = [ "hotpath/async-channel", "hotpath/parking_lot", "hotpath/reqwest-0-13", + "rustfs-checksums/hotpath", + "rustfs-common/hotpath", + "rustfs-concurrency/hotpath", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-data-usage/hotpath", "rustfs-filemeta/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-lifecycle/hotpath", + "rustfs-lock/hotpath", + "rustfs-madmin/hotpath", + "rustfs-object-capacity/hotpath", + "rustfs-policy/hotpath", + "rustfs-protos/hotpath", + "rustfs-replication/hotpath", "rustfs-rio/hotpath", + "rustfs-rio-v2?/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-signer/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", + "rustfs-crypto/hotpath", ] hotpath-alloc = [ + "hotpath", "hotpath/hotpath-alloc", + "rustfs-checksums/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-concurrency/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", "rustfs-filemeta/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-lifecycle/hotpath-alloc", + "rustfs-lock/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-object-capacity/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-protos/hotpath-alloc", + "rustfs-replication/hotpath-alloc", "rustfs-rio/hotpath-alloc", + "rustfs-rio-v2?/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-signer/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", ] hotpath-cpu = [ "hotpath", "hotpath/hotpath-cpu", + "rustfs-checksums/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-concurrency/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", "rustfs-filemeta/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-lifecycle/hotpath-cpu", + "rustfs-lock/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-object-capacity/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-protos/hotpath-cpu", + "rustfs-replication/hotpath-cpu", "rustfs-rio/hotpath-cpu", + "rustfs-rio-v2?/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-signer/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", ] # Exposes shared lifecycle/tier test utilities (MockWarmBackend, fault # injection, xl.meta transition assertions) via `api::tier::test_util`. diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index e0c7d7b6d..2ed643ead 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![recursion_limit = "256"] + /// Scope-based hotpath measurement for `#[async_trait]` methods, where /// `#[hotpath::measure]` would only time the boxed-future construction. /// The guard records wall time from this statement until the enclosing diff --git a/crates/extension-schema/Cargo.toml b/crates/extension-schema/Cargo.toml index 84d9449a1..5e6af760d 100644 --- a/crates/extension-schema/Cargo.toml +++ b/crates/extension-schema/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools"] [lib] doctest = false +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true serde = { workspace = true, features = ["derive"] } thiserror.workspace = true diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index f1d2bbad2..78d6e1c1c 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -27,9 +27,9 @@ documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/" [features] default = [] -hotpath = ["hotpath/hotpath", "hotpath/tokio"] -hotpath-alloc = ["hotpath/hotpath-alloc"] -hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-utils/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-utils/hotpath-cpu"] [dependencies] hotpath.workspace = true diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index e213c7da5..668c4af24 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -29,7 +29,48 @@ categories = ["web-programming", "development-tools", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-common/hotpath", + "rustfs-concurrency/hotpath", + "rustfs-config/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-madmin/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-concurrency/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-concurrency/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true } rustfs-concurrency = { workspace = true } rustfs-ecstore = { workspace = true } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 2f4c6f87d..d08367f3f 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -159,6 +159,7 @@ impl ErasureSetHealer { /// execute erasure set heal with resume #[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))] + #[hotpath::measure] pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> { debug!( target: "rustfs::heal::erasure_healer", diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index dbd505bb3..8057f6967 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -584,6 +584,7 @@ impl HealTask { } #[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))] + #[hotpath::measure] pub async fn execute(&self) -> Result<()> { // update status and timestamps atomically to avoid race conditions let now = SystemTime::now(); @@ -759,6 +760,7 @@ impl HealTask { // specific heal implementation method #[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))] + #[hotpath::measure] async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -1404,6 +1406,7 @@ impl HealTask { self.heal_bucket_objects(bucket, prefix).await } + #[hotpath::measure] async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> { let mut continuation_token: Option = None; let mut scanned = 0u64; diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index 8c4a3c2aa..0ef8ef1e9 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -28,7 +28,55 @@ documentation = "https://docs.rs/rustfs-iam/latest/rustfs_iam/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-crypto/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-madmin/hotpath", + "rustfs-policy/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-credentials = { workspace = true } rustfs-config = { workspace = true, features = ["server-config-model"] } tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index f7e652118..03530084e 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -469,6 +469,7 @@ impl ObjectStore { }); } + #[hotpath::measure] async fn list_all_iamconfig_items(&self) -> Result>> { let (tx, mut rx) = mpsc::channel::(100); @@ -508,6 +509,7 @@ impl ObjectStore { Ok(res) } + #[hotpath::measure] async fn load_policy_doc_concurrent(&self, names: &[String], mode: LoadMode) -> Result> { let mut futures = Vec::with_capacity(names.len()); diff --git a/crates/io-core/Cargo.toml b/crates/io-core/Cargo.toml index 5d53c9288..9c309633b 100644 --- a/crates/io-core/Cargo.toml +++ b/crates/io-core/Cargo.toml @@ -27,7 +27,14 @@ categories = ["development-tools", "filesystem"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] } diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index d77725ae5..64eac650b 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -28,7 +28,32 @@ categories = ["development-tools", "filesystem"] name = "metrics_pipeline" harness = false +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-common/hotpath", + "rustfs-s3-ops/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-s3-ops/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-s3-ops/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true metrics = { workspace = true } rustfs-common = { workspace = true } rustfs-s3-ops = { workspace = true } diff --git a/crates/keystone/Cargo.toml b/crates/keystone/Cargo.toml index 50110221e..e67adf7f5 100644 --- a/crates/keystone/Cargo.toml +++ b/crates/keystone/Cargo.toml @@ -28,7 +28,34 @@ authors.workspace = true [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-credentials/hotpath", + "rustfs-policy/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } reqwest = { workspace = true, features = ["json"] } serde = { workspace = true, features = ["derive"] } diff --git a/crates/keystone/src/client.rs b/crates/keystone/src/client.rs index ffb5fcdd4..d5df169e8 100644 --- a/crates/keystone/src/client.rs +++ b/crates/keystone/src/client.rs @@ -95,6 +95,7 @@ impl KeystoneClient { } /// Validate a Keystone token + #[hotpath::measure] pub async fn validate_token(&self, token: &str) -> Result { match self.version { KeystoneVersion::V3 => self.validate_token_v3(token).await, @@ -238,6 +239,7 @@ impl KeystoneClient { } /// Get EC2 credentials for a user + #[hotpath::measure] pub async fn get_ec2_credentials(&self, user_id: &str, project_id: Option<&str>) -> Result> { let admin_token = self.get_admin_token().await?; diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 33fa8e3bd..1dd5b9385 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -28,6 +28,7 @@ categories = ["cryptography", "web-programming", "authentication"] workspace = true [dependencies] +hotpath.workspace = true # Core dependencies async-trait = { workspace = true } tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread", "sync", "time"] } @@ -84,3 +85,17 @@ tokio = { workspace = true, features = ["net", "test-util"] } [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/reqwest-0-13", + "rustfs-security-governance/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-security-governance/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-security-governance/hotpath-cpu", "rustfs-utils/hotpath-cpu"] diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 7ae6bc055..9d77e07b4 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -66,16 +66,19 @@ impl KmsManager { } /// Encrypt data with a master key + #[hotpath::measure] pub async fn encrypt(&self, request: EncryptRequest) -> Result { self.backend.encrypt(request).await } /// Decrypt data with a master key + #[hotpath::measure] pub async fn decrypt(&self, request: DecryptRequest) -> Result { self.backend.decrypt(request).await } /// Generate a data encryption key + #[hotpath::measure] pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result { self.backend.generate_data_key(request).await } diff --git a/crates/lifecycle/Cargo.toml b/crates/lifecycle/Cargo.toml index 43dfac6aa..8f5f63556 100644 --- a/crates/lifecycle/Cargo.toml +++ b/crates/lifecycle/Cargo.toml @@ -25,7 +25,35 @@ keywords = ["lifecycle", "storage", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-lifecycle/latest/rustfs_lifecycle/" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-replication/hotpath", + "rustfs-storage-api/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-replication/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-replication/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true async-trait.workspace = true metrics.workspace = true rustfs-common.workspace = true diff --git a/crates/lock/Cargo.toml b/crates/lock/Cargo.toml index 7b0993adb..dcaccf4a6 100644 --- a/crates/lock/Cargo.toml +++ b/crates/lock/Cargo.toml @@ -28,7 +28,21 @@ documentation = "https://docs.rs/rustfs-lock/latest/rustfs_lock/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "rustfs-io-metrics/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu", "rustfs-utils/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-io-metrics = { workspace = true } rustfs-utils = { workspace = true } async-trait.workspace = true diff --git a/crates/lock/src/distributed_lock.rs b/crates/lock/src/distributed_lock.rs index 97b8af06d..e3465d732 100644 --- a/crates/lock/src/distributed_lock.rs +++ b/crates/lock/src/distributed_lock.rs @@ -540,6 +540,7 @@ impl DistributedLock { } /// Acquire a lock and return a RAII guard + #[hotpath::measure] pub(crate) async fn acquire_guard(&self, request: &LockRequest) -> Result> { if self.clients.is_empty() { return Err(LockError::internal("No lock clients available")); @@ -626,6 +627,7 @@ impl DistributedLock { } /// Convenience: acquire exclusive lock as a guard + #[hotpath::measure] pub async fn lock_guard( &self, resource: ObjectKey, @@ -640,6 +642,7 @@ impl DistributedLock { } /// Convenience: acquire exclusive lock with expected contention logs suppressed + #[hotpath::measure] pub async fn lock_guard_quiet( &self, resource: ObjectKey, @@ -655,6 +658,7 @@ impl DistributedLock { } /// Convenience: acquire shared lock as a guard + #[hotpath::measure] pub async fn rlock_guard( &self, resource: ObjectKey, diff --git a/crates/log-analyzer/Cargo.toml b/crates/log-analyzer/Cargo.toml index b8830e4d0..2c0eebcb4 100644 --- a/crates/log-analyzer/Cargo.toml +++ b/crates/log-analyzer/Cargo.toml @@ -32,7 +32,14 @@ doctest = false name = "la-dump-anchors" path = "src/bin/la_dump_anchors.rs" +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true chrono = { workspace = true, features = ["serde"] } flate2 = { workspace = true } regex = { workspace = true } diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index e5b395735..2d704d2d4 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -28,7 +28,14 @@ documentation = "https://docs.rs/rustfs-madmin/latest/rustfs_madmin/" [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true chrono = { workspace = true, features = ["serde"] } humantime.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index e6808f364..ddbb5d6de 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -26,9 +26,40 @@ categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/" [features] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-config/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-s3-ops/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-targets/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-s3-ops/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-targets/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-s3-ops/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-targets/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] demo-examples = [] [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["notify", "server-config-model"] } rustfs-ecstore = { workspace = true } rustfs-s3-types = { workspace = true } diff --git a/crates/object-capacity/Cargo.toml b/crates/object-capacity/Cargo.toml index 0ae8161df..fd17815f4 100644 --- a/crates/object-capacity/Cargo.toml +++ b/crates/object-capacity/Cargo.toml @@ -34,7 +34,33 @@ harness = false [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-config/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["constants"] } rustfs-io-metrics = { workspace = true } rustfs-utils = { workspace = true, features = ["os"] } diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 876fa9be6..ed3107b9b 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -1035,6 +1035,7 @@ impl HybridCapacityManager { /// /// Joiners subscribe to the watch channel *before* releasing the mutex, which guarantees /// they cannot miss the completion notification even if the leader finishes very quickly. + #[hotpath::measure] pub async fn refresh_or_join(&self, source: DataSource, refresh_fn: F) -> Result where F: FnOnce() -> Fut, @@ -1142,6 +1143,7 @@ impl HybridCapacityManager { } /// Start a background refresh if one is not already in flight. + #[hotpath::measure] pub async fn spawn_refresh_if_needed(self: Arc, source: DataSource, refresh_fn: F) -> bool where F: FnOnce() -> Fut + Send + 'static, diff --git a/crates/object-capacity/src/scan.rs b/crates/object-capacity/src/scan.rs index 2021dd84b..62bb01b93 100644 --- a/crates/object-capacity/src/scan.rs +++ b/crates/object-capacity/src/scan.rs @@ -338,6 +338,7 @@ pub async fn select_capacity_refresh_disks( } } +#[hotpath::measure] pub async fn refresh_capacity_with_scope(disks: Vec, dirty_subset: bool) -> Result { let scan_started_at = Instant::now(); let report = calculate_data_dir_used_capacity_report(&disks) diff --git a/crates/object-data-cache/Cargo.toml b/crates/object-data-cache/Cargo.toml index a89ce6a50..53a7ad5d3 100644 --- a/crates/object-data-cache/Cargo.toml +++ b/crates/object-data-cache/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools"] [lib] doctest = false +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } metrics = { workspace = true } moka = { workspace = true, features = ["future"] } diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 6f42090e5..8b627c353 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -27,6 +27,50 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/" [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/crossbeam", + "rustfs-audit/hotpath", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-iam/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-notify/hotpath", + "rustfs-security-governance/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-audit/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-iam/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-notify/hotpath-alloc", + "rustfs-security-governance/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-audit/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-iam/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-notify/hotpath-cpu", + "rustfs-security-governance/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] # Tokio runtime-level telemetry. Requires a `--cfg tokio_unstable` build; the # build script fails the compile when that flag is missing. Off by default so # ordinary builds neither pay for nor depend on Tokio's unstable API. @@ -58,6 +102,7 @@ required-features = ["dial9"] workspace = true [dependencies] +hotpath.workspace = true rustfs-audit = { workspace = true } rustfs-common = { workspace = true } rustfs-config = { workspace = true, features = ["observability"] } diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml index b85ded8b5..1732045fb 100644 --- a/crates/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -30,9 +30,27 @@ workspace = true [features] default = [] -hotpath = ["hotpath/hotpath", "hotpath/reqwest-0-13"] -hotpath-alloc = ["hotpath/hotpath-alloc"] -hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] +hotpath = [ + "hotpath/hotpath", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-crypto/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", +] [dependencies] rustfs-credentials = { workspace = true } diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 2d53eca84..91da8efce 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -30,6 +30,55 @@ workspace = true [features] default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-ecstore?/hotpath", + "rustfs-iam/hotpath", + "rustfs-keystone?/hotpath", + "rustfs-policy/hotpath", + "rustfs-rio?/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-tls-runtime?/hotpath", + "rustfs-trusted-proxies?/hotpath", + "rustfs-utils/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-ecstore?/hotpath-alloc", + "rustfs-iam/hotpath-alloc", + "rustfs-keystone?/hotpath-alloc", + "rustfs-policy/hotpath-alloc", + "rustfs-rio?/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-tls-runtime?/hotpath-alloc", + "rustfs-trusted-proxies?/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-ecstore?/hotpath-cpu", + "rustfs-iam/hotpath-cpu", + "rustfs-keystone?/hotpath-cpu", + "rustfs-policy/hotpath-cpu", + "rustfs-rio?/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-tls-runtime?/hotpath-cpu", + "rustfs-trusted-proxies?/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"] swift = [ "dep:rustfs-keystone", @@ -62,6 +111,7 @@ webdav = ["dep:dav-server", "dep:hyper", "dep:hyper-util", "dep:http-body-util", sftp = ["dep:russh", "dep:russh-sftp", "dep:uuid", "dep:subtle", "dep:tokio-util", "dep:socket2"] [dependencies] +hotpath.workspace = true # Core RustFS dependencies rustfs-iam = { workspace = true } rustfs-credentials = { workspace = true } diff --git a/crates/protos/Cargo.toml b/crates/protos/Cargo.toml index 3f57bbc6b..27d723a48 100644 --- a/crates/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -32,7 +32,38 @@ workspace = true name = "gproto" path = "src/main.rs" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-common.workspace = true rustfs-io-metrics.workspace = true rustfs-config.workspace = true diff --git a/crates/replication/Cargo.toml b/crates/replication/Cargo.toml index 5dc9a1108..e5554b922 100644 --- a/crates/replication/Cargo.toml +++ b/crates/replication/Cargo.toml @@ -25,7 +25,14 @@ keywords = ["replication", "storage", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-replication/latest/rustfs_replication/" +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true bytes = { workspace = true, features = ["serde"] } byteorder.workspace = true regex.workspace = true diff --git a/crates/rio-v2/Cargo.toml b/crates/rio-v2/Cargo.toml index b03ac2b88..b941ed7fc 100644 --- a/crates/rio-v2/Cargo.toml +++ b/crates/rio-v2/Cargo.toml @@ -28,7 +28,32 @@ documentation = "https://docs.rs/rustfs-rio-v2/latest/rustfs_rio_v2/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-rio/hotpath", + "rustfs-utils/hotpath", + "rustfs-filemeta/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-rio/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-rio/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true aes-gcm = { workspace = true, features = ["rand_core"] } bytes = { workspace = true, features = ["serde"] } chacha20poly1305.workspace = true diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index 4d0f3916e..72bb71af0 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -30,9 +30,32 @@ workspace = true [features] default = [] -hotpath = ["hotpath/hotpath", "hotpath/tokio", "hotpath/futures", "hotpath/reqwest-0-13"] -hotpath-alloc = ["hotpath/hotpath-alloc"] -hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] [dependencies] hotpath.workspace = true diff --git a/crates/s3-ops/Cargo.toml b/crates/s3-ops/Cargo.toml index 170f423c3..23dbd432a 100644 --- a/crates/s3-ops/Cargo.toml +++ b/crates/s3-ops/Cargo.toml @@ -27,7 +27,14 @@ categories = ["data-structures"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "rustfs-s3-types/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3-types/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3-types/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-s3-types = { workspace = true } [lib] diff --git a/crates/s3-types/Cargo.toml b/crates/s3-types/Cargo.toml index 0d25caf4b..76c408594 100644 --- a/crates/s3-types/Cargo.toml +++ b/crates/s3-types/Cargo.toml @@ -27,7 +27,14 @@ categories = ["data-structures"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/s3select-api/Cargo.toml b/crates/s3select-api/Cargo.toml index 78edb9daf..4e850b62b 100644 --- a/crates/s3select-api/Cargo.toml +++ b/crates/s3select-api/Cargo.toml @@ -28,7 +28,37 @@ documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/ [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "rustfs-common/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-test-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true metrics = { workspace = true } async-trait.workspace = true bytes = { workspace = true, features = ["serde"] } diff --git a/crates/s3select-query/Cargo.toml b/crates/s3select-query/Cargo.toml index 5ce13b9fd..619997edb 100644 --- a/crates/s3select-query/Cargo.toml +++ b/crates/s3select-query/Cargo.toml @@ -28,7 +28,20 @@ documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_qu [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "rustfs-s3select-api/hotpath", +] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3select-api/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3select-api/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-s3select-api = { workspace = true } async-recursion = { workspace = true } async-trait.workspace = true diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 33dde7715..127604853 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -29,7 +29,48 @@ documentation = "https://docs.rs/rustfs-scanner/latest/rustfs_scanner/" [lints] workspace = true +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "rustfs-common/hotpath", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-data-usage/hotpath", + "rustfs-ecstore/hotpath", + "rustfs-filemeta/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-filemeta/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-filemeta/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["server-config-model"] } rustfs-common = { workspace = true } rustfs-credentials = { workspace = true } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 91f9e1597..95c5fb194 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -2531,6 +2531,7 @@ where } #[instrument(skip_all)] +#[hotpath::measure] async fn run_data_scanner_cycle( ctx: &CancellationToken, storeapi: &Arc, diff --git a/crates/security-governance/Cargo.toml b/crates/security-governance/Cargo.toml index f71e1b032..da0b48470 100644 --- a/crates/security-governance/Cargo.toml +++ b/crates/security-governance/Cargo.toml @@ -30,5 +30,12 @@ doctest = false [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true thiserror = { workspace = true } diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 24cb667eb..b13c03275 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -25,7 +25,14 @@ keywords = ["digital-signature", "verification", "integrity", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "cryptography"] documentation = "https://docs.rs/rustfs-signer/latest/rustfs_signer/" +[features] +default = [] +hotpath = ["hotpath/hotpath", "rustfs-utils/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-utils/hotpath-cpu"] + [dependencies] +hotpath.workspace = true tracing.workspace = true bytes = { workspace = true, features = ["serde"] } http.workspace = true diff --git a/crates/storage-api/Cargo.toml b/crates/storage-api/Cargo.toml index f67d34331..74b519d3d 100644 --- a/crates/storage-api/Cargo.toml +++ b/crates/storage-api/Cargo.toml @@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools", "filesystem"] [lib] doctest = false +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-filemeta/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-filemeta/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"] + [dependencies] +hotpath.workspace = true async-trait.workspace = true # Storage-facing replication contracts are isolated in src/replication.rs until # the underlying wire types can move without creating a replication/storage-api cycle. diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index bd0671a40..ae759eaf7 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -11,7 +11,41 @@ keywords = ["file-system", "notification", "target", "rustfs", "Minio"] categories = ["web-programming", "development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/futures", + "hotpath/parking_lot", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-extension-schema/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-extension-schema/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-utils/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-extension-schema/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-utils/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-config = { workspace = true, features = ["notify", "audit", "server-config-model"] } rustfs-extension-schema = { workspace = true } rustfs-tls-runtime = { workspace = true } diff --git a/crates/targets/src/runtime/mod.rs b/crates/targets/src/runtime/mod.rs index 3a235e069..7999a33ce 100644 --- a/crates/targets/src/runtime/mod.rs +++ b/crates/targets/src/runtime/mod.rs @@ -522,6 +522,7 @@ fn snapshot_from_delivery(target_id: TargetID, delivery: TargetDeliverySnapshot) } } +#[hotpath::measure] pub async fn init_target_and_optionally_start_replay( target: Box + Send + Sync>, on_replay_start: F, @@ -557,6 +558,7 @@ where Some((shared, cancel)) } +#[hotpath::measure] pub(crate) async fn prepare_target( target: Box + Send + Sync>, cancellation: Option<&CancellationToken>, @@ -598,6 +600,7 @@ where type ActivatedTarget = (SharedTarget, Option<(mpsc::Sender<()>, JoinHandle<()>)>); +#[hotpath::measure] pub async fn activate_targets_with_replay( targets: Vec + Send + Sync>>, mut activate_one: F, @@ -670,6 +673,7 @@ fn seed_interval_start(now: tokio::time::Instant, interval: Duration) -> tokio:: now.checked_sub(interval).unwrap_or(now) } +#[hotpath::measure] async fn stream_replay_worker( store: &mut (dyn Store + Send), target: SharedTarget, @@ -805,6 +809,7 @@ async fn stream_replay_worker( /// Returns `true` if a cancel signal was observed while processing (e.g. during /// retry backoff), so the caller can stop promptly instead of continuing to /// drain a store that a replacement worker may already own. +#[hotpath::measure] async fn process_replay_batch( store: &(dyn Store + Send), batch_keys: &mut Vec, diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index e39dc9d49..b7ac97f0b 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -85,6 +85,7 @@ fn classify_probe_error(err: &reqwest::Error) -> TargetHealthReason { TargetHealthReason::Unreachable } +#[hotpath::measure] async fn probe_health_url(client: &Client, health_check_url: &Url) -> TargetHealth { match tokio::time::timeout(WEBHOOK_HEALTH_TIMEOUT, client.head(health_check_url.as_str()).send()).await { Ok(Ok(_)) => TargetHealth::online(TargetHealthReason::Reachable), @@ -480,6 +481,7 @@ where build_queued_payload(event) } + #[hotpath::measure] async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { debug!( event = EVENT_WEBHOOK_DELIVERY_STATE, diff --git a/crates/test-utils/Cargo.toml b/crates/test-utils/Cargo.toml index 81d2a53c0..bf56d069a 100644 --- a/crates/test-utils/Cargo.toml +++ b/crates/test-utils/Cargo.toml @@ -25,7 +25,32 @@ keywords = ["testing", "storage", "rustfs", "Minio"] categories = ["development-tools", "filesystem"] documentation = "https://docs.rs/rustfs-test-utils/latest/rustfs_test_utils/" +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "rustfs-ecstore/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-data-usage/hotpath", +] +hotpath-alloc = [ + "hotpath", + "hotpath/hotpath-alloc", + "rustfs-ecstore/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", +] +hotpath-cpu = [ + "hotpath", + "hotpath/hotpath-cpu", + "rustfs-ecstore/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", +] + [dependencies] +hotpath.workspace = true rustfs-ecstore = { workspace = true } rustfs-storage-api = { workspace = true } tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } diff --git a/crates/tls-runtime/Cargo.toml b/crates/tls-runtime/Cargo.toml index fba5f3fc2..fb8920547 100644 --- a/crates/tls-runtime/Cargo.toml +++ b/crates/tls-runtime/Cargo.toml @@ -27,7 +27,14 @@ categories = ["network-programming", "web-programming", "development-tools"] [lints] workspace = true +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-common/hotpath", "rustfs-config/hotpath"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-common/hotpath-alloc", "rustfs-config/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-common/hotpath-cpu", "rustfs-config/hotpath-cpu"] + [dependencies] +hotpath.workspace = true rustfs-common.workspace = true rustfs-config.workspace = true arc-swap.workspace = true diff --git a/crates/trusted-proxies/Cargo.toml b/crates/trusted-proxies/Cargo.toml index 0a15e5a7f..1494d8a35 100644 --- a/crates/trusted-proxies/Cargo.toml +++ b/crates/trusted-proxies/Cargo.toml @@ -24,7 +24,20 @@ description = " RustFS Trusted Proxies module provides secure and efficient mana keywords = ["trusted-proxies", "network-security", "rustfs", "proxy-management"] categories = ["network-programming", "security", "web-programming"] +[features] +default = [] +hotpath = [ + "hotpath/hotpath", + "hotpath/tokio", + "hotpath/reqwest-0-13", + "rustfs-config/hotpath", + "rustfs-utils/hotpath", +] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-config/hotpath-alloc", "rustfs-utils/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-config/hotpath-cpu", "rustfs-utils/hotpath-cpu"] + [dependencies] +hotpath.workspace = true async-trait = { workspace = true } axum = { workspace = true } http = { workspace = true } diff --git a/crates/trusted-proxies/src/cloud/ranges.rs b/crates/trusted-proxies/src/cloud/ranges.rs index cd3b8e4e0..dbf814674 100644 --- a/crates/trusted-proxies/src/cloud/ranges.rs +++ b/crates/trusted-proxies/src/cloud/ranges.rs @@ -227,6 +227,7 @@ pub struct GoogleCloudIpRanges; impl GoogleCloudIpRanges { /// Fetches the latest Google Cloud IP ranges from their official source. + #[hotpath::measure] pub async fn fetch() -> Result, AppError> { let client = Client::builder() .timeout(Duration::from_secs(10)) diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 5ba901965..2e3c7c6d8 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -25,6 +25,7 @@ keywords = ["utilities", "hashing", "compression", "network", "rustfs"] categories = ["web-programming", "development-tools", "cryptography"] [dependencies] +hotpath.workspace = true base64-simd = { workspace = true, optional = true } blake2 = { workspace = true, optional = true } brotli = { workspace = true, optional = true } @@ -76,6 +77,9 @@ workspace = true [features] default = ["ip"] # features that are enabled by default +hotpath = ["hotpath/hotpath", "hotpath/tokio", "hotpath/futures", "hotpath/reqwest-0-13"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] ip = ["dep:local-ip-address"] # ip characteristics and their dependencies net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:hyper", "dep:tokio"] # network features with DNS resolver egress = ["ip", "dep:reqwest", "dep:tokio", "dep:url"] diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 8348d3189..54cdd9547 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -32,7 +32,14 @@ doctest = false name = "zip_benchmark" harness = false +[features] +default = [] +hotpath = ["hotpath/hotpath", "hotpath/tokio"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] + [dependencies] +hotpath.workspace = true async-compression = { workspace = true, features = [ "tokio", "bzip2", diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 53db5ca19..24a18a938 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -60,25 +60,137 @@ hotpath = [ "hotpath/crossbeam", "hotpath/parking_lot", "hotpath/reqwest-0-13", + "rustfs-audit/hotpath", + "rustfs-common/hotpath", + "rustfs-concurrency/hotpath", + "rustfs-config/hotpath", + "rustfs-credentials/hotpath", + "rustfs-crypto/hotpath", + "rustfs-data-usage/hotpath", "rustfs-ecstore/hotpath", + "rustfs-extension-schema/hotpath", "rustfs-filemeta/hotpath", + "rustfs-heal/hotpath", + "rustfs-iam/hotpath", + "rustfs-io-core/hotpath", + "rustfs-io-metrics/hotpath", + "rustfs-keystone/hotpath", + "rustfs-kms/hotpath", + "rustfs-lock/hotpath", + "rustfs-log-analyzer/hotpath", + "rustfs-madmin/hotpath", + "rustfs-notify/hotpath", + "rustfs-object-capacity/hotpath", + "rustfs-object-data-cache/hotpath", + "rustfs-obs/hotpath", "rustfs-policy/hotpath", + "rustfs-protocols/hotpath", + "rustfs-protos/hotpath", "rustfs-rio/hotpath", + "rustfs-s3-ops/hotpath", + "rustfs-s3-types/hotpath", + "rustfs-s3select-api/hotpath", + "rustfs-s3select-query/hotpath", + "rustfs-scanner/hotpath", + "rustfs-security-governance/hotpath", + "rustfs-signer/hotpath", + "rustfs-storage-api/hotpath", + "rustfs-targets/hotpath", + "rustfs-tls-runtime/hotpath", + "rustfs-trusted-proxies/hotpath", + "rustfs-utils/hotpath", + "rustfs-zip/hotpath", + "rustfs-test-utils/hotpath", ] hotpath-alloc = [ + "hotpath", "hotpath/hotpath-alloc", + "rustfs-audit/hotpath-alloc", + "rustfs-common/hotpath-alloc", + "rustfs-concurrency/hotpath-alloc", + "rustfs-config/hotpath-alloc", + "rustfs-credentials/hotpath-alloc", + "rustfs-crypto/hotpath-alloc", + "rustfs-data-usage/hotpath-alloc", "rustfs-ecstore/hotpath-alloc", + "rustfs-extension-schema/hotpath-alloc", "rustfs-filemeta/hotpath-alloc", + "rustfs-heal/hotpath-alloc", + "rustfs-iam/hotpath-alloc", + "rustfs-io-core/hotpath-alloc", + "rustfs-io-metrics/hotpath-alloc", + "rustfs-keystone/hotpath-alloc", + "rustfs-kms/hotpath-alloc", + "rustfs-lock/hotpath-alloc", + "rustfs-log-analyzer/hotpath-alloc", + "rustfs-madmin/hotpath-alloc", + "rustfs-notify/hotpath-alloc", + "rustfs-object-capacity/hotpath-alloc", + "rustfs-object-data-cache/hotpath-alloc", + "rustfs-obs/hotpath-alloc", "rustfs-policy/hotpath-alloc", + "rustfs-protocols/hotpath-alloc", + "rustfs-protos/hotpath-alloc", "rustfs-rio/hotpath-alloc", + "rustfs-s3-ops/hotpath-alloc", + "rustfs-s3-types/hotpath-alloc", + "rustfs-s3select-api/hotpath-alloc", + "rustfs-s3select-query/hotpath-alloc", + "rustfs-scanner/hotpath-alloc", + "rustfs-security-governance/hotpath-alloc", + "rustfs-signer/hotpath-alloc", + "rustfs-storage-api/hotpath-alloc", + "rustfs-targets/hotpath-alloc", + "rustfs-tls-runtime/hotpath-alloc", + "rustfs-trusted-proxies/hotpath-alloc", + "rustfs-utils/hotpath-alloc", + "rustfs-zip/hotpath-alloc", + "rustfs-test-utils/hotpath-alloc", ] hotpath-cpu = [ "hotpath", "hotpath/hotpath-cpu", + "rustfs-audit/hotpath-cpu", + "rustfs-common/hotpath-cpu", + "rustfs-concurrency/hotpath-cpu", + "rustfs-config/hotpath-cpu", + "rustfs-credentials/hotpath-cpu", + "rustfs-crypto/hotpath-cpu", + "rustfs-data-usage/hotpath-cpu", "rustfs-ecstore/hotpath-cpu", + "rustfs-extension-schema/hotpath-cpu", "rustfs-filemeta/hotpath-cpu", + "rustfs-heal/hotpath-cpu", + "rustfs-iam/hotpath-cpu", + "rustfs-io-core/hotpath-cpu", + "rustfs-io-metrics/hotpath-cpu", + "rustfs-keystone/hotpath-cpu", + "rustfs-kms/hotpath-cpu", + "rustfs-lock/hotpath-cpu", + "rustfs-log-analyzer/hotpath-cpu", + "rustfs-madmin/hotpath-cpu", + "rustfs-notify/hotpath-cpu", + "rustfs-object-capacity/hotpath-cpu", + "rustfs-object-data-cache/hotpath-cpu", + "rustfs-obs/hotpath-cpu", "rustfs-policy/hotpath-cpu", + "rustfs-protocols/hotpath-cpu", + "rustfs-protos/hotpath-cpu", "rustfs-rio/hotpath-cpu", + "rustfs-s3-ops/hotpath-cpu", + "rustfs-s3-types/hotpath-cpu", + "rustfs-s3select-api/hotpath-cpu", + "rustfs-s3select-query/hotpath-cpu", + "rustfs-scanner/hotpath-cpu", + "rustfs-security-governance/hotpath-cpu", + "rustfs-signer/hotpath-cpu", + "rustfs-storage-api/hotpath-cpu", + "rustfs-targets/hotpath-cpu", + "rustfs-tls-runtime/hotpath-cpu", + "rustfs-trusted-proxies/hotpath-cpu", + "rustfs-utils/hotpath-cpu", + "rustfs-zip/hotpath-cpu", + "rustfs-test-utils/hotpath-cpu", ] [lints] From 76b3c085b58000ff2cc544a80ff7f4cd747b6831 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 31 Jul 2026 16:31:21 +0800 Subject: [PATCH 3/8] refactor(kms): fold the KmsClient layer into KmsBackend and complete lifecycle overrides (#5501) --- crates/kms/src/backends/contract_tests.rs | 81 +++-- crates/kms/src/backends/local.rs | 63 ++-- crates/kms/src/backends/mod.rs | 183 ----------- ...tests__vault_kv2_backend_capabilities.snap | 4 +- crates/kms/src/backends/static_kms.rs | 219 +++++-------- crates/kms/src/backends/vault.rs | 305 ++++++++++++------ crates/kms/src/backends/vault_transit.rs | 137 +++++--- crates/kms/src/deletion_worker.rs | 1 - 8 files changed, 470 insertions(+), 523 deletions(-) diff --git a/crates/kms/src/backends/contract_tests.rs b/crates/kms/src/backends/contract_tests.rs index a436a17e2..52d9beb18 100644 --- a/crates/kms/src/backends/contract_tests.rs +++ b/crates/kms/src/backends/contract_tests.rs @@ -27,11 +27,11 @@ //! server, so they are `#[ignore]`d in CI. Static is covered by its own //! stateless contract below. +use super::KmsBackend; use super::local::LocalKmsBackend; use super::static_kms::StaticKmsBackend; use super::vault::VaultKmsBackend; use super::vault_transit::VaultTransitKmsBackend; -use super::{KmsBackend, KmsClient}; use crate::config::KmsConfig; use crate::error::{KmsError, Result}; use crate::manager::KmsManager; @@ -46,6 +46,24 @@ use rand::RngExt as _; use std::collections::HashMap; use std::sync::Arc; +fn expect_unsupported(result: Result) { + match result { + Err(KmsError::UnsupportedCapability { .. }) => {} + other => panic!("expected UnsupportedCapability, got {other:?}"), + } +} + +/// Rotation while not Enabled: backends with rotation support must reject it +/// through the state machine; backends without it report the capability gap. +async fn expect_rotate_rejected(backend: &dyn KmsBackend, key_id: &str) { + let result = backend.rotate_key(key_id).await; + if backend.capabilities().rotate { + expect_invalid_key_state(result, ""); + } else { + expect_unsupported(result); + } +} + fn expect_invalid_key_state(result: Result, expected_fragment: &str) { match result { Err(KmsError::InvalidOperation { message }) => assert!( @@ -117,11 +135,9 @@ async fn assert_key_state(backend: &dyn KmsBackend, key_id: &str, expected: KeyS assert_eq!(described.key_metadata.key_state, expected, "unexpected state for key {key_id}"); } -/// Drives one freshly created (Enabled) key through the full state matrix. -/// -/// `backend` is the product surface; `client` drives the lifecycle -/// transitions not yet exposed through `KmsBackend`. -async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn KmsClient, key_id: &str) { +/// Drives one freshly created (Enabled) key through the full state matrix, +/// entirely through the `KmsBackend` product surface. +async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) { // Enabled: cryptographic use is allowed. Keep an envelope around to prove // decryption keeps working in later states. let data_key = backend @@ -134,16 +150,13 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn Km .expect("Enabled key must encrypt"); // Enabled -> Disabled. - client - .disable_key(key_id, None) - .await - .expect("disable from Enabled must succeed"); + backend.disable_key(key_id).await.expect("disable from Enabled must succeed"); assert_key_state(backend, key_id, KeyState::Disabled).await; // Disabled: new cryptographic use and rotation are rejected... expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "disabled"); expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "disabled"); - expect_invalid_key_state(client.rotate_key(key_id, None).await, ""); + expect_rotate_rejected(backend, key_id).await; // ...but decryption of existing data keeps working (explicit AWS deviation)... let decrypted = backend .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) @@ -151,17 +164,14 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn Km .expect("decrypt with a disabled key must keep working"); assert_eq!(decrypted.plaintext, data_key.plaintext_key, "decrypt must recover the original data key"); // ...disable stays idempotent, cancel has nothing to cancel, and enable recovers. - client.disable_key(key_id, None).await.expect("disable must be idempotent"); + backend.disable_key(key_id).await.expect("disable must be idempotent"); expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion"); - client - .enable_key(key_id, None) - .await - .expect("enable from Disabled must succeed"); + backend.enable_key(key_id).await.expect("enable from Disabled must succeed"); assert_key_state(backend, key_id, KeyState::Enabled).await; // Disabled keys may still be scheduled for deletion. - client - .disable_key(key_id, None) + backend + .disable_key(key_id) .await .expect("disable before scheduling must succeed"); backend @@ -173,10 +183,9 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, client: &dyn Km // PendingDeletion: everything except decryption and cancellation is rejected. expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "pending deletion"); expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "pending deletion"); - expect_invalid_key_state(client.enable_key(key_id, None).await, "pending deletion"); - expect_invalid_key_state(client.disable_key(key_id, None).await, "pending deletion"); - expect_invalid_key_state(client.rotate_key(key_id, None).await, ""); - expect_invalid_key_state(client.schedule_key_deletion(key_id, 7, None).await, "pending deletion"); + expect_invalid_key_state(backend.enable_key(key_id).await, "pending deletion"); + expect_invalid_key_state(backend.disable_key(key_id).await, "pending deletion"); + expect_rotate_rejected(backend, key_id).await; expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "pending deletion"); let decrypted = backend .decrypt(decrypt_request(data_key.ciphertext_blob.clone())) @@ -215,7 +224,7 @@ async fn local_fixture() -> (tempfile::TempDir, KmsConfig, LocalKmsBackend, Stri #[tokio::test] async fn local_backend_state_machine_contract() { let (_temp_dir, _config, backend, key_id) = local_fixture().await; - assert_state_machine_contract(&backend, backend.lifecycle_client(), &key_id).await; + assert_state_machine_contract(&backend, &key_id).await; } /// SSE-shaped regression: disabling a key must not break decryption of data @@ -256,10 +265,7 @@ async fn static_backend_stateless_contract() { rand::rng().fill(&mut raw_key[..]); let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key)); let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build"); - // StaticKmsBackend implements both traits with overlapping method names, - // so pin each surface once instead of qualifying every call. let backend: &dyn KmsBackend = &static_backend; - let client: &dyn KmsClient = &static_backend; let data_key = backend .generate_data_key(generate_request(key_id)) @@ -275,9 +281,11 @@ async fn static_backend_stateless_contract() { expect_invalid_key_state(backend.create_key(create_request("another-key".to_string())).await, "read-only"); expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "read-only"); expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "read-only"); - expect_invalid_key_state(client.disable_key(key_id, None).await, "read-only"); - expect_invalid_key_state(client.schedule_key_deletion(key_id, 7, None).await, "read-only"); - expect_invalid_key_state(client.rotate_key(key_id, None).await, "read-only"); + // Enable/disable and rotation are capability gaps at the product + // surface, not state-machine rejections. + expect_unsupported(backend.enable_key(key_id).await); + expect_unsupported(backend.disable_key(key_id).await); + expect_unsupported(backend.rotate_key(key_id).await); } fn vault_dev_config(constructor: fn(url::Url, String) -> KmsConfig) -> KmsConfig { @@ -298,7 +306,15 @@ async fn vault_kv2_backend_state_machine_contract() { .await .expect("key should be created"); - assert_state_machine_contract(&backend, backend.lifecycle_client(), &created.key_id).await; + assert_state_machine_contract(&backend, &created.key_id).await; + + // KV2 additionally supports version-retaining rotation, which must only + // work while the key is Enabled (the shared matrix covered the + // rejections). + backend + .rotate_key(&created.key_id) + .await + .expect("rotation of an Enabled KV2 key must succeed"); // Cleanup: leave the key pending deletion so repeated runs stay tidy. let _ = backend.delete_key(schedule_request(&created.key_id)).await; @@ -316,13 +332,12 @@ async fn vault_transit_backend_state_machine_contract() { .await .expect("key should be created"); - assert_state_machine_contract(&backend, backend.lifecycle_client(), &created.key_id).await; + assert_state_machine_contract(&backend, &created.key_id).await; // Transit additionally supports rotation, which must only work while the // key is Enabled (the shared matrix already covered the rejections). backend - .lifecycle_client() - .rotate_key(&created.key_id, None) + .rotate_key(&created.key_id) .await .expect("rotation of an Enabled transit key must succeed"); diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 2d410f6d7..e1e347cca 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -14,9 +14,7 @@ //! Local file-based KMS backend implementation -use crate::backends::{ - BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_status_permits, -}; +use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits}; use crate::config::KmsConfig; use crate::config::LocalConfig; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -937,9 +935,12 @@ impl LocalKmsClient { } } -#[async_trait] -impl KmsClient for LocalKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result { +impl LocalKmsClient { + pub(crate) async fn generate_data_key( + &self, + request: &GenerateKeyRequest, + context: Option<&OperationContext>, + ) -> Result { debug!("Generating data key for master key: {}", request.master_key_id); let key_info = self.describe_key(&request.master_key_id, context).await?; @@ -980,7 +981,7 @@ impl KmsClient for LocalKmsClient { Ok(data_key) } - async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result { + pub(crate) async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result { debug!("Encrypting data with key: {}", request.key_id); // Verify key exists and its state allows encryption @@ -997,7 +998,7 @@ impl KmsClient for LocalKmsClient { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { + pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { debug!("Decrypting data"); // Parse the data key envelope from ciphertext @@ -1031,7 +1032,14 @@ impl KmsClient for LocalKmsClient { Ok(plaintext) } - async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn create_key( + &self, + key_id: &str, + algorithm: &str, + context: Option<&OperationContext>, + ) -> Result { debug!("Creating master key: {}", key_id); // Check if key already exists @@ -1060,14 +1068,18 @@ impl KmsClient for LocalKmsClient { Ok(master_key) } - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { debug!("Describing key: {}", key_id); let master_key = self.load_master_key(key_id).await?; Ok(master_key.into()) } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn list_keys( + &self, + request: &ListKeysRequest, + _context: Option<&OperationContext>, + ) -> Result { debug!("Listing keys"); let mut keys = Vec::new(); @@ -1111,7 +1123,7 @@ impl KmsClient for LocalKmsClient { }) } - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; @@ -1129,7 +1141,7 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; @@ -1146,7 +1158,9 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn schedule_key_deletion( + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn schedule_key_deletion( &self, key_id: &str, pending_window_days: u32, @@ -1170,7 +1184,9 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Canceling deletion for key: {}", key_id); let _write_guard = self.lock_key_for_write(key_id).await; @@ -1190,7 +1206,9 @@ impl KmsClient for LocalKmsClient { Ok(()) } - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { if !fs::try_exists(self.master_key_path(key_id)?).await? { return Err(KmsError::key_not_found(key_id)); } @@ -1199,7 +1217,7 @@ impl KmsClient for LocalKmsClient { )) } - async fn health_check(&self) -> Result<()> { + pub(crate) async fn health_check(&self) -> Result<()> { // Check if key directory is accessible if !self.config.key_dir.exists() { return Err(KmsError::backend_error("Key directory does not exist")); @@ -1210,17 +1228,6 @@ impl KmsClient for LocalKmsClient { Ok(()) } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new( - "local".to_string(), - env!("CARGO_PKG_VERSION").to_string(), - self.config.key_dir.to_string_lossy().to_string(), - true, // We'll assume healthy for now - ) - .with_metadata("key_dir".to_string(), self.config.key_dir.to_string_lossy().to_string()) - .with_metadata("encrypted_at_rest".to_string(), self.master_cipher.is_some().to_string()) - } } /// LocalKmsBackend wraps LocalKmsClient and implements the KmsBackend trait diff --git a/crates/kms/src/backends/mod.rs b/crates/kms/src/backends/mod.rs index a74ed9a38..bf9a54e55 100644 --- a/crates/kms/src/backends/mod.rs +++ b/crates/kms/src/backends/mod.rs @@ -19,7 +19,6 @@ use crate::types::*; use async_trait::async_trait; use jiff::Zoned; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; #[cfg(test)] mod contract_tests; @@ -99,136 +98,6 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat ensure_key_state_permits(key_id, &state, operation) } -/// Abstract KMS client interface that all backends must implement -#[async_trait] -pub trait KmsClient: Send + Sync { - /// Generate a new data encryption key (DEK) - /// - /// Creates a new data key using the specified master key. The returned DataKey - /// contains both the plaintext and encrypted versions of the key. - /// - /// # Arguments - /// * `request` - The key generation request - /// * `context` - Optional operation context for auditing - /// - /// # Returns - /// Returns a DataKey containing both plaintext and encrypted key material - async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result; - - /// Encrypt data directly using a master key - /// - /// Encrypts the provided plaintext using the specified master key. - /// This is different from generate_data_key as it encrypts user data directly. - /// - /// # Arguments - /// * `request` - The encryption request containing plaintext and key ID - /// * `context` - Optional operation context for auditing - async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result; - - /// Decrypt data using a master key - /// - /// Decrypts the provided ciphertext. The KMS automatically determines - /// which key was used for encryption based on the ciphertext metadata. - /// - /// # Arguments - /// * `request` - The decryption request containing ciphertext - /// * `context` - Optional operation context for auditing - async fn decrypt(&self, request: &DecryptRequest, context: Option<&OperationContext>) -> Result>; - - /// Create a new master key - /// - /// Creates a new master key in the KMS with the specified ID. - /// Returns an error if a key with the same ID already exists. - /// - /// # Arguments - /// * `key_id` - Unique identifier for the new key - /// * `algorithm` - Key algorithm (e.g., "AES_256") - /// * `context` - Optional operation context for auditing - async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result; - - /// Get information about a specific key - /// - /// Returns metadata and information about the specified key. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn describe_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result; - - /// List available keys - /// - /// Returns a paginated list of keys available in the KMS. - /// - /// # Arguments - /// * `request` - List request parameters (pagination, filters) - /// * `context` - Optional operation context for auditing - async fn list_keys(&self, request: &ListKeysRequest, context: Option<&OperationContext>) -> Result; - - /// Enable a key - /// - /// Enables a previously disabled key, allowing it to be used for cryptographic operations. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn enable_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>; - - /// Disable a key - /// - /// Disables a key, preventing it from being used for new cryptographic operations. - /// Existing encrypted data can still be decrypted. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn disable_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>; - - /// Schedule key deletion - /// - /// Schedules a key for deletion after a specified number of days. - /// This allows for a grace period to recover the key if needed. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `pending_window_days` - Number of days before actual deletion - /// * `context` - Optional operation context for auditing - async fn schedule_key_deletion( - &self, - key_id: &str, - pending_window_days: u32, - context: Option<&OperationContext>, - ) -> Result<()>; - - /// Cancel key deletion - /// - /// Cancels a previously scheduled key deletion. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn cancel_key_deletion(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>; - - /// Rotate a key - /// - /// Creates a new version of the specified key. Previous versions remain - /// available for decryption but new operations will use the new version. - /// - /// # Arguments - /// * `key_id` - The key identifier - /// * `context` - Optional operation context for auditing - async fn rotate_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result; - - /// Health check - /// - /// Performs a health check on the KMS backend to ensure it's operational. - async fn health_check(&self) -> Result<()>; - - /// Get backend information - /// - /// Returns information about the KMS backend (type, version, etc.). - fn backend_info(&self) -> BackendInfo; -} - /// Simplified KMS backend interface for manager #[async_trait] pub trait KmsBackend: Send + Sync { @@ -327,58 +196,6 @@ pub enum ExpiredKeyRemoval { NotExpired, } -/// Information about a KMS backend -#[derive(Debug, Clone)] -pub struct BackendInfo { - /// Backend type name (e.g., "local", "vault") - pub backend_type: String, - /// Backend version - pub version: String, - /// Backend endpoint or location - pub endpoint: String, - /// Whether the backend is currently healthy - pub healthy: bool, - /// Additional metadata about the backend - pub metadata: HashMap, -} - -impl BackendInfo { - /// Create a new backend info - /// - /// # Arguments - /// * `backend_type` - The type of the backend - /// * `version` - The version of the backend - /// * `endpoint` - The endpoint or location of the backend - /// * `healthy` - Whether the backend is healthy - /// - /// # Returns - /// A new BackendInfo instance - /// - pub fn new(backend_type: String, version: String, endpoint: String, healthy: bool) -> Self { - Self { - backend_type, - version, - endpoint, - healthy, - metadata: HashMap::new(), - } - } - - /// Add metadata to the backend info - /// - /// # Arguments - /// * `key` - Metadata key - /// * `value` - Metadata value - /// - /// # Returns - /// Updated BackendInfo instance - /// - pub fn with_metadata(mut self, key: String, value: String) -> Self { - self.metadata.insert(key, value); - self - } -} - /// Set of operations a KMS backend supports. /// /// Reported by [`KmsBackend::capabilities`] so callers (manager, admin API) diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap index 57be08b27..2c0bd7fce 100644 --- a/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap @@ -8,7 +8,7 @@ expression: capabilities_snapshot(backend.capabilities()) "encrypt": true, "generate_data_key": true, "physical_delete": true, - "rotate": false, + "rotate": true, "schedule_deletion": true, - "versioning": false + "versioning": true } diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 73b988eeb..f3fc065dc 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -21,7 +21,7 @@ //! //! encrypted_data(plaintext_len+16) || nonce (12 bytes) -use crate::backends::{BackendCapabilities, BackendInfo, KmsBackend, KmsClient}; +use crate::backends::{BackendCapabilities, KmsBackend}; use crate::config::{BackendConfig, KmsConfig}; use crate::encryption::DataKeyEnvelope; use crate::error::{KmsError, Result}; @@ -98,9 +98,10 @@ impl StaticKmsBackend { } } -#[async_trait] -impl KmsClient for StaticKmsBackend { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result { +impl StaticKmsBackend { + /// Generate a fresh data key and wrap it in the standard KMS envelope, + /// authenticated against the canonical encryption context. + pub(crate) fn generate_data_key_envelope(&self, request: &GenerateKeyRequest) -> Result { if request.master_key_id != self.key_id { return Err(KmsError::key_not_found(&request.master_key_id)); } @@ -151,7 +152,8 @@ impl KmsClient for StaticKmsBackend { )) } - async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result { + /// Encrypt caller-provided plaintext into the standard KMS envelope. + pub(crate) fn encrypt_to_envelope(&self, request: &EncryptRequest) -> Result { if request.key_id != self.key_id { return Err(KmsError::key_not_found(&request.key_id)); } @@ -196,7 +198,8 @@ impl KmsClient for StaticKmsBackend { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { + /// Open a KMS envelope produced by this backend. + pub(crate) fn decrypt_envelope(&self, request: &DecryptRequest) -> Result> { let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext) .map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?; if envelope.master_key_id != self.key_id { @@ -235,14 +238,8 @@ impl KmsClient for StaticKmsBackend { Ok(plaintext) } - async fn create_key(&self, key_id: &str, _algorithm: &str, _context: Option<&OperationContext>) -> Result { - if key_id == self.key_id { - return Err(KmsError::key_already_exists(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot create new keys")) - } - - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + /// Describe the single configured key. + pub(crate) fn configured_key_info(&self, key_id: &str) -> Result { if key_id != self.key_id { return Err(KmsError::key_not_found(key_id)); } @@ -263,7 +260,8 @@ impl KmsClient for StaticKmsBackend { }) } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result { + /// List the single configured key, honouring the pagination marker. + pub(crate) fn list_configured_key(&self, request: &ListKeysRequest) -> Result { let key_info = KeyInfo { key_id: self.key_id.clone(), description: Some("Static single-key KMS backend".to_string()), @@ -295,57 +293,6 @@ impl KmsClient for StaticKmsBackend { truncated: false, }) } - - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - // Static KMS key is always enabled - Ok(()) - } - - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot disable keys")) - } - - async fn schedule_key_deletion( - &self, - key_id: &str, - _pending_window_days: u32, - _context: Option<&OperationContext>, - ) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot schedule key deletion")) - } - - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot cancel key deletion")) - } - - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { - if key_id != self.key_id { - return Err(KmsError::key_not_found(key_id)); - } - Err(KmsError::invalid_operation("Static KMS is read-only: cannot rotate keys")) - } - - async fn health_check(&self) -> Result<()> { - // Static KMS is always healthy if it was successfully initialized - Ok(()) - } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new("static".to_string(), env!("CARGO_PKG_VERSION").to_string(), "local".to_string(), true) - .with_metadata("key_id".to_string(), self.key_id.clone()) - } } #[async_trait] @@ -359,12 +306,12 @@ impl KmsBackend for StaticKmsBackend { } async fn encrypt(&self, request: EncryptRequest) -> Result { - ::encrypt(self, &request, None).await + self.encrypt_to_envelope(&request) } async fn decrypt(&self, request: DecryptRequest) -> Result { let key_id = self.key_id.clone(); - let plaintext = ::decrypt(self, &request, None).await?; + let plaintext = self.decrypt_envelope(&request)?; Ok(DecryptResponse { plaintext, key_id, @@ -380,7 +327,7 @@ impl KmsBackend for StaticKmsBackend { encryption_context: request.encryption_context, grant_tokens: Vec::new(), }; - let data_key = ::generate_data_key(self, &gen_req, None).await?; + let data_key = self.generate_data_key_envelope(&gen_req)?; let plaintext_key = data_key .plaintext @@ -395,7 +342,7 @@ impl KmsBackend for StaticKmsBackend { } async fn describe_key(&self, request: DescribeKeyRequest) -> Result { - let key_info = ::describe_key(self, &request.key_id, None).await?; + let key_info = self.configured_key_info(&request.key_id)?; let key_metadata = KeyMetadata { key_id: key_info.key_id.clone(), key_state: if key_info.status == KeyStatus::Active { @@ -415,7 +362,7 @@ impl KmsBackend for StaticKmsBackend { } async fn list_keys(&self, request: ListKeysRequest) -> Result { - ::list_keys(self, &request, None).await + self.list_configured_key(&request) } async fn delete_key(&self, request: DeleteKeyRequest) -> Result { @@ -446,7 +393,7 @@ impl KmsBackend for StaticKmsBackend { #[cfg(test)] mod tests { use super::*; - use crate::backends::{KmsBackend as KmsBackendTrait, KmsClient}; + use crate::backends::KmsBackend as KmsBackendTrait; use crate::config::{BackendConfig, KmsBackend, StaticConfig}; use crate::encryption::is_data_key_envelope; use base64::Engine as _; @@ -491,8 +438,8 @@ mod tests { // Generate data key let request = GenerateKeyRequest::new(key_id.clone(), "AES_256".to_string()) .with_context("bucket".to_string(), "test-bucket".to_string()); - let data_key = KmsClient::generate_data_key(&backend, &request, None) - .await + let data_key = backend + .generate_data_key_envelope(&request) .expect("Failed to generate data key"); assert_eq!(data_key.key_id, key_id); @@ -509,9 +456,7 @@ mod tests { // Decrypt the data key let decrypt_request = DecryptRequest::new(data_key.ciphertext.clone()).with_context("bucket".to_string(), "test-bucket".to_string()); - let decrypted = KmsClient::decrypt(&backend, &decrypt_request, None) - .await - .expect("Failed to decrypt"); + let decrypted = backend.decrypt_envelope(&decrypt_request).expect("Failed to decrypt"); assert_eq!(decrypted.as_slice(), data_key.plaintext.as_deref().expect("plaintext should exist")); } @@ -523,8 +468,8 @@ mod tests { .with_context("bucket".to_string(), "source-bucket".to_string()) .with_context("object".to_string(), "source-object".to_string()); - let data_key = KmsClient::generate_data_key(&backend, &request, None) - .await + let data_key = backend + .generate_data_key_envelope(&request) .expect("generate static KMS data key"); assert!( @@ -568,8 +513,8 @@ mod tests { let (backend, key_id, _key) = create_test_backend().await; let request = GenerateKeyRequest::new(key_id, "AES_256".to_string()) .with_context("bucket".to_string(), "source-bucket".to_string()); - let generated = KmsClient::generate_data_key(&backend, &request, None) - .await + let generated = backend + .generate_data_key_envelope(&request) .expect("generate context-bound data key"); let mut envelope: DataKeyEnvelope = serde_json::from_slice(&generated.ciphertext).expect("parse static KMS envelope"); envelope @@ -578,8 +523,8 @@ mod tests { let decrypt_request = DecryptRequest::new(serde_json::to_vec(&envelope).expect("serialize tampered envelope")) .with_context("bucket".to_string(), "different-bucket".to_string()); - let error = KmsClient::decrypt(&backend, &decrypt_request, None) - .await + let error = backend + .decrypt_envelope(&decrypt_request) .expect_err("tampering with authenticated envelope context must fail"); assert!(matches!(error, KmsError::CryptographicError { .. })); @@ -590,7 +535,7 @@ mod tests { let (backend, _key_id, _key) = create_test_backend().await; let request = GenerateKeyRequest::new("wrong-key-id".to_string(), "AES_256".to_string()); - let result = KmsClient::generate_data_key(&backend, &request, None).await; + let result = backend.generate_data_key_envelope(&request); assert!(result.is_err()); assert!(result.expect_err("should be Err").to_string().contains("wrong-key-id")); } @@ -602,7 +547,7 @@ mod tests { // Ciphertext too short let short = vec![0u8; 10]; let request = DecryptRequest::new(short); - let result = KmsClient::decrypt(&backend, &request, None).await; + let result = backend.decrypt_envelope(&request); assert!(result.is_err()); } @@ -612,9 +557,7 @@ mod tests { // Generate a valid ciphertext first let gen_request = GenerateKeyRequest::new(key_id, "AES_256".to_string()); - let data_key = KmsClient::generate_data_key(&backend, &gen_request, None) - .await - .expect("generate"); + let data_key = backend.generate_data_key_envelope(&gen_request).expect("generate"); // Tamper with the ciphertext (flip a bit in the encrypted portion) let mut tampered = data_key.ciphertext.clone(); @@ -623,7 +566,7 @@ mod tests { } let request = DecryptRequest::new(tampered); - let result = KmsClient::decrypt(&backend, &request, None).await; + let result = backend.decrypt_envelope(&request); assert!(result.is_err()); } @@ -632,7 +575,14 @@ mod tests { let (backend, key_id, _key) = create_test_backend().await; // Creating the pre-configured key should return KeyAlreadyExists - let result = KmsClient::create_key(&backend, &key_id, "AES_256", None).await; + let result = KmsBackendTrait::create_key( + &backend, + CreateKeyRequest { + key_name: Some(key_id.clone()), + ..Default::default() + }, + ) + .await; assert!(result.is_err()); assert!(result.expect_err("should be Err").to_string().contains("already exists")); } @@ -642,7 +592,14 @@ mod tests { let (backend, _key_id, _key) = create_test_backend().await; // Creating any other key should return invalid operation (read-only) - let result = KmsClient::create_key(&backend, "other-key", "AES_256", None).await; + let result = KmsBackendTrait::create_key( + &backend, + CreateKeyRequest { + key_name: Some("other-key".to_string()), + ..Default::default() + }, + ) + .await; assert!(result.is_err()); let err_msg = result.expect_err("should be Err").to_string(); assert!(err_msg.contains("read-only") || err_msg.contains("cannot create")); @@ -652,15 +609,13 @@ mod tests { async fn test_describe_key() { let (backend, key_id, _key) = create_test_backend().await; - let key_info = KmsClient::describe_key(&backend, &key_id, None) - .await - .expect("describe_key should succeed"); + let key_info = backend.configured_key_info(&key_id).expect("describe_key should succeed"); assert_eq!(key_info.key_id, key_id); assert_eq!(key_info.status, KeyStatus::Active); assert_eq!(key_info.algorithm, "AES_256"); // Wrong key ID - let result = KmsClient::describe_key(&backend, "nonexistent", None).await; + let result = backend.configured_key_info("nonexistent"); assert!(result.is_err()); } @@ -668,8 +623,8 @@ mod tests { async fn test_list_keys() { let (backend, key_id, _key) = create_test_backend().await; - let response = KmsClient::list_keys(&backend, &ListKeysRequest::default(), None) - .await + let response = backend + .list_configured_key(&ListKeysRequest::default()) .expect("list_keys should succeed"); assert_eq!(response.keys.len(), 1); assert_eq!(response.keys[0].key_id, key_id); @@ -677,61 +632,45 @@ mod tests { } #[tokio::test] - async fn test_disable_key_returns_error() { + async fn lifecycle_mutations_are_unsupported_at_the_product_surface() { let (backend, key_id, _key) = create_test_backend().await; - let result = KmsClient::disable_key(&backend, &key_id, None).await; - assert!(result.is_err()); - assert!(result.expect_err("should be Err").to_string().contains("read-only")); - } - - #[tokio::test] - async fn test_enable_key_is_noop() { - let (backend, key_id, _key) = create_test_backend().await; - - // Enable should succeed (no-op for static KMS) - KmsClient::enable_key(&backend, &key_id, None) - .await - .expect("enable_key should be no-op"); - - // Wrong key should still fail - let result = KmsClient::enable_key(&backend, "wrong", None).await; - assert!(result.is_err()); + // The static backend advertises no enable/disable or rotation + // capability, so the shared KmsBackend defaults reject all three. + for result in [ + KmsBackendTrait::enable_key(&backend, &key_id).await, + KmsBackendTrait::disable_key(&backend, &key_id).await, + KmsBackendTrait::rotate_key(&backend, &key_id).await, + ] { + let error = result.expect_err("static lifecycle mutations must be rejected"); + assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "got {error:?}"); + } } #[tokio::test] async fn test_delete_key_returns_error() { let (backend, key_id, _key) = create_test_backend().await; - let result = KmsClient::schedule_key_deletion(&backend, &key_id, 7, None).await; + let result = KmsBackendTrait::delete_key( + &backend, + DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + }, + ) + .await; assert!(result.is_err()); assert!(result.expect_err("should be Err").to_string().contains("read-only")); } - #[tokio::test] - async fn test_rotate_key_returns_error() { - let (backend, key_id, _key) = create_test_backend().await; - - let result = KmsClient::rotate_key(&backend, &key_id, None).await; - assert!(result.is_err()); - } - #[tokio::test] async fn test_health_check() { let (backend, _key_id, _key) = create_test_backend().await; - KmsClient::health_check(&backend).await.expect("health_check should succeed"); - } - - #[tokio::test] - async fn test_backend_info() { - let (backend, key_id, _key) = create_test_backend().await; - - let info = KmsClient::backend_info(&backend); - assert_eq!(info.backend_type, "static"); - assert_eq!(info.endpoint, "local"); - assert!(info.healthy); - assert_eq!(info.metadata.get("key_id"), Some(&key_id)); + KmsBackendTrait::health_check(&backend) + .await + .expect("health_check should succeed"); } #[tokio::test] @@ -740,17 +679,13 @@ mod tests { let plaintext = b"Hello, static KMS world!"; let enc_request = EncryptRequest::new(key_id.clone(), plaintext.to_vec()); - let enc_response = KmsClient::encrypt(&backend, &enc_request, None) - .await - .expect("encrypt should succeed"); + let enc_response = backend.encrypt_to_envelope(&enc_request).expect("encrypt should succeed"); assert_eq!(enc_response.key_id, key_id); assert!(!enc_response.ciphertext.is_empty()); let dec_request = DecryptRequest::new(enc_response.ciphertext); - let decrypted = KmsClient::decrypt(&backend, &dec_request, None) - .await - .expect("decrypt should succeed"); + let decrypted = backend.decrypt_envelope(&dec_request).expect("decrypt should succeed"); assert_eq!(decrypted, plaintext); } diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index ebc417833..24d9c9c39 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -19,8 +19,7 @@ use crate::backends::vault_credentials::{ token_source_for, }; use crate::backends::{ - BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits, - ensure_key_status_permits, + BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits, }; use crate::config::{KmsConfig, VaultConfig}; use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material}; @@ -42,7 +41,6 @@ use vaultrs::{api::kv2::requests::SetSecretRequestOptions, error::ClientError, k /// Vault KMS client implementation pub struct VaultKmsClient { credentials: Arc, - config: VaultConfig, /// Mount path for the KV engine (typically "kv" or "secret") kv_mount: String, /// Path prefix for storing keys @@ -200,7 +198,6 @@ impl VaultKmsClient { credentials, kv_mount: config.kv_mount.clone(), key_path_prefix: config.key_path_prefix.clone(), - config, dek_crypto: AesDekCrypto::new(), retry: RetryPolicy::from_config(kms_config), cancel: CancellationToken::new(), @@ -254,13 +251,6 @@ impl VaultKmsClient { Ok(general_purpose::STANDARD.encode(key_material)) } - /// Decode key material from KV2 storage (plain Base64, see `encrypt_key_material`). - async fn decrypt_key_material(&self, encrypted_material: &str) -> Result> { - general_purpose::STANDARD - .decode(encrypted_material) - .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string())) - } - /// Read the immutable material record of one key version. /// /// A missing record fails closed with [`KmsError::KeyVersionNotFound`]; falling @@ -589,9 +579,12 @@ impl VaultKmsClient { } } -#[async_trait] -impl KmsClient for VaultKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result { +impl VaultKmsClient { + pub(crate) async fn generate_data_key( + &self, + request: &GenerateKeyRequest, + _context: Option<&OperationContext>, + ) -> Result { debug!("Generating data key for master key: {}", request.master_key_id); let key_data = self.get_key_data(&request.master_key_id).await?; @@ -632,20 +625,32 @@ impl KmsClient for VaultKmsClient { Ok(data_key) } - async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result { debug!("Encrypting data with key: {}", request.key_id); - // Get the master key and verify its state allows encryption + // Single read of the key record: the material we wrap with and the + // version stamped into the envelope must come from the same snapshot + // (see generate_data_key). let key_data = self.get_key_data(&request.key_id).await?; ensure_key_status_permits(&request.key_id, &key_data.status, StateGatedOperation::Encrypt)?; - let key_material = self.decrypt_key_material(&key_data.encrypted_key_material).await?; + let key_material = decode_stored_key_material(&request.key_id, &key_data.encrypted_key_material) + .inspect_err(|error| warn!(key_id = %request.key_id, %error, "Vault KMS key material failed validation"))?; + let (encrypted_key, nonce) = self.dek_crypto.encrypt(&key_material, &request.plaintext).await?; - // For simplicity, we'll use a basic encryption approach - // In practice, you'd use proper AEAD encryption - let mut ciphertext = request.plaintext.clone(); - for (i, byte) in ciphertext.iter_mut().enumerate() { - *byte ^= key_material[i % key_material.len()]; - } + // Wrap the ciphertext in the same authenticated envelope that + // generate_data_key emits, so decrypt() round-trips it and resolves + // the wrapping master key version after rotations. + let envelope = DataKeyEnvelope { + key_id: uuid::Uuid::new_v4().to_string(), + master_key_id: request.key_id.clone(), + key_spec: "AES_256".to_string(), + encrypted_key, + nonce, + encryption_context: request.encryption_context.clone(), + created_at: Zoned::now(), + master_key_version: Some(key_data.version), + }; + let ciphertext = serde_json::to_vec(&envelope)?; Ok(EncryptResponse { ciphertext, @@ -655,7 +660,7 @@ impl KmsClient for VaultKmsClient { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { + pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { debug!("Decrypting data"); // Parse the data key envelope from ciphertext @@ -697,7 +702,12 @@ impl KmsClient for VaultKmsClient { Ok(plaintext) } - async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn create_key( + &self, + key_id: &str, + algorithm: &str, + _context: Option<&OperationContext>, + ) -> Result { debug!("Creating master key: {} with algorithm: {}", key_id, algorithm); // Existence pre-check with read-confirm recovery: a create whose @@ -779,7 +789,7 @@ impl KmsClient for VaultKmsClient { Ok(master_key) } - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { debug!("Describing key: {}", key_id); let key_data = self.get_key_data(key_id).await?; @@ -799,7 +809,11 @@ impl KmsClient for VaultKmsClient { }) } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn list_keys( + &self, + request: &ListKeysRequest, + _context: Option<&OperationContext>, + ) -> Result { debug!("Listing keys with limit: {:?}", request.limit); let all_keys = self.list_vault_keys().await?; @@ -836,7 +850,7 @@ impl KmsClient for VaultKmsClient { }) } - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; @@ -848,7 +862,7 @@ impl KmsClient for VaultKmsClient { Ok(()) } - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); let mut key_data = self.get_key_data(key_id).await?; @@ -860,39 +874,6 @@ impl KmsClient for VaultKmsClient { Ok(()) } - async fn schedule_key_deletion( - &self, - key_id: &str, - pending_window_days: u32, - _context: Option<&OperationContext>, - ) -> Result<()> { - debug!("Scheduling key deletion: {}", key_id); - - let mut key_data = self.get_key_data(key_id).await?; - ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?; - key_data.status = KeyStatus::PendingDeletion; - key_data.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); - self.store_key_data(key_id, &key_data).await?; - - debug!(key_id, "Vault KMS key deletion scheduled"); - Ok(()) - } - - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - debug!("Canceling key deletion: {}", key_id); - - let mut key_data = self.get_key_data(key_id).await?; - if key_data.status != KeyStatus::PendingDeletion { - return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); - } - key_data.status = KeyStatus::Active; - key_data.deletion_date = None; - self.store_key_data(key_id, &key_data).await?; - - debug!(key_id, "Vault KMS key deletion canceled"); - Ok(()) - } - /// Rotate the master key while keeping every historical version decryptable. /// /// Commit protocol (all writes check-and-set, in this order): @@ -908,10 +889,11 @@ impl KmsClient for VaultKmsClient { /// or interrupted rotation never exposes half-committed material. Concurrent /// rotations are serialized by the check-and-set writes: at most one caller /// commits each version and the losers fail without side effects on current. - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { debug!("Rotating master key: {}", key_id); let (mut cas, mut key_data) = self.get_key_data_versioned(key_id).await?; + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Rotate)?; // The material about to be frozen must be decodable: freezing poisoned // material would give legacy envelopes a permanently broken baseline. This @@ -992,7 +974,7 @@ impl KmsClient for VaultKmsClient { }) } - async fn health_check(&self) -> Result<()> { + pub(crate) async fn health_check(&self) -> Result<()> { debug!("Performing Vault health check"); // Use list_vault_keys but handle the case where no keys exist (which is normal) @@ -1014,15 +996,6 @@ impl KmsClient for VaultKmsClient { } } } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new("vault-kv2".to_string(), "0.1.0".to_string(), self.config.address.clone(), true) - .with_metadata("kv_mount".to_string(), self.kv_mount.clone()) - .with_metadata("key_prefix".to_string(), self.key_path_prefix.clone()) - // Master key material is protected only by Vault ACLs and KV2 at-rest - // encryption; there is no additional cryptographic wrapping. - .with_metadata("at_rest_protection".to_string(), "vault-kv2-acl".to_string()) - } } /// VaultKmsBackend wraps VaultKmsClient and implements the KmsBackend trait @@ -1031,12 +1004,6 @@ pub struct VaultKmsBackend { } impl VaultKmsBackend { - /// Lifecycle driver for the shared state-machine contract tests. - #[cfg(test)] - pub(crate) fn lifecycle_client(&self) -> &VaultKmsClient { - &self.client - } - /// Create a new VaultKmsBackend pub async fn new(config: KmsConfig) -> Result { config.validate()?; @@ -1296,17 +1263,32 @@ impl KmsBackend for VaultKmsBackend { }) } + async fn enable_key(&self, key_id: &str) -> Result<()> { + self.client.enable_key(key_id, None).await + } + + async fn disable_key(&self, key_id: &str) -> Result<()> { + self.client.disable_key(key_id, None).await + } + + async fn rotate_key(&self, key_id: &str) -> Result<()> { + self.client.rotate_key(key_id, None).await.map(|_| ()) + } + async fn health_check(&self) -> Result { self.client.health_check().await.map(|_| true) } fn capabilities(&self) -> BackendCapabilities { - // Rotation is unadvertised: the KV2 backend cannot rotate without - // replacing key material in place, and no historical versions are - // retained, so versioning is unsupported as well. + // Rotation freezes the outgoing material as an immutable version + // record before switching the current pointer, and envelopes resolve + // their wrapping version on decrypt, so every historical version + // stays decryptable after a rotation. BackendCapabilities::minimal() + .with_rotate(true) .with_enable_disable(true) .with_schedule_deletion(true) + .with_versioning(true) .with_physical_delete(true) } @@ -1720,19 +1702,6 @@ mod tests { assert!(!is_cas_conflict(¬_found)); } - #[tokio::test] - async fn test_vault_kv2_backend_info_reports_at_rest_protection() { - let client = VaultKmsClient::new(integration_vault_config(), &KmsConfig::default()) - .await - .expect("client"); - - let info = client.backend_info(); - assert_eq!(info.backend_type, "vault-kv2"); - assert_eq!(info.metadata.get("at_rest_protection").map(String::as_str), Some("vault-kv2-acl")); - // The KV2 backend must not present itself as Transit-backed. - assert!(!format!("{info:?}").contains("Transit")); - } - fn integration_generate_request(key_id: &str) -> GenerateKeyRequest { GenerateKeyRequest { master_key_id: key_id.to_string(), @@ -2081,4 +2050,150 @@ mod tests { let legacy: VaultKeyData = serde_json::from_value(value).expect("legacy record must deserialize"); assert!(legacy.deletion_date.is_none()); } + + /// KV2 write acknowledgement (`SecretVersionMetadata`) for `kv2::set`. + fn kv2_write_ack() -> serde_json::Value { + serde_json::json!({ + "created_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "deletion_time": "", + "destroyed": false, + "version": 2, + }) + } + + #[tokio::test] + async fn wired_kv2_encrypt_round_trips_through_decrypt() { + // One key-record read for the encrypt, one for the decrypt. + let (_vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ]) + .await; + let context = HashMap::from([("bucket".to_string(), "kv2".to_string())]); + + let encrypted = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"kv2-direct-encrypt".to_vec(), + encryption_context: context.clone(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect("encrypt must produce an envelope"); + + // The ciphertext is a real KMS envelope wrapping AEAD output that + // decrypt() can open, not an XOR of the plaintext with the master key + // material. + let envelope: DataKeyEnvelope = serde_json::from_slice(&encrypted.ciphertext).expect("envelope must parse"); + assert_eq!(envelope.master_key_id, "wired-key"); + assert_eq!(envelope.master_key_version, Some(1)); + + let decrypted = client + .decrypt( + &DecryptRequest { + ciphertext: encrypted.ciphertext.clone(), + encryption_context: context, + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect("decrypt must round-trip the envelope"); + assert_eq!(decrypted, b"kv2-direct-encrypt".to_vec()); + + // A different object context must not decrypt (checked before any + // Vault read, so no scripted response is consumed). + let error = client + .decrypt( + &DecryptRequest { + ciphertext: encrypted.ciphertext, + encryption_context: HashMap::from([("bucket".to_string(), "other".to_string())]), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("a different context must not decrypt"); + assert!(matches!(error, KmsError::ContextMismatch { .. }), "got {error:?}"); + } + + /// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the + /// current secret version used as the rotation check-and-set base. + fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value { + serde_json::json!({ + "cas_required": false, + "created_time": "2026-01-01T00:00:00Z", + "current_version": current_version, + "delete_version_after": "0s", + "max_versions": 0, + "oldest_version": 0, + "updated_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "versions": {}, + }) + } + + #[tokio::test] + async fn wired_kv2_rotate_rejected_while_disabled() { + let mut key_data = healthy_key_data(); + key_data.status = KeyStatus::Disabled; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&key_data)), + ]) + .await; + + let error = client + .rotate_key("wired-key", None) + .await + .expect_err("rotation of a disabled key must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!( + requests.len(), + 2, + "the state gate must reject after the versioned read, before any write: {requests:?}" + ); + assert!(requests.iter().all(|line| line.starts_with("GET ")), "{requests:?}"); + } + + #[tokio::test] + async fn wired_backend_lifecycle_overrides_reach_the_client() { + let mut disabled = healthy_key_data(); + disabled.status = KeyStatus::Disabled; + let vault = ScriptedVault::serve(vec![ + // disable: read the Active record, persist it Disabled. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_write_ack()), + // enable: read the Disabled record, persist it Active. + ScriptedResponse::ok(kv2_read_data(&disabled)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + backend + .disable_key("wired-key") + .await + .expect("KmsBackend::disable_key must persist through the client"); + backend + .enable_key("wired-key") + .await + .expect("KmsBackend::enable_key must persist through the client"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 4, "each transition is one read plus one write: {requests:?}"); + assert!(requests[0].starts_with("GET ") && requests[2].starts_with("GET "), "{requests:?}"); + assert!(requests[1].starts_with("POST ") && requests[3].starts_with("POST "), "{requests:?}"); + } } diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 79d5219b7..2b01e8478 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -18,9 +18,7 @@ use crate::backends::vault_credentials::{ CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider, token_source_for, }; -use crate::backends::{ - BackendCapabilities, BackendInfo, ExpiredKeyRemoval, KmsBackend, KmsClient, StateGatedOperation, ensure_key_state_permits, -}; +use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits}; use crate::config::{KmsConfig, VaultTransitConfig}; use crate::encryption::{DataKeyEnvelope, generate_key_material}; use crate::error::{KmsError, Result}; @@ -506,9 +504,12 @@ impl VaultTransitKmsClient { } } -#[async_trait] -impl KmsClient for VaultTransitKmsClient { - async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result { +impl VaultTransitKmsClient { + pub(crate) async fn generate_data_key( + &self, + request: &GenerateKeyRequest, + _context: Option<&OperationContext>, + ) -> Result { self.ensure_key_state_allows(&request.master_key_id, StateGatedOperation::GenerateDataKey) .await?; @@ -540,7 +541,7 @@ impl KmsClient for VaultTransitKmsClient { )) } - async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result { let metadata = self .ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt) .await?; @@ -556,7 +557,7 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { + pub(crate) async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result> { let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext) .map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?; @@ -578,7 +579,14 @@ impl KmsClient for VaultTransitKmsClient { .await } - async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn create_key( + &self, + key_id: &str, + algorithm: &str, + _context: Option<&OperationContext>, + ) -> Result { if algorithm != "AES_256" { return Err(KmsError::unsupported_algorithm(algorithm)); } @@ -645,11 +653,17 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { self.key_info(key_id).await } - async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn list_keys( + &self, + request: &ListKeysRequest, + _context: Option<&OperationContext>, + ) -> Result { let all_keys = self .run("vault_transit_list_keys", OpClass::ReadIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; @@ -692,7 +706,7 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { // A pending deletion must be reverted through cancel_key_deletion, not // silently by enabling, so the gate rejects PendingDeletion here. let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Enable).await?; @@ -701,13 +715,15 @@ impl KmsClient for VaultTransitKmsClient { self.store_key_metadata(key_id, &metadata).await } - async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { + pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Disable).await?; metadata.key_state = KeyState::Disabled; self.store_key_metadata(key_id, &metadata).await } - async fn schedule_key_deletion( + /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. + #[cfg(test)] + pub(crate) async fn schedule_key_deletion( &self, key_id: &str, pending_window_days: u32, @@ -721,17 +737,7 @@ impl KmsClient for VaultTransitKmsClient { self.store_key_metadata(key_id, &metadata).await } - async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - let mut metadata = self.get_key_metadata(key_id).await?; - if metadata.key_state != KeyState::PendingDeletion { - return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); - } - metadata.key_state = KeyState::Enabled; - metadata.deletion_date = None; - self.store_key_metadata(key_id, &metadata).await - } - - async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { + pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { self.ensure_key_state_allows(key_id, StateGatedOperation::Rotate).await?; // Single attempt, never retried: replaying a rotate whose response was @@ -768,7 +774,7 @@ impl KmsClient for VaultTransitKmsClient { }) } - async fn health_check(&self) -> Result<()> { + pub(crate) async fn health_check(&self) -> Result<()> { self.run("vault_transit_health_check", OpClass::ReadIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; key::list(&vault.client, &self.config.mount_path) @@ -780,11 +786,6 @@ impl KmsClient for VaultTransitKmsClient { }) .await } - - fn backend_info(&self) -> BackendInfo { - BackendInfo::new("vault-transit".to_string(), "0.1.0".to_string(), self.config.address.clone(), true) - .with_metadata("mount_path".to_string(), self.config.mount_path.clone()) - } } pub struct VaultTransitKmsBackend { @@ -792,14 +793,6 @@ pub struct VaultTransitKmsBackend { } impl VaultTransitKmsBackend { - /// Lifecycle driver for the shared state-machine contract tests. Using the - /// backend's own client keeps its in-process metadata cache coherent with - /// the transitions the tests perform. - #[cfg(test)] - pub(crate) fn lifecycle_client(&self) -> &VaultTransitKmsClient { - &self.client - } - pub async fn new(config: KmsConfig) -> Result { config.validate()?; @@ -1000,6 +993,18 @@ impl KmsBackend for VaultTransitKmsBackend { }) } + async fn enable_key(&self, key_id: &str) -> Result<()> { + self.client.enable_key(key_id, None).await + } + + async fn disable_key(&self, key_id: &str) -> Result<()> { + self.client.disable_key(key_id, None).await + } + + async fn rotate_key(&self, key_id: &str) -> Result<()> { + self.client.rotate_key(key_id, None).await.map(|_| ()) + } + async fn health_check(&self) -> Result { self.client.health_check().await.map(|_| true) } @@ -1437,4 +1442,58 @@ mod tests { assert_eq!(metadata.key_state, KeyState::Enabled); assert!(metadata.deletion_date.is_none()); } + + /// KV2 write acknowledgement (`SecretVersionMetadata`) for `kv2::set`. + fn kv2_write_ack() -> serde_json::Value { + serde_json::json!({ + "created_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "deletion_time": "", + "destroyed": false, + "version": 2, + }) + } + + #[tokio::test] + async fn wired_backend_lifecycle_overrides_reach_the_client() { + let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // disable: metadata cache miss reads KV, then persists Disabled. + ScriptedResponse::ok(metadata_read_data(&metadata)), + ScriptedResponse::ok(kv2_write_ack()), + // enable: the state gate hits the metadata cache, so only the + // persisting write goes out. + ScriptedResponse::ok(kv2_write_ack()), + // rotate: the gate hits the cache again; the single rotate + // attempt fails and must not be retried. + ScriptedResponse::error(503, "standby"), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + backend + .disable_key("wired-key") + .await + .expect("KmsBackend::disable_key must persist through the client"); + backend + .enable_key("wired-key") + .await + .expect("KmsBackend::enable_key must persist through the client"); + let error = backend + .rotate_key("wired-key") + .await + .expect_err("the scripted 503 must fail the rotation"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 4, "gated reads, two writes and one rotate attempt: {requests:?}"); + assert_eq!(requests[3], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); + } } diff --git a/crates/kms/src/deletion_worker.rs b/crates/kms/src/deletion_worker.rs index fba80b121..3fca15b25 100644 --- a/crates/kms/src/deletion_worker.rs +++ b/crates/kms/src/deletion_worker.rs @@ -182,7 +182,6 @@ impl DeletionWorker { #[cfg(test)] mod tests { use super::*; - use crate::backends::KmsClient as _; use crate::backends::local::LocalKmsBackend; use crate::config::KmsConfig; use crate::error::KmsError; From 40eee6177a74470b3cb554f39fabe7f30daeb554 Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 31 Jul 2026 19:33:10 +0800 Subject: [PATCH 4/8] test: add formal hotpath ABBA runner (#5507) Co-authored-by: heihutu --- crates/kms/tests/vault_fault_injection.rs | 39 +- docs/operations/hotpath-warp-abba-runbook.md | 277 ++++++++++++ rustfs/src/admin/router.rs | 12 +- scripts/run_hotpath_warp_abba.sh | 428 +++++++++++++++++++ 4 files changed, 733 insertions(+), 23 deletions(-) create mode 100644 docs/operations/hotpath-warp-abba-runbook.md create mode 100755 scripts/run_hotpath_warp_abba.sh diff --git a/crates/kms/tests/vault_fault_injection.rs b/crates/kms/tests/vault_fault_injection.rs index 14176fdb9..f7f86113a 100644 --- a/crates/kms/tests/vault_fault_injection.rs +++ b/crates/kms/tests/vault_fault_injection.rs @@ -33,9 +33,11 @@ use std::time::Duration; use metrics_util::MetricKind; use metrics_util::debugging::{DebugValue, DebuggingRecorder}; -use rustfs_kms::backends::KmsClient; -use rustfs_kms::backends::vault::VaultKmsClient; -use rustfs_kms::{KmsConfig, KmsError, VaultAuthMethod, VaultConfig}; +use rustfs_kms::backends::KmsBackend as KmsBackendTrait; +use rustfs_kms::backends::vault::VaultKmsBackend; +use rustfs_kms::{ + BackendConfig, DescribeKeyRequest, KmsBackend as KmsBackendKind, KmsConfig, KmsError, VaultAuthMethod, VaultConfig, +}; const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; @@ -54,14 +56,23 @@ fn vault_config(address: &str, token: &str) -> VaultConfig { } } -fn kms_config(attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig { +fn kms_config(vault_config: VaultConfig, attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig { KmsConfig { + backend: KmsBackendKind::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(vault_config)), + allow_insecure_dev_defaults: true, timeout: attempt_timeout, retry_attempts, ..KmsConfig::default() } } +fn describe_key_request(key_id: &str) -> DescribeKeyRequest { + DescribeKeyRequest { + key_id: key_id.to_string(), + } +} + type MetricEntry = ( metrics_util::CompositeKey, Option, @@ -118,11 +129,10 @@ fn connection_refused_is_retried_within_budget() { let snapshot = record_metrics(|| { Box::pin(async move { - let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_secs(2), 2)) + let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_secs(2), 2)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-refused", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-refused")) .await .expect_err("a refused connection must fail the operation"); assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); @@ -166,11 +176,10 @@ fn stalled_connection_is_cut_off_by_the_attempt_timeout() { } }); - let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_millis(250), 1)) + let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_millis(250), 1)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-stalled", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled")) .await .expect_err("a stalled request must be cut off by the attempt timeout"); assert!( @@ -201,11 +210,10 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() { let snapshot = record_metrics(|| { Box::pin(async { let config = vault_config(&real_vault_address(), "fault-injection-invalid-token"); - let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-forbidden", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden")) .await .expect_err("an invalid token must be rejected"); assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); @@ -235,11 +243,10 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() { let snapshot = record_metrics(|| { Box::pin(async move { let config = vault_config(&real_vault_address(), &token); - let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3)) + let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3)) .await .expect("client construction performs no network calls"); - let error = client - .describe_key("fault-injection-definitely-missing", None) + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-definitely-missing")) .await .expect_err("a missing key must resolve to key-not-found"); assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}"); diff --git a/docs/operations/hotpath-warp-abba-runbook.md b/docs/operations/hotpath-warp-abba-runbook.md new file mode 100644 index 000000000..5d73b59d1 --- /dev/null +++ b/docs/operations/hotpath-warp-abba-runbook.md @@ -0,0 +1,277 @@ +# Hotpath warp ABBA validation runbook + +This runbook describes how to collect formal Linux or production-cluster +evidence for hotpath performance changes. Use it when a short local A/B smoke +run is too noisy to decide whether a regression is real. + +The ABBA runner executes each workload and drive-sync cell as: + +```text +A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline +``` + +`B1` and `B2` are compared with `A1` to measure the candidate delta. `A2` is +also compared with `A1` to measure baseline drift. Treat a candidate regression +as actionable only when the `A2` drift is passing or materially smaller than +the `B1` and `B2` delta for the same workload. + +## Scope + +Use this runbook for hotpath profiling and performance validation of RustFS +object I/O changes, especially when CPU, memory allocation, lock/channel wait +time, request throughput, or tail latency is the review question. + +The script validates the same workload matrix as the hotpath warp A/B gate: + +| Workload | mode | size | +| --- | --- | --- | +| `put-4kib` | put | 4KiB | +| `put-4mib` | put | 4MiB | +| `get-4kib` | get | 4KiB | +| `get-4mib` | get | 4MiB | +| `get-10mib` | get | 10MiB | +| `mixed-256k` | mixed | 256KiB | + +Each workload runs with `RUSTFS_DRIVE_SYNC_ENABLE=true` and +`RUSTFS_DRIVE_SYNC_ENABLE=false`, so a full ABBA pass produces 48 measurement +cells: 6 workloads x 2 drive-sync modes x 4 ABBA legs. + +## Prerequisites + +Run the formal pass on Linux, not on a laptop smoke environment. + +Required tools on the bench host: + +- `bash`, `curl`, `git`, and core GNU userland. +- `warp` on `PATH`, or pass `--warp-bin`. +- Two RustFS Linux binaries: one baseline and one candidate. +- Enough isolated disks or directories for the local runner, or an externally + managed RustFS cluster for production-like validation. +- Stable host telemetry collection such as `pidstat`, `mpstat`, `iostat`, + `sar`, `perf`, `heaptrack`, or the platform's equivalent observability stack. + +Cluster-mode requirements: + +- A deploy hook that can replace the RustFS binary on every node. +- The hook must apply `RUSTFS_DRIVE_SYNC_ENABLE` for the current ABBA leg. +- The hook must restart RustFS and return only after the rollout command has + been accepted. The ABBA script performs the HTTP readiness wait. +- The benchmark client should run outside the RustFS nodes when possible. +- Do not run against a production data set unless the workload bucket and test + credentials are isolated and approved for destructive benchmark traffic. + +## Build the binaries + +Build the baseline from the comparison commit, usually `origin/main` or the +previous accepted release: + +```bash +git fetch origin main +git switch --detach origin/main +cargo build --release -p rustfs --bins +cp target/release/rustfs /tmp/rustfs-baseline +``` + +Build the candidate from the PR commit: + +```bash +git switch +cargo build --release -p rustfs --bins +cp target/release/rustfs /tmp/rustfs-candidate +``` + +For cross-compiled cluster binaries, keep both outputs on the bench host and +make the deploy hook copy the selected binary to the cluster. The ABBA runner +passes the selected binary path through `HOTPATH_ABBA_BINARY`. + +## Local Linux runner + +Use local mode for a dedicated Linux runner with disposable data paths. This is +not a substitute for a production-like cluster, but it is useful before spending +cluster time. + +```bash +scripts/run_hotpath_warp_abba.sh \ + --baseline-bin /tmp/rustfs-baseline \ + --candidate-bin /tmp/rustfs-candidate \ + --address 127.0.0.1:9000 \ + --data-root /var/tmp/rustfs-hotpath-abba \ + --disks 4 \ + --duration 120s \ + --rounds 3 \ + --cooldown 30 \ + --concurrency 16 \ + --out-dir target/hotpath-abba/linux-local +``` + +The script starts and stops RustFS for each ABBA leg. The data root is +throwaway and should not contain important data. + +## Production-like cluster runner + +Use external mode when RustFS lifecycle is managed by ansible, systemd, a +cluster scheduler, or a dedicated deployment harness. In this mode the ABBA +script does not start RustFS directly; it calls `--deploy-hook` before each leg +and then waits for `http://`. + +The deploy hook receives: + +| Environment variable | Value | +| --- | --- | +| `HOTPATH_ABBA_LEG` | `A1`, `B1`, `B2`, or `A2` | +| `HOTPATH_ABBA_PHASE` | `baseline` or `candidate` | +| `HOTPATH_ABBA_BINARY` | selected baseline or candidate binary path | +| `HOTPATH_ABBA_DRIVE_SYNC` | `true` or `false` | + +Example ansible-shaped command: + +```bash +scripts/run_hotpath_warp_abba.sh \ + --baseline-bin /srv/rustfs-binaries/rustfs-baseline \ + --candidate-bin /srv/rustfs-binaries/rustfs-candidate \ + --endpoint rustfs-bench.example.internal:9000 \ + --deploy-hook ' + set -euo pipefail + cd /srv/rustfs-ansible + cp "${HOTPATH_ABBA_BINARY:?}" roles/rustfs/files/rustfs + export RUSTFS_DRIVE_SYNC_ENABLE="${HOTPATH_ABBA_DRIVE_SYNC:?}" + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags stop + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags config + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags binary-copy + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags start + ' \ + --duration 180s \ + --rounds 5 \ + --cooldown 45 \ + --concurrency 32 \ + --out-dir target/hotpath-abba/cluster-pr-XXXX +``` + +For formal evidence, prefer `--rounds 5` or higher when the cluster budget +allows it. The script enforces `--rounds >= 3`. + +## CPU and memory evidence + +ABBA warp output answers whether the candidate changed throughput or latency. +Collect host telemetry at the same time to explain why. + +Recommended minimum: + +```bash +mkdir -p target/hotpath-abba/cluster-pr-XXXX/telemetry + +pidstat -durh 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/pidstat.txt & +PIDSTAT_PID=$! + +mpstat 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/mpstat.txt & +MPSTAT_PID=$! + +iostat -xz 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/iostat.txt & +IOSTAT_PID=$! +``` + +Stop the collectors after the ABBA script exits: + +```bash +kill "$PIDSTAT_PID" "$MPSTAT_PID" "$IOSTAT_PID" +``` + +For deeper CPU attribution, run `perf record` around one representative +workload after the ABBA gate identifies a candidate regression or improvement: + +```bash +perf record -F 99 -g -- sleep 180 +perf report --stdio > target/hotpath-abba/cluster-pr-XXXX/telemetry/perf-report.txt +``` + +For allocation profiling, build the candidate with: + +```bash +cargo build --release -p rustfs --bins --features hotpath-alloc +``` + +Then run the same ABBA command with that binary. Compare allocation-heavy +function sections only within the same build mode. Do not compare +`hotpath-alloc` binaries directly with default release binaries for throughput +acceptance, because allocation instrumentation intentionally changes what is +measured. + +For CPU hotpath sections emitted by hotpath, build with: + +```bash +cargo build --release -p rustfs --bins --features hotpath-cpu +``` + +Use the CPU-enabled report to explain hotspots after the default or plain +`hotpath` ABBA gate shows a real effect. + +## Output layout + +The ABBA runner writes: + +```text +/ + manifest.env + abba_schedule.csv + candidate_gate.md + baseline_drift_gate.md + summary.md + ///median_summary.csv + ///baseline_compare.csv +``` + +Attach or link at least these files in the issue or PR: + +- `summary.md` +- `candidate_gate.md` +- `baseline_drift_gate.md` +- `abba_schedule.csv` +- every `median_summary.csv` and `baseline_compare.csv` for a failed or + borderline workload +- host telemetry files used to explain CPU, memory, or disk saturation + +## Interpretation + +Use this decision table: + +| Candidate gate | A2 drift gate | Interpretation | +| --- | --- | --- | +| PASS | PASS | Candidate is acceptable for the measured matrix. | +| WARN | PASS | Candidate has a small measurable signal; inspect telemetry and decide if it is expected. | +| FAIL | PASS | Candidate likely regressed the affected workload; investigate before merge. | +| FAIL | FAIL on the same workload | Environment drift is high; rerun on a quieter runner or increase duration and rounds. | +| PASS | FAIL | Candidate did not exceed the budget, but the rig was unstable; avoid using the numbers as proof of improvement. | + +When `B1` and `B2` disagree, treat the result as inconclusive even if the gate +passes. Increase duration, rounds, cooldown, or runner isolation before drawing +a conclusion. + +## AI execution checklist + +When delegating the run to an AI agent or an automation runner, provide these +inputs explicitly: + +- repository checkout and candidate branch or commit; +- baseline commit or binary path; +- candidate binary path; +- runner type: local Linux or external cluster; +- endpoint, access key, secret key source, and region; +- deploy hook path or exact command for cluster mode; +- output directory; +- required duration, rounds, cooldown, concurrency, and fail/warn budgets; +- where to upload artifacts after the run. + +The AI agent should execute this sequence: + +1. Confirm `uname -a`, RustFS commits, binary SHA256 sums, `warp --version`, + CPU model, memory size, disk layout, and whether the run is local or cluster. +2. Run `scripts/run_hotpath_warp_abba.sh --dry-run` with the final arguments. +3. Run the real ABBA command with `--rounds >= 3`. +4. Preserve the full output directory without editing generated CSV files. +5. Read `summary.md`, `candidate_gate.md`, and `baseline_drift_gate.md`. +6. Summarize only measured facts: candidate deltas, baseline drift, CPU or + memory saturation, and any failed workloads. +7. Post the summary and artifact location to the tracking issue or PR. + +Do not report a performance win or loss when the baseline drift gate failed on +the same workload and no rerun was collected. diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index a72858a8a..7bec6c88a 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -2708,18 +2708,16 @@ impl S3Router { pub fn insert(&mut self, method: Method, path: &str, operation: T) -> std::io::Result<()> { let path = Self::make_route_str(method, path); + #[cfg(test)] + let registered_path = path.clone(); // warn!("set uri {}", &path); - #[cfg(test)] - { - self.router.insert(path.clone(), operation).map_err(std::io::Error::other)?; - self.registered_routes.push(path); - } - - #[cfg(not(test))] self.router.insert(path, operation).map_err(std::io::Error::other)?; + #[cfg(test)] + self.registered_routes.push(registered_path); + Ok(()) } diff --git a/scripts/run_hotpath_warp_abba.sh b/scripts/run_hotpath_warp_abba.sh new file mode 100755 index 000000000..ccaff476d --- /dev/null +++ b/scripts/run_hotpath_warp_abba.sh @@ -0,0 +1,428 @@ +#!/usr/bin/env bash +# Formal Linux / production-cluster ABBA runner for the hotpath warp matrix. +# +# This script is intentionally a thin orchestrator around the existing +# run_object_batch_bench_enhanced.sh load driver and hotpath_warp_ab_gate.sh +# relative-budget gate. It runs each durability/workload cell as: +# +# A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline +# +# Candidate legs are compared against A1. The final A2 leg is also compared +# against A1 to quantify baseline drift separately from candidate deltas. + +set -euo pipefail + +PROJECT_ROOT="$(git rev-parse --show-toplevel)" +ENHANCED_BENCH="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh" +GATE="${PROJECT_ROOT}/scripts/hotpath_warp_ab_gate.sh" + +BASELINE_BIN="" +CANDIDATE_BIN="" +ENDPOINT="" +DEPLOY_HOOK="" +HEALTH_PATH="/health" +ADDRESS="127.0.0.1:9000" +DATA_ROOT="/tmp/rustfs-hotpath-abba" +DISKS=4 +ACCESS_KEY="rustfsadmin" +SECRET_KEY="rustfsadmin" +REGION="us-east-1" +WARP_BIN="warp" +CONCURRENCY=8 +DURATION="60s" +ROUNDS=3 +COOLDOWN_SECS=20 +HEALTH_TIMEOUT_SECS=180 +FAIL_PCT=10 +WARN_PCT=5 +ALLOW_REGRESSION=false +EXEMPTION_REASON="deliberate correctness tradeoff" +OUT_DIR="${PROJECT_ROOT}/target/hotpath-abba/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)" +DRY_RUN=false + +WORKLOADS=( + "put-4kib|put|4KiB" + "put-4mib|put|4MiB" + "get-4kib|get|4KiB" + "get-4mib|get|4MiB" + "get-10mib|get|10MiB" + "mixed-256k|mixed|256KiB" +) +DRIVE_SYNC_MATRIX=("sync-on|true" "sync-off|false") + +usage() { + cat <<'USAGE' +Usage: scripts/run_hotpath_warp_abba.sh --baseline-bin --candidate-bin [options] + +Formal ABBA mode for Linux runners or production-like clusters. The schedule is +A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline for every workload +and drive-sync cell. + +Required: + --baseline-bin Baseline RustFS binary. + --candidate-bin Candidate RustFS binary. + +Local Linux runner mode: + --address Local RustFS address (default 127.0.0.1:9000). + --disks Throwaway local disks per node (default 4). + --data-root Local disk root (default /tmp/rustfs-hotpath-abba). + +Production / cluster mode: + --endpoint Existing cluster endpoint. Enables external mode. + --deploy-hook Command run before each ABBA leg. It receives: + HOTPATH_ABBA_LEG=A1|B1|B2|A2 + HOTPATH_ABBA_PHASE=baseline|candidate + HOTPATH_ABBA_BINARY= + HOTPATH_ABBA_DRIVE_SYNC=true|false + --health-path Readiness path (default /health). + +Benchmark: + --duration warp duration per cell (default 60s). + --rounds rounds per cell; must be >= 3 (default 3). + --cooldown cooldown seconds between rounds/sizes (default 20). + --concurrency warp concurrency (default 8). + --warp-bin warp binary (default warp). + +Credentials: + --access-key S3 access key (default rustfsadmin). + --secret-key S3 secret key (default rustfsadmin). + --region S3 region (default us-east-1). + +Gate: + --fail-pct Regression budget that fails gate (default 10). + --warn-pct Regression budget that warns (default 5). + --allow-regression Downgrade candidate gate FAIL to WARN. + --exemption-reason Reason recorded when allow-regression is used. + +Output: + --out-dir Output dir (default target/hotpath-abba/). + --dry-run Print commands without starting servers or warp. + -h, --help + +Outputs: + /abba_schedule.csv + /candidate_gate.md + /baseline_drift_gate.md + /summary.md + ////{median_summary.csv,baseline_compare.csv} +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +log() { + printf '[hotpath-abba] %s\n' "$*" >&2 +} + +run() { + if [[ "$DRY_RUN" == "true" ]]; then + { printf 'DRY-RUN:'; printf ' %q' "$@"; printf '\n'; } >&2 + return 0 + fi + "$@" +} + +validate_positive_int() { + local value="$1" name="$2" + [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --baseline-bin) BASELINE_BIN="$2"; shift 2 ;; + --candidate-bin) CANDIDATE_BIN="$2"; shift 2 ;; + --endpoint) ENDPOINT="$2"; shift 2 ;; + --deploy-hook) DEPLOY_HOOK="$2"; shift 2 ;; + --health-path) HEALTH_PATH="$2"; shift 2 ;; + --address) ADDRESS="$2"; shift 2 ;; + --data-root) DATA_ROOT="$2"; shift 2 ;; + --disks) DISKS="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --cooldown) COOLDOWN_SECS="$2"; shift 2 ;; + --health-timeout) HEALTH_TIMEOUT_SECS="$2"; shift 2 ;; + --fail-pct) FAIL_PCT="$2"; shift 2 ;; + --warn-pct) WARN_PCT="$2"; shift 2 ;; + --allow-regression) ALLOW_REGRESSION=true; shift ;; + --exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +validate_positive_int "$DISKS" "--disks" +validate_positive_int "$CONCURRENCY" "--concurrency" +validate_positive_int "$ROUNDS" "--rounds" +validate_positive_int "$COOLDOWN_SECS" "--cooldown" +validate_positive_int "$HEALTH_TIMEOUT_SECS" "--health-timeout" +validate_positive_int "$FAIL_PCT" "--fail-pct" +validate_positive_int "$WARN_PCT" "--warn-pct" +[[ "$ROUNDS" -ge 3 ]] || die "--rounds must be >= 3 for formal ABBA evidence" +[[ -n "$BASELINE_BIN" ]] || die "--baseline-bin is required" +[[ -n "$CANDIDATE_BIN" ]] || die "--candidate-bin is required" +[[ "$DRY_RUN" == "true" || -x "$BASELINE_BIN" ]] || die "baseline binary is not executable: $BASELINE_BIN" +[[ "$DRY_RUN" == "true" || -x "$CANDIDATE_BIN" ]] || die "candidate binary is not executable: $CANDIDATE_BIN" +[[ -x "$ENHANCED_BENCH" ]] || die "missing load driver: $ENHANCED_BENCH" +[[ -x "$GATE" ]] || die "missing gate: $GATE" +if [[ "$DRY_RUN" != "true" ]] && ! command -v "$WARP_BIN" >/dev/null 2>&1; then + die "warp not found on PATH; install warp or pass --warp-bin" +fi + +EXTERNAL=false +if [[ -n "$ENDPOINT" ]]; then + EXTERNAL=true + ADDRESS="$ENDPOINT" + [[ -n "$DEPLOY_HOOK" ]] || log "warning: external mode without --deploy-hook; binaries must be swapped out of band" +fi + +mkdir -p "$OUT_DIR" +SERVER_LOG_DIR="$OUT_DIR/server-logs" +run mkdir -p "$SERVER_LOG_DIR" + +SERVER_PID="" +SERVER_LOG="" +tear_down() { + [[ "$EXTERNAL" == "true" ]] && return 0 + [[ -n "$SERVER_PID" ]] || return 0 + run kill "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" +} +trap tear_down EXIT INT TERM + +dump_server_log() { + [[ -n "$SERVER_LOG" && -f "$SERVER_LOG" ]] || return 0 + echo "----- last 80 lines of $SERVER_LOG -----" >&2 + tail -n 80 "$SERVER_LOG" >&2 || true + echo "----------------------------------------" >&2 +} + +wait_health() { + [[ "$DRY_RUN" == "true" ]] && return 0 + local i + for ((i = 0; i < HEALTH_TIMEOUT_SECS; i++)); do + if [[ "$EXTERNAL" != "true" && -n "$SERVER_PID" ]] && ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo "error: rustfs server (pid $SERVER_PID) exited before becoming healthy after ${i}s" >&2 + dump_server_log + return 1 + fi + if curl -fsS "http://${ADDRESS}${HEALTH_PATH}" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "error: endpoint http://${ADDRESS}${HEALTH_PATH} did not become healthy within ${HEALTH_TIMEOUT_SECS}s" >&2 + dump_server_log + return 1 +} + +binary_for_leg() { + case "$1" in + A1|A2) echo "$BASELINE_BIN" ;; + B1|B2) echo "$CANDIDATE_BIN" ;; + *) die "unknown ABBA leg: $1" ;; + esac +} + +phase_for_leg() { + case "$1" in + A1|A2) echo "baseline" ;; + B1|B2) echo "candidate" ;; + *) die "unknown ABBA leg: $1" ;; + esac +} + +bring_up() { + local leg="$1" drive_sync="$2" + local phase bin + phase="$(phase_for_leg "$leg")" + bin="$(binary_for_leg "$leg")" + + if [[ "$EXTERNAL" == "true" ]]; then + if [[ -n "$DEPLOY_HOOK" ]]; then + log "deploy hook: leg=$leg phase=$phase drive_sync=$drive_sync" + HOTPATH_ABBA_LEG="$leg" HOTPATH_ABBA_PHASE="$phase" HOTPATH_ABBA_BINARY="$bin" HOTPATH_ABBA_DRIVE_SYNC="$drive_sync" \ + run bash -c "$DEPLOY_HOOK" + fi + wait_health + return 0 + fi + + local node_dir="$DATA_ROOT/$leg-sync-$drive_sync" + local disks=() d + for ((d = 1; d <= DISKS; d++)); do + disks+=("$node_dir/d$d") + done + run mkdir -p "${disks[@]}" + SERVER_LOG="$SERVER_LOG_DIR/$leg-sync-$drive_sync.log" + + if [[ "$DRY_RUN" == "true" ]]; then + { printf 'DRY-RUN: RUSTFS_DRIVE_SYNC_ENABLE=%s %q server' "$drive_sync" "$bin" + printf ' %q' "${disks[@]}"; printf '\n'; } >&2 + SERVER_PID="dry-run" + return 0 + fi + + cat >"$SERVER_LOG_DIR/$leg-sync-$drive_sync.env" </dev/null || echo unknown) +warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown) +EOF + RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \ + RUSTFS_ADDRESS="$ADDRESS" \ + RUSTFS_ACCESS_KEY="$ACCESS_KEY" \ + RUSTFS_SECRET_KEY="$SECRET_KEY" \ + RUSTFS_REGION="$REGION" \ + RUSTFS_CONSOLE_ENABLE=false \ + RUSTFS_DRIVE_SYNC_ENABLE="$drive_sync" \ + "$bin" server "${disks[@]}" >"$SERVER_LOG" 2>&1 & + SERVER_PID=$! + wait_health +} + +measure() { + local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" baseline_csv="${6:-}" + local cell="$OUT_DIR/$workload/$sync_label/$leg" + local args=( + --tool warp --warp-bin "$WARP_BIN" --warp-mode "$mode" + --endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" + --region "$REGION" --sizes "$size" --concurrency "$CONCURRENCY" + --duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS" + --out-dir "$cell" + ) + [[ -n "$baseline_csv" ]] && args+=(--baseline-csv "$baseline_csv") + run "$ENHANCED_BENCH" "${args[@]}" >&2 + echo "$cell" +} + +write_schedule_header() { + echo "sync_label,drive_sync,workload,mode,size,leg,phase,binary,out_dir" >"$OUT_DIR/abba_schedule.csv" +} + +append_schedule() { + local sync_label="$1" drive_sync="$2" workload="$3" mode="$4" size="$5" leg="$6" + local phase bin + phase="$(phase_for_leg "$leg")" + bin="$(binary_for_leg "$leg")" + echo "$sync_label,$drive_sync,$workload,$mode,$size,$leg,$phase,$bin,$OUT_DIR/$workload/$sync_label/$leg" >>"$OUT_DIR/abba_schedule.csv" +} + +write_manifest() { + cat >"$OUT_DIR/manifest.env" </dev/null || echo unknown) +runner=$(uname -srm 2>/dev/null || echo unknown) +schedule=ABBA +rounds=$ROUNDS +duration=$DURATION +cooldown_secs=$COOLDOWN_SECS +concurrency=$CONCURRENCY +baseline_bin=$BASELINE_BIN +candidate_bin=$CANDIDATE_BIN +external=$EXTERNAL +endpoint=$ADDRESS +warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown) +EOF +} + +declare -a CANDIDATE_COMPARE_CSVS=() +declare -a DRIFT_COMPARE_CSVS=() + +write_manifest +write_schedule_header + +for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do + IFS='|' read -r sync_label drive_sync <<<"$ds_spec" + + for leg in A1 B1 B2 A2; do + log "=== $sync_label leg $leg ($(phase_for_leg "$leg")) ===" + bring_up "$leg" "$drive_sync" + + for wl_spec in "${WORKLOADS[@]}"; do + IFS='|' read -r workload mode size <<<"$wl_spec" + append_schedule "$sync_label" "$drive_sync" "$workload" "$mode" "$size" "$leg" + + baseline_csv="" + if [[ "$leg" != "A1" ]]; then + baseline_csv="$OUT_DIR/$workload/$sync_label/A1/median_summary.csv" + fi + + cell="$(measure "$leg" "$workload" "$mode" "$size" "$sync_label" "$baseline_csv")" + case "$leg" in + B1|B2) CANDIDATE_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;; + A2) DRIFT_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;; + esac + done + + tear_down + done +done + +gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/candidate_gate.md") +for csv in "${CANDIDATE_COMPARE_CSVS[@]}"; do + gate_args+=(--compare-csv "$csv") +done +[[ "$ALLOW_REGRESSION" == "true" ]] && gate_args+=(--allow-regression --exemption-reason "$EXEMPTION_REASON") + +drift_gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/baseline_drift_gate.md") +for csv in "${DRIFT_COMPARE_CSVS[@]}"; do + drift_gate_args+=(--compare-csv "$csv") +done + +if [[ "$DRY_RUN" == "true" ]]; then + log "dry-run complete; candidate compare CSVs=${#CANDIDATE_COMPARE_CSVS[@]} baseline drift CSVs=${#DRIFT_COMPARE_CSVS[@]}" + { printf 'DRY-RUN:'; printf ' %q' "$GATE" "${gate_args[@]}"; printf '\n'; } >&2 + { printf 'DRY-RUN:'; printf ' %q' "$GATE" "${drift_gate_args[@]}"; printf '\n'; } >&2 + exit 0 +fi + +log "applying candidate relative-budget gate" +set +e +"$GATE" "${gate_args[@]}" +candidate_status=$? +set -e + +log "applying A2-vs-A1 baseline drift gate" +set +e +"$GATE" "${drift_gate_args[@]}" +drift_status=$? +set -e + +cat >"$OUT_DIR/summary.md" < B1 candidate -> B2 candidate -> A2 baseline +- runner: $(uname -srm 2>/dev/null || echo unknown) +- warp: $("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown) +- matrix: duration=$DURATION rounds=$ROUNDS cooldown=$COOLDOWN_SECS disks=$DISKS concurrency=$CONCURRENCY +- endpoint: $ADDRESS +- baseline binary: $BASELINE_BIN +- candidate binary: $CANDIDATE_BIN +- candidate gate: $OUT_DIR/candidate_gate.md (exit $candidate_status) +- baseline drift gate: $OUT_DIR/baseline_drift_gate.md (exit $drift_status) + +Interpretation: + +- Treat candidate gate failures as actionable only when the A2-vs-A1 drift gate is PASS or the affected workload's A2 drift is materially smaller than the B1/B2 candidate delta. +- If both candidate and baseline drift fail on the same workload, rerun with longer duration, more rounds, or a quieter runner before assigning causality. +EOF + +log "summary written to $OUT_DIR/summary.md" +if [[ "$candidate_status" -ne 0 || "$drift_status" -ne 0 ]]; then + exit 1 +fi From db8039dece301279226ea7b035af7d8890321cc6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 31 Jul 2026 23:57:39 +0800 Subject: [PATCH 5/8] feat(kms): export local backend key material as sealed backup bundles (#5499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): export local backend key material as sealed backup bundles Adds the producer side of the Local backup series on top of the #5483 contract: a directory-wide export fence gives the snapshot a single consistent generation, every artifact is AEAD-wrapped under a caller-supplied backup KEK that is separate from the business trust hierarchy, and the sealed manifest with completeness marker is written last so an interrupted export can never be mistaken for a restorable bundle. Restore and the admin API land in follow-up changes. * fix(kms): verify manifest digest against raw bytes, not re-serialized fields The decode path recomputed the digest by re-serializing the parsed manifest, which silently assumes every field's stored spelling survives a parse-and-reprint round trip. Timestamps do not guarantee that: the time zone annotation jiff emits depends on the host (IANA name, POSIX TZ string, Etc/Unknown), and the legacy-compat parser rewrites bracket-less spellings to +00:00[UTC]. On CI this made freshly written bundles fail digest verification while passing locally. Digest verification now operates on the raw stored bytes, normalized only through the JSON value layer with the digest slot emptied in place; parsed typed fields are never re-serialized on the decode path. Sealing uses the same value-layer canonical form, and the export additionally pins created_at to UTC so bundles are host-independent. A regression test seals a manifest whose created_at spelling cannot round-trip and proves decoding still verifies. One behavior sharpens: inserting an explicit null reserved slot after sealing is now rejected as a digest mismatch instead of being tolerated. * fix(kms): make manifest digest canonicalization independent of map ordering The canonical digest form serialized serde_json values directly, which inherits the key order of serde_json's map type: sorted by default, but insertion-ordered when any crate in the unified build graph enables the preserve_order feature. The workspace-wide CI build unified that feature while a per-crate local build did not, so the frozen fixture digest matched in one environment and not the other — and a bundle sealed by one build flavor would fail verification in the other. Canonicalization now rebuilds every JSON object with bytewise-sorted keys (array order preserved) before hashing, so the digest bytes are identical regardless of feature unification. Reproduced by enabling preserve_order in dev-dependencies (fixture test red), then verified green with the fix under both map flavors. --- crates/kms/src/backends/local.rs | 53 +- crates/kms/src/backup/local_export.rs | 999 ++++++++++++++++++++++++++ crates/kms/src/backup/manifest.rs | 159 +++- crates/kms/src/backup/mod.rs | 17 +- 4 files changed, 1195 insertions(+), 33 deletions(-) create mode 100644 crates/kms/src/backup/local_export.rs diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index e1e347cca..1a12b8593 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -397,6 +397,20 @@ pub struct LocalKmsClient { /// Per-key write locks serializing read-modify-write updates within this /// process (see [`Self::lock_key_for_write`]). key_write_locks: Mutex>>>, + /// Directory-wide writer fence for backup export (see + /// [`Self::acquire_export_fence`]). Writers hold the read side; an export + /// snapshot holds the write side so it observes a single-generation view. + export_fence: Arc>, +} + +/// Guard pairing the export-fence read lock with a per-key write mutex. +/// +/// Dropping it releases both, so every existing `lock_key_for_write` call +/// site participates in the export fence without changes. +#[must_use] +struct KeyWriteGuard { + _fence: tokio::sync::OwnedRwLockReadGuard<()>, + _key: tokio::sync::OwnedMutexGuard<()>, } // pub(crate) so the backup contract tests can anchor the manifest's @@ -463,6 +477,7 @@ impl LocalKmsClient { legacy_master_cipher, dek_crypto: AesDekCrypto::new(), key_write_locks: Mutex::new(HashMap::new()), + export_fence: Arc::new(tokio::sync::RwLock::new(())), }; client.validate_existing_keys().await?; Ok(client) @@ -505,6 +520,7 @@ impl LocalKmsClient { legacy_master_cipher, dek_crypto: AesDekCrypto::new(), key_write_locks: Mutex::new(HashMap::new()), + export_fence: Arc::new(tokio::sync::RwLock::new(())), }) } @@ -515,12 +531,40 @@ impl LocalKmsClient { /// delete with a rewrite. Cross-process writers sharing a key directory /// remain unsupported. Entries live for the client's lifetime; the table /// is bounded by the number of distinct key ids this process touches. - async fn lock_key_for_write(&self, key_id: &str) -> tokio::sync::OwnedMutexGuard<()> { + async fn lock_key_for_write(&self, key_id: &str) -> KeyWriteGuard { + // Fence first, per-key mutex second: the ordering is uniform across + // all writers, so an export waiting on the write side can never + // deadlock with a writer holding a key mutex. + let fence = Arc::clone(&self.export_fence).read_owned().await; let lock = { let mut locks = self.key_write_locks.lock().expect("Local KMS key write lock table poisoned"); Arc::clone(locks.entry(key_id.to_string()).or_default()) }; - lock.lock_owned().await + KeyWriteGuard { + _fence: fence, + _key: lock.lock_owned().await, + } + } + + /// Block every key-directory writer while a backup export collects its + /// snapshot, so all records belong to one generation. + /// + /// Mutating operations hold the read side (via [`Self::lock_key_for_write`] + /// or [`Self::save_new_master_key`]); the export holds the write side only + /// for the collection phase, never while encrypting or writing the bundle. + pub(crate) async fn acquire_export_fence(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> { + Arc::clone(&self.export_fence).write_owned().await + } + + /// Key directory root, exposed for the backup export module. + pub(crate) fn key_directory(&self) -> &Path { + &self.config.key_dir + } + + /// Absolute path of the master-key KDF salt file, exposed for the backup + /// export module. + pub(crate) fn master_key_salt_file(&self) -> PathBuf { + Self::master_key_salt_path(&self.config) } /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. @@ -797,6 +841,11 @@ impl LocalKmsClient { } async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> { + // Creates never take the per-key write lock (`NoClobber` publishing + // already linearizes them), so they join the export fence here. This + // must stay the only fence acquisition on the create path: the fence + // read lock is not reentrant while an export waits for the write side. + let _fence = Arc::clone(&self.export_fence).read_owned().await; let key_path = self.master_key_path(&master_key.key_id)?; let content = self.encode_master_key(master_key, key_material)?; let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4())); diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs new file mode 100644 index 000000000..d3905b31f --- /dev/null +++ b/crates/kms/src/backup/local_export.rs @@ -0,0 +1,999 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Local backend backup export: sealed, KEK-protected bundle production. +//! +//! This is the producer side only; restore lives in a follow-up change. The +//! admin API is not wired here either — callers construct the request and +//! supply the backup KEK explicitly. +//! +//! # Bundle layout +//! +//! A bundle is a directory (simple to produce, artifacts stream one file at a +//! time, and partial output is trivially recognizable because the manifest is +//! written last): +//! +//! ```text +//! / +//! manifest.json # sealed BackupManifest, written last +//! artifacts/keys/.key.enc # one per stored key record +//! artifacts/master-key.salt.enc # present when the salt file exists +//! ``` +//! +//! # Artifact payload framing +//! +//! Every artifact payload is `nonce (12 bytes) || AES-256-GCM ciphertext`, +//! encrypted under the caller-supplied backup KEK with an AAD binding of +//! `(context, backup_id, snapshot_generation, artifact path)`, so an artifact +//! cannot be swapped into another bundle or renamed within its own bundle. +//! Records already encrypted at rest stay encrypted inside the wrap; +//! plaintext-dev-only records become ciphertext-only in the bundle, which is +//! their mandatory re-wrap under the backup KEK. +//! +//! # Write protocol +//! +//! Artifacts are written and fsynced first, re-read and digest-verified, and +//! only then is the sealed manifest (completeness marker plus final digest) +//! published. A crash at any earlier point leaves a bundle without a +//! manifest, which decodes as an incomplete bundle and can never be restored. + +use crate::backends::local::{LocalKmsClient, StoredKeyProtection}; +use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; +use crate::backup::error::BackupError; +use crate::backup::manifest::{ + AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, + DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, +}; +use crate::error::{KmsError, Result}; +use aes_gcm::{ + Aes256Gcm, Key, Nonce, + aead::{Aead, KeyInit, Payload}, +}; +use jiff::Zoned; +use rand::RngExt; +use serde::Deserialize; +use std::path::{Path, PathBuf}; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use zeroize::Zeroizing; + +/// File name of the sealed manifest inside a bundle directory. +pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json"; +const ARTIFACTS_DIR: &str = "artifacts"; +const KEYS_DIR: &str = "artifacts/keys"; +const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc"; +const AEAD_NONCE_LEN: usize = 12; +/// Domain-separation context for the artifact AAD binding. +const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1"; + +/// Caller-supplied backup KEK: a trust root deliberately separate from the +/// business KMS hierarchy (it must not be a key that is itself part of the +/// state being backed up). Where the KEK comes from is the admin layer's +/// concern; this module only consumes it. +pub struct BackupKek { + kek_id: String, + kek_version: u32, + key: Zeroizing<[u8; 32]>, +} + +impl BackupKek { + /// Wrap 32 bytes of KEK material. The material is zeroized on drop; + /// callers should zeroize their own copy of the input. + pub fn new(kek_id: impl Into, kek_version: u32, key: [u8; 32]) -> Result { + let kek_id = kek_id.into(); + if kek_id.is_empty() { + return Err(KmsError::validation_error("backup KEK id must not be empty")); + } + Ok(Self { + kek_id, + kek_version, + key: Zeroizing::new(key), + }) + } + + /// Manifest descriptor for this KEK. + pub fn descriptor(&self) -> BackupKekDescriptor { + BackupKekDescriptor { + kek_id: self.kek_id.clone(), + kek_version: self.kek_version, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + } + } + + fn cipher(&self) -> Aes256Gcm { + Aes256Gcm::new(&Key::::from(*self.key)) + } +} + +/// Parameters of one export run. +/// +/// `snapshot_generation` is injected by the caller: the contract only +/// requires it to be monotonic per deployment, and the source (persisted +/// counter, coordinated clock) is decided by the admin layer, which keeps +/// this module free of ambient time or state lookups. +#[derive(Debug, Clone)] +pub struct LocalBackupExportRequest { + /// Unique identifier for this backup. + pub backup_id: String, + /// Opaque identity of the producing deployment. + pub deployment_identity: String, + /// RustFS version string recorded in the manifest. + pub rustfs_version: String, + /// Monotonic snapshot generation this bundle belongs to. + pub snapshot_generation: u64, + /// Bundle output directory; must not exist yet or must be empty. + pub destination: PathBuf, +} + +impl LocalBackupExportRequest { + fn validate(&self) -> Result<()> { + for (field, value) in [ + ("backup_id", &self.backup_id), + ("deployment_identity", &self.deployment_identity), + ("rustfs_version", &self.rustfs_version), + ] { + if value.is_empty() { + return Err(KmsError::validation_error(format!("backup export {field} must not be empty"))); + } + } + Ok(()) + } +} + +/// Minimal projection of a stored key record: only the fields the exporter +/// needs. Unknown fields are ignored on purpose — the record travels into the +/// bundle byte-identical, so the exporter must not constrain its schema. +#[derive(Deserialize)] +struct StoredRecordProbe { + key_id: String, + #[serde(default)] + at_rest_protection: StoredKeyProtection, +} + +struct CollectedRecord { + key_id: String, + protection: StoredKeyProtection, + /// Raw record bytes exactly as stored. Zeroized on drop because + /// plaintext-dev-only records embed key material. + raw: Zeroizing>, +} + +struct CollectedSnapshot { + records: Vec, + salt: Option>, +} + +/// Export the Local backend's key directory as a sealed backup bundle. +/// +/// The directory scan runs under the export fence, so concurrent +/// create/update/delete operations are either fully included or fully +/// excluded — never half a record. Encryption and bundle writing happen after +/// the fence is released to keep it short. +/// +/// Returns the sealed manifest that was written to the bundle. +pub async fn export_local_backup( + client: &LocalKmsClient, + kek: &BackupKek, + request: &LocalBackupExportRequest, +) -> Result { + request.validate()?; + prepare_destination(&request.destination).await?; + + let snapshot = collect_snapshot(client).await?; + if snapshot.records.is_empty() { + return Err(KmsError::invalid_operation( + "Local backup export found no key records; refusing to publish an empty bundle", + )); + } + + let has_encrypted = snapshot + .records + .iter() + .any(|record| record.protection == StoredKeyProtection::EncryptedMasterKey); + if has_encrypted && snapshot.salt.is_none() { + return Err(KmsError::invalid_operation( + "key directory contains encrypted-master-key records but the master key salt file is missing; \ + the bundle would be unrestorable", + )); + } + + let manifest = build_and_write_bundle(kek, request, &snapshot).await?; + Ok(manifest) +} + +/// Read and fully validate the manifest of a local bundle directory. +/// +/// A directory without a manifest is an interrupted export: the manifest is +/// written last, so its absence means the bundle never sealed. +pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result { + let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE); + let bytes = match fs::read(&manifest_path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(BackupError::incomplete_bundle("bundle has no manifest; the export never sealed it").into()); + } + Err(error) => return Err(error.into()), + }; + let manifest = BackupManifest::decode(&bytes)?; + if manifest.backend != BackupBackendKind::Local { + return Err( + BackupError::corrupted(format!("bundle manifest declares backend {:?}, expected Local", manifest.backend)).into(), + ); + } + Ok(manifest) +} + +/// Read, verify, and decrypt one artifact of a local bundle. +/// +/// Fail-closed order: KEK identity, artifact presence, declared length, +/// encrypted digest, then AEAD authentication. The returned plaintext is +/// zeroized on drop. +pub async fn decrypt_bundle_artifact( + bundle_dir: &Path, + manifest: &BackupManifest, + descriptor: &ArtifactDescriptor, + kek: &BackupKek, +) -> Result>> { + manifest.backup_kek.ensure_matches(&kek.kek_id, kek.kek_version)?; + if descriptor.aead_algorithm != AeadAlgorithm::Aes256Gcm { + return Err(KmsError::unsupported_algorithm(format!( + "{:?} (local bundles are produced with AES-256-GCM)", + descriptor.aead_algorithm + ))); + } + + let artifact_path = bundle_dir.join(&descriptor.path); + let payload = match fs::read(&artifact_path).await { + Ok(payload) => payload, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(BackupError::missing_artifact(descriptor.path.clone()).into()); + } + Err(error) => return Err(error.into()), + }; + + if (payload.len() as u64) < descriptor.len { + return Err(BackupError::truncated(format!( + "artifact '{}' is {} bytes, manifest declares {}", + descriptor.path, + payload.len(), + descriptor.len + )) + .into()); + } + if payload.len() as u64 != descriptor.len { + return Err(BackupError::corrupted(format!( + "artifact '{}' is {} bytes, manifest declares {}", + descriptor.path, + payload.len(), + descriptor.len + )) + .into()); + } + if ContentDigest::sha256_of(&payload) != descriptor.encrypted_digest { + return Err(BackupError::corrupted(format!("artifact '{}' does not match its manifest digest", descriptor.path)).into()); + } + if payload.len() < AEAD_NONCE_LEN { + return Err(BackupError::corrupted(format!("artifact '{}' is too short to carry a nonce", descriptor.path)).into()); + } + + let (nonce_bytes, ciphertext) = payload.split_at(AEAD_NONCE_LEN); + let mut nonce = [0u8; AEAD_NONCE_LEN]; + nonce.copy_from_slice(nonce_bytes); + let aad = artifact_aad(&manifest.backup_id, manifest.snapshot_generation, &descriptor.path); + let plaintext = kek + .cipher() + .decrypt( + &Nonce::from(nonce), + Payload { + msg: ciphertext, + aad: &aad, + }, + ) + .map_err(|_| { + KmsError::from(BackupError::corrupted(format!( + "artifact '{}' failed authenticated decryption under the supplied backup KEK", + descriptor.path + ))) + })?; + Ok(Zeroizing::new(plaintext)) +} + +/// Scan the key directory under the export fence. +async fn collect_snapshot(client: &LocalKmsClient) -> Result { + let _fence = client.acquire_export_fence().await; + + let mut records = Vec::new(); + let mut entries = fs::read_dir(client.key_directory()).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + if !path.extension().is_some_and(|extension| extension == "key") { + continue; + } + let stem = path + .file_stem() + .and_then(|stem| stem.to_str()) + .ok_or_else(|| KmsError::configuration_error("Local KMS key file name must be valid UTF-8"))? + .to_string(); + + let raw = Zeroizing::new(fs::read(&path).await?); + // Any unreadable record aborts the export: a bundle silently missing + // one key is worse than no bundle at all. + let probe: StoredRecordProbe = serde_json::from_slice(&raw) + .map_err(|error| KmsError::material_corrupt(&stem, format!("stored key record does not deserialize: {error}")))?; + if probe.key_id != stem { + return Err(KmsError::invalid_key(format!( + "Local KMS key file identity mismatch: expected {stem:?}, found {:?}", + probe.key_id + ))); + } + + records.push(CollectedRecord { + key_id: stem, + protection: probe.at_rest_protection, + raw, + }); + } + + let salt_path = client.master_key_salt_file(); + let salt = match fs::read(&salt_path).await { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + + records.sort_by(|a, b| a.key_id.cmp(&b.key_id)); + Ok(CollectedSnapshot { records, salt }) +} + +async fn build_and_write_bundle( + kek: &BackupKek, + request: &LocalBackupExportRequest, + snapshot: &CollectedSnapshot, +) -> Result { + let mut artifacts = Vec::with_capacity(snapshot.records.len() + 1); + for record in &snapshot.records { + let artifact_path = format!("{KEYS_DIR}/{}.key.enc", record.key_id); + let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::KeyMaterial, &artifact_path, &record.raw).await?; + artifacts.push(descriptor); + } + if let Some(salt) = &snapshot.salt { + let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::MasterKeySalt, SALT_ARTIFACT_PATH, salt).await?; + artifacts.push(descriptor); + } + + // Make the artifact directory entries durable before sealing: the sealed + // manifest must never survive a crash that its artifacts did not. + fsync_dir(&request.destination.join(KEYS_DIR)).await?; + fsync_dir(&request.destination.join(ARTIFACTS_DIR)).await?; + + let manifest = BackupManifest { + format_version: BackupManifest::FORMAT_VERSION, + backup_id: request.backup_id.clone(), + // Normalized to UTC so the stored spelling is host-independent: the + // local zone's name (or a POSIX TZ string in minimal containers) has + // no business inside a portable bundle. + created_at: Zoned::now().with_time_zone(jiff::tz::TimeZone::UTC), + rustfs_version: request.rustfs_version.clone(), + deployment_identity: request.deployment_identity.clone(), + backend: BackupBackendKind::Local, + at_rest_protection: weakest_observed_protection(&snapshot.records), + responsibility: BackupResponsibility::FullMaterial, + snapshot_generation: request.snapshot_generation, + backup_kek: kek.descriptor(), + artifacts, + local_kdf: Some(local_kdf_descriptor(snapshot)), + key_versions: None, + capability_discovery: None, + completeness: CompletenessState::InProgress, + manifest_digest: ContentDigest { + algorithm: DigestAlgorithm::Sha256, + hex: String::new(), + }, + }; + let manifest = manifest.seal()?; + let manifest_bytes = manifest.encode()?; + + write_new_file(&request.destination.join(LOCAL_BUNDLE_MANIFEST_FILE), &manifest_bytes).await?; + fsync_dir(&request.destination).await?; + Ok(manifest) +} + +/// Encrypt one artifact, write it durably, and re-read it to verify the +/// digest before it is allowed into the manifest. +async fn encrypt_and_write_artifact( + kek: &BackupKek, + request: &LocalBackupExportRequest, + kind: ArtifactKind, + artifact_path: &str, + plaintext: &[u8], +) -> Result { + let mut nonce = [0u8; AEAD_NONCE_LEN]; + rand::rng().fill(&mut nonce[..]); + let aad = artifact_aad(&request.backup_id, request.snapshot_generation, artifact_path); + let ciphertext = kek + .cipher() + .encrypt( + &Nonce::from(nonce), + Payload { + msg: plaintext, + aad: &aad, + }, + ) + .map_err(|error| KmsError::cryptographic_error("backup_artifact_encrypt", error.to_string()))?; + + let mut payload = Vec::with_capacity(AEAD_NONCE_LEN + ciphertext.len()); + payload.extend_from_slice(&nonce); + payload.extend_from_slice(&ciphertext); + + let absolute_path = request.destination.join(artifact_path); + write_new_file(&absolute_path, &payload).await?; + + // Verify what actually landed on disk, not the in-memory buffer. + let written = fs::read(&absolute_path).await?; + let digest = ContentDigest::sha256_of(&written); + if written != payload { + return Err(KmsError::internal_error(format!( + "bundle artifact '{artifact_path}' read back differently than written" + ))); + } + + Ok(ArtifactDescriptor { + kind, + path: artifact_path.to_string(), + len: payload.len() as u64, + aead_algorithm: AeadAlgorithm::Aes256Gcm, + encrypted_digest: digest, + }) +} + +/// AAD binding an artifact to its bundle identity and path. A JSON tuple +/// gives unambiguous field boundaries without a hand-rolled framing format. +fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec { + serde_json::to_vec(&(BUNDLE_AAD_CONTEXT, backup_id, snapshot_generation, artifact_path)) + .expect("AAD tuple of strings and integers always serializes") +} + +/// The bundle-level protection label is the weakest state observed across +/// records: any plaintext-dev-only record marks the whole bundle, then any +/// legacy-unspecified marker (unknown until read), and only a uniformly +/// encrypted directory is labeled encrypted-master-key. +fn weakest_observed_protection(records: &[CollectedRecord]) -> AtRestProtection { + let mut has_legacy = false; + for record in records { + match record.protection { + StoredKeyProtection::PlaintextDevOnly => return AtRestProtection::PlaintextDevOnly, + StoredKeyProtection::LegacyUnspecified => has_legacy = true, + StoredKeyProtection::EncryptedMasterKey => {} + } + } + if has_legacy { + AtRestProtection::LegacyUnspecified + } else { + AtRestProtection::EncryptedMasterKey + } +} + +fn local_kdf_descriptor(snapshot: &CollectedSnapshot) -> LocalKdfDescriptor { + let mut modes = Vec::new(); + for (marker, mode) in [ + (StoredKeyProtection::EncryptedMasterKey, AtRestProtection::EncryptedMasterKey), + (StoredKeyProtection::PlaintextDevOnly, AtRestProtection::PlaintextDevOnly), + (StoredKeyProtection::LegacyUnspecified, AtRestProtection::LegacyUnspecified), + ] { + if snapshot.records.iter().any(|record| record.protection == marker) { + modes.push(mode); + } + } + + // With a salt on disk the backend derives via Argon2id; without one only + // the pre-beta.9 SHA-256 derivation can apply. For plaintext-only + // directories the derivation is informational. + let derivation = if snapshot.salt.is_some() { + LocalKeyDerivation::current_argon2id() + } else { + LocalKeyDerivation::LegacySha256 + }; + + LocalKdfDescriptor { + derivation, + protection_modes: modes, + // The verifier shape is left to the restore change; the schema keeps + // it optional so bundles without one stay valid. + master_key_verifier: None, + } +} + +async fn prepare_destination(destination: &Path) -> Result<()> { + if fs::try_exists(destination).await? { + let mut entries = fs::read_dir(destination) + .await + .map_err(|error| KmsError::invalid_operation(format!("backup destination is not a readable directory: {error}")))?; + if entries.next_entry().await?.is_some() { + return Err(KmsError::invalid_operation( + "backup destination directory is not empty; refusing to mix bundles", + )); + } + } + fs::create_dir_all(destination.join(KEYS_DIR)).await?; + Ok(()) +} + +async fn write_new_file(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await + .map_err(|error| KmsError::io_error(format!("failed to create bundle file {}: {error}", path.display())))?; + file.write_all(bytes).await?; + file.sync_all().await?; + Ok(()) +} + +/// Fsync a directory so freshly created bundle entries survive power loss. +/// No-op on non-Unix platforms where directories cannot be opened for +/// syncing (mirrors the local backend's durable commit helper). +async fn fsync_dir(path: &Path) -> Result<()> { + #[cfg(unix)] + { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || std::fs::File::open(&path)?.sync_all()) + .await + .map_err(|error| KmsError::io_error(error.to_string()))??; + } + #[cfg(not(unix))] + let _ = path; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backends::KmsClient; + use crate::config::LocalConfig; + use std::sync::Arc; + use tempfile::TempDir; + + async fn encrypted_client() -> (LocalKmsClient, TempDir) { + let temp = TempDir::new().expect("temp dir"); + let client = LocalKmsClient::new(LocalConfig { + key_dir: temp.path().to_path_buf(), + master_key: Some("test-master-key".to_string()), + file_permissions: Some(0o600), + }) + .await + .expect("client should initialize"); + (client, temp) + } + + async fn dev_client() -> (LocalKmsClient, TempDir) { + let temp = TempDir::new().expect("temp dir"); + let client = LocalKmsClient::new(LocalConfig { + key_dir: temp.path().to_path_buf(), + master_key: None, + file_permissions: Some(0o600), + }) + .await + .expect("client should initialize"); + (client, temp) + } + + fn test_kek() -> BackupKek { + BackupKek::new("backup-kek-test", 1, [0x42; 32]).expect("kek") + } + + fn export_request(destination: PathBuf) -> LocalBackupExportRequest { + LocalBackupExportRequest { + backup_id: "backup-0001".to_string(), + deployment_identity: "deployment-test".to_string(), + rustfs_version: "1.0.0-test".to_string(), + snapshot_generation: 7, + destination, + } + } + + fn walk_files(dir: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).expect("read dir") { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + walk_files(&path, out); + } else { + out.push(path); + } + } + } + + fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|window| window == needle) + } + + #[tokio::test] + async fn export_round_trips_and_decrypts_to_source_records() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("alpha", "AES_256", None).await.expect("create alpha"); + client.create_key("beta", "AES_256", None).await.expect("create beta"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + + assert_eq!(manifest.backend, BackupBackendKind::Local); + assert_eq!(manifest.responsibility, BackupResponsibility::FullMaterial); + assert_eq!(manifest.at_rest_protection, AtRestProtection::EncryptedMasterKey); + assert_eq!(manifest.snapshot_generation, 7); + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.derivation, LocalKeyDerivation::current_argon2id()); + assert_eq!(kdf.protection_modes, vec![AtRestProtection::EncryptedMasterKey]); + + // alpha, beta (sorted), then the salt artifact. + assert_eq!(manifest.artifacts.len(), 3); + assert_eq!(manifest.artifacts[0].path, "artifacts/keys/alpha.key.enc"); + assert_eq!(manifest.artifacts[1].path, "artifacts/keys/beta.key.enc"); + assert_eq!(manifest.artifacts[2].kind, ArtifactKind::MasterKeySalt); + + let reread = read_local_bundle_manifest(&destination) + .await + .expect("manifest should decode"); + assert_eq!(reread, manifest); + + for (artifact, key_id) in [(&manifest.artifacts[0], "alpha"), (&manifest.artifacts[1], "beta")] { + let decrypted = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect("artifact should decrypt"); + let source = fs::read(client.key_directory().join(format!("{key_id}.key"))) + .await + .expect("source record"); + assert_eq!(decrypted.as_slice(), source.as_slice(), "record {key_id} must round-trip verbatim"); + } + + let salt = decrypt_bundle_artifact(&destination, &manifest, &manifest.artifacts[2], &kek) + .await + .expect("salt should decrypt"); + let source_salt = fs::read(client.master_key_salt_file()).await.expect("source salt"); + assert_eq!(salt.as_slice(), source_salt.as_slice()); + } + + #[tokio::test] + async fn plaintext_dev_only_material_is_rewrapped_and_absent_from_bundle() { + let (client, _key_dir) = dev_client().await; + client.create_key("dev-key", "AES_256", None).await.expect("create key"); + + let material = client + .decrypt_key_material_for_export("dev-key") + .await + .expect("material should be readable"); + let source_record = fs::read(client.key_directory().join("dev-key.key")).await.expect("record"); + let record_json: serde_json::Value = serde_json::from_slice(&source_record).expect("record parses"); + let material_base64 = record_json + .get("encrypted_key_material") + .and_then(|value| value.as_str()) + .expect("material field") + .to_string(); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + + assert_eq!(manifest.at_rest_protection, AtRestProtection::PlaintextDevOnly); + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.protection_modes, vec![AtRestProtection::PlaintextDevOnly]); + assert_eq!(kdf.derivation, LocalKeyDerivation::LegacySha256); + assert!( + !manifest.artifacts.iter().any(|a| a.kind == ArtifactKind::MasterKeySalt), + "dev-mode directory has no salt to bundle" + ); + + // Byte-level: neither the raw material nor its base64 form may appear + // anywhere in the bundle. The mandatory KEK re-wrap is what hides it. + let mut files = Vec::new(); + walk_files(&destination, &mut files); + assert!(!files.is_empty()); + for file in files { + let bytes = std::fs::read(&file).expect("bundle file"); + assert!( + !contains_subslice(&bytes, material.as_ref()), + "raw key material leaked into {}", + file.display() + ); + assert!( + !contains_subslice(&bytes, material_base64.as_bytes()), + "base64 key material leaked into {}", + file.display() + ); + } + + // The wrapped record still round-trips for restore. + let decrypted = decrypt_bundle_artifact(&destination, &manifest, &manifest.artifacts[0], &kek) + .await + .expect("artifact should decrypt"); + assert_eq!(decrypted.as_slice(), source_record.as_slice()); + } + + #[tokio::test] + async fn export_fence_blocks_writers_until_released() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("existing", "AES_256", None).await.expect("create key"); + let client = Arc::new(client); + + let fence = client.acquire_export_fence().await; + + let writer = { + let client = Arc::clone(&client); + tokio::spawn(async move { + client.create_key("new-key", "AES_256", None).await.expect("create"); + client.disable_key("existing", None).await.expect("disable"); + }) + }; + + for _ in 0..64 { + tokio::task::yield_now().await; + } + assert!(!writer.is_finished(), "writers must stay blocked while the export fence is held"); + + drop(fence); + writer.await.expect("writer should finish after fence release"); + assert!( + fs::try_exists(client.key_directory().join("new-key.key")) + .await + .expect("exists") + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_writers_yield_complete_records() { + let (client, _key_dir) = encrypted_client().await; + for index in 0..5 { + client + .create_key(&format!("seed-{index}"), "AES_256", None) + .await + .expect("seed key"); + } + let client = Arc::new(client); + + let writer = { + let client = Arc::clone(&client); + tokio::spawn(async move { + for index in 0..30 { + client + .create_key(&format!("concurrent-{index}"), "AES_256", None) + .await + .expect("create"); + let target = format!("seed-{}", index % 5); + if index % 2 == 0 { + client.disable_key(&target, None).await.expect("disable"); + } else { + client.enable_key(&target, None).await.expect("enable"); + } + } + }) + }; + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed under concurrent writers"); + writer.await.expect("writer task"); + + // Whatever subset of writers landed before the fence, every record in + // the bundle must be complete: parseable, self-identifying, and with + // non-empty material. No torn records, no half-updates. + let reread = read_local_bundle_manifest(&destination).await.expect("manifest decodes"); + assert_eq!(reread, manifest); + for artifact in manifest.artifacts.iter().filter(|a| a.kind == ArtifactKind::KeyMaterial) { + let record = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect("record decrypts"); + let value: serde_json::Value = serde_json::from_slice(&record).expect("record is complete JSON"); + let key_id = value.get("key_id").and_then(|v| v.as_str()).expect("key_id present"); + assert_eq!(artifact.path, format!("artifacts/keys/{key_id}.key.enc")); + let material = value + .get("encrypted_key_material") + .and_then(|v| v.as_str()) + .expect("material present"); + assert!(!material.is_empty()); + } + } + + #[tokio::test] + async fn tampered_and_truncated_bundles_fail_closed() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("victim", "AES_256", None).await.expect("create key"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + let artifact = &manifest.artifacts[0]; + let artifact_file = destination.join(&artifact.path); + let original_artifact = std::fs::read(&artifact_file).expect("artifact bytes"); + + // Tampered artifact byte: digest verification rejects it. + let mut tampered = original_artifact.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0x01; + std::fs::write(&artifact_file, &tampered).expect("write tampered"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect_err("tampered artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + + // Truncated artifact: typed truncation error. + std::fs::write(&artifact_file, &original_artifact[..original_artifact.len() - 4]).expect("truncate"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &kek) + .await + .expect_err("truncated artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); + std::fs::write(&artifact_file, &original_artifact).expect("restore artifact"); + + // Tampered manifest (generation flip): sealed digest mismatch. + let manifest_file = destination.join(LOCAL_BUNDLE_MANIFEST_FILE); + let original_manifest = std::fs::read(&manifest_file).expect("manifest bytes"); + let tampered_manifest = String::from_utf8(original_manifest.clone()) + .expect("manifest is utf-8") + .replace("\"snapshot_generation\":7", "\"snapshot_generation\":8"); + assert_ne!(tampered_manifest.as_bytes(), original_manifest.as_slice(), "tamper must apply"); + std::fs::write(&manifest_file, tampered_manifest).expect("write tampered manifest"); + let error = read_local_bundle_manifest(&destination) + .await + .expect_err("tampered manifest must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + + // Truncated manifest. + std::fs::write(&manifest_file, &original_manifest[..original_manifest.len() / 2]).expect("truncate manifest"); + let error = read_local_bundle_manifest(&destination) + .await + .expect_err("truncated manifest must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); + + // Missing manifest: the bundle never sealed. + std::fs::remove_file(&manifest_file).expect("remove manifest"); + let error = read_local_bundle_manifest(&destination) + .await + .expect_err("bundle without manifest must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::IncompleteBundle { .. })), "got {error:?}"); + } + + #[tokio::test] + async fn wrong_kek_is_rejected_before_decryption() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("victim", "AES_256", None).await.expect("create key"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + let artifact = &manifest.artifacts[0]; + + let wrong_id = BackupKek::new("other-kek", 1, [0x42; 32]).expect("kek"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &wrong_id) + .await + .expect_err("mismatched KEK id must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); + + let wrong_version = BackupKek::new("backup-kek-test", 2, [0x42; 32]).expect("kek"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &wrong_version) + .await + .expect_err("mismatched KEK version must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); + + // Right identity, wrong material: AEAD authentication fails closed. + let wrong_material = BackupKek::new("backup-kek-test", 1, [0x24; 32]).expect("kek"); + let error = decrypt_bundle_artifact(&destination, &manifest, artifact, &wrong_material) + .await + .expect_err("wrong KEK material must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + } + + #[tokio::test] + async fn missing_salt_with_encrypted_records_fails_export() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("victim", "AES_256", None).await.expect("create key"); + fs::remove_file(client.master_key_salt_file()).await.expect("remove salt"); + + let bundle = TempDir::new().expect("bundle dir"); + let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect_err("export without salt must fail"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("salt"), "got {error}"); + } + + #[tokio::test] + async fn refuses_empty_key_dir_and_nonempty_destination() { + let (client, _key_dir) = dev_client().await; + let bundle = TempDir::new().expect("bundle dir"); + let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect_err("empty key dir must not produce a bundle"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + client.create_key("dev-key", "AES_256", None).await.expect("create key"); + let occupied = bundle.path().join("occupied"); + std::fs::create_dir_all(&occupied).expect("mkdir"); + std::fs::write(occupied.join("stale"), b"leftover").expect("occupy"); + let error = export_local_backup(&client, &test_kek(), &export_request(occupied)) + .await + .expect_err("non-empty destination must be refused"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + } + + #[tokio::test] + async fn legacy_records_export_verbatim_with_weakest_protection_label() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("modern", "AES_256", None).await.expect("create key"); + client.create_key("legacy-key", "AES_256", None).await.expect("create key"); + + // Strip the protection marker to fabricate a pre-beta.9 record, the + // same way the local backend's own legacy-compat tests do. + let legacy_path = client.key_directory().join("legacy-key.key"); + let mut record: serde_json::Value = + serde_json::from_slice(&fs::read(&legacy_path).await.expect("record")).expect("record parses"); + record + .as_object_mut() + .expect("record is an object") + .remove("at_rest_protection"); + let legacy_bytes = serde_json::to_vec_pretty(&record).expect("record serializes"); + fs::write(&legacy_path, &legacy_bytes).await.expect("write legacy record"); + + let bundle = TempDir::new().expect("bundle dir"); + let destination = bundle.path().join("bundle"); + let kek = test_kek(); + let manifest = export_local_backup(&client, &kek, &export_request(destination.clone())) + .await + .expect("export should succeed"); + + assert_eq!(manifest.at_rest_protection, AtRestProtection::LegacyUnspecified); + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!( + kdf.protection_modes, + vec![AtRestProtection::EncryptedMasterKey, AtRestProtection::LegacyUnspecified] + ); + + let legacy_artifact = manifest + .artifacts + .iter() + .find(|a| a.path == "artifacts/keys/legacy-key.key.enc") + .expect("legacy artifact"); + let decrypted = decrypt_bundle_artifact(&destination, &manifest, legacy_artifact, &kek) + .await + .expect("legacy artifact decrypts"); + assert_eq!(decrypted.as_slice(), legacy_bytes.as_slice(), "legacy record must travel verbatim"); + } + + #[tokio::test] + async fn record_identity_mismatch_aborts_export() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("good", "AES_256", None).await.expect("create key"); + std::fs::copy(client.key_directory().join("good.key"), client.key_directory().join("evil.key")) + .expect("plant mismatched record"); + + let bundle = TempDir::new().expect("bundle dir"); + let error = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect_err("identity mismatch must abort the export"); + assert!(matches!(error, KmsError::InvalidKey { .. }), "got {error:?}"); + } +} diff --git a/crates/kms/src/backup/manifest.rs b/crates/kms/src/backup/manifest.rs index 052642428..3eff4fad0 100644 --- a/crates/kms/src/backup/manifest.rs +++ b/crates/kms/src/backup/manifest.rs @@ -355,8 +355,10 @@ struct ManifestProbe { /// /// The manifest is the authoritative description of one backup bundle: what /// was captured, under which snapshot generation, protected by which backup -/// KEK, and which restore responsibility applies. Field order is part of the -/// canonical digest form and is frozen for this format version. +/// KEK, and which restore responsibility applies. The digest's canonical +/// form is the manifest's JSON value with the digest hex emptied (see +/// [`Self::compute_digest`]); decoders verify it against the raw stored +/// bytes and never re-serialize parsed fields. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BackupManifest { @@ -415,8 +417,14 @@ impl BackupManifest { /// /// Fail-closed order: truncated or malformed input, then unknown format /// version, then a missing completeness marker, then schema decoding - /// (unknown fields, duplicate fields, missing fields), then semantic - /// validation including digest verification. + /// (unknown fields, duplicate fields, missing fields), then digest + /// verification against the raw input bytes, then semantic validation. + /// + /// The digest is verified against the bytes as stored — parsed typed + /// fields are never re-serialized for verification, so a field whose + /// string form does not round-trip byte-identically through its parsed + /// representation (timestamps in environment-dependent time zone + /// spellings, for example) cannot produce a spurious mismatch. pub fn decode(bytes: &[u8]) -> Result { let probe: ManifestProbe = serde_json::from_slice(bytes).map_err(map_serde_error)?; if probe.format_version != Self::FORMAT_VERSION { @@ -429,7 +437,9 @@ impl BackupManifest { return Err(BackupError::incomplete_bundle("manifest has no completeness marker")); } let manifest: Self = serde_json::from_slice(bytes).map_err(map_serde_error)?; - manifest.validate()?; + manifest.validate_pre_digest()?; + Self::verify_digest_in_bytes(bytes, &manifest.manifest_digest)?; + manifest.validate_content()?; Ok(manifest) } @@ -449,23 +459,30 @@ impl BackupManifest { Ok(self) } - /// Compute the digest over the canonical manifest bytes. + /// Compute the digest over the canonical manifest form. /// - /// Canonical form: compact JSON serialization of this manifest with the - /// digest hex emptied. Field order is struct declaration order and is - /// frozen for format version 1, so the same manifest content always - /// hashes to the same value. + /// Canonical form: the JSON *value* of the manifest with the digest hex + /// emptied, object keys rebuilt in bytewise-sorted order at every level + /// (see [`canonicalize_value`]), then serialized compactly. The value + /// layer is what makes sealing and decoding agree byte-for-byte — a + /// decoder recovers the identical value from the raw stored bytes + /// without round-tripping any typed field through parse-and-reprint — + /// and the explicit key sort makes the bytes independent of + /// `serde_json`'s map implementation (`preserve_order` on or off). pub fn compute_digest(&self) -> Result { let mut unsealed = self.clone(); unsealed.manifest_digest = ContentDigest::placeholder(self.manifest_digest.algorithm); - let canonical = serde_json::to_vec(&unsealed) + let value = serde_json::to_value(&unsealed) .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?; - match self.manifest_digest.algorithm { - DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)), - } + Self::digest_of_canonical_value(value, self.manifest_digest.algorithm) } - /// Verify the sealed digest against the current manifest content. + /// Verify the sealed digest against the current in-memory content. + /// + /// This is the producer-side check (sealing and [`Self::encode`]). + /// Decoders must use the raw stored bytes instead (see [`Self::decode`]): + /// re-serializing parsed fields is not guaranteed to reproduce the + /// stored spelling byte-for-byte. pub fn verify_digest(&self) -> Result<(), BackupError> { if !self.manifest_digest.is_well_formed() { return Err(BackupError::corrupted("manifest digest is not a well-formed digest value")); @@ -478,6 +495,33 @@ impl BackupManifest { Ok(()) } + /// Verify a declared digest against raw manifest bytes, normalizing only + /// through the JSON value layer and emptying the digest slot in place. + fn verify_digest_in_bytes(bytes: &[u8], declared: &ContentDigest) -> Result<(), BackupError> { + if !declared.is_well_formed() { + return Err(BackupError::corrupted("manifest digest is not a well-formed digest value")); + } + let mut value: serde_json::Value = serde_json::from_slice(bytes).map_err(map_serde_error)?; + let Some(slot) = value.get_mut("manifest_digest").and_then(|digest| digest.get_mut("hex")) else { + return Err(BackupError::corrupted("manifest has no digest slot")); + }; + *slot = serde_json::Value::String(String::new()); + if Self::digest_of_canonical_value(value, declared.algorithm)? != *declared { + return Err(BackupError::corrupted( + "manifest digest mismatch: content does not match the sealed digest", + )); + } + Ok(()) + } + + fn digest_of_canonical_value(value: serde_json::Value, algorithm: DigestAlgorithm) -> Result { + let canonical = serde_json::to_vec(&canonicalize_value(value)) + .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?; + match algorithm { + DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)), + } + } + /// Look up a required artifact by kind, failing closed when absent. pub fn require_artifact(&self, kind: ArtifactKind) -> Result<&ArtifactDescriptor, BackupError> { self.artifacts @@ -486,11 +530,21 @@ impl BackupManifest { .ok_or_else(|| BackupError::missing_artifact(artifact_kind_name(kind))) } - /// Validate the full manifest contract. + /// Validate the full manifest contract against the in-memory content. /// - /// This is decode-side validation and also guards [`Self::encode`], so a - /// producer cannot publish a manifest a decoder would reject. + /// This guards [`Self::encode`], so a producer cannot publish a manifest + /// a decoder would reject. [`Self::decode`] runs the same checks but + /// verifies the digest against the raw input bytes instead. pub fn validate(&self) -> Result<(), BackupError> { + self.validate_pre_digest()?; + self.verify_digest()?; + self.validate_content() + } + + /// Checks that must run before any digest verification: an unknown + /// version or an unsealed bundle is reported as its own typed error, not + /// as a digest mismatch. + fn validate_pre_digest(&self) -> Result<(), BackupError> { if self.format_version != Self::FORMAT_VERSION { return Err(BackupError::UnknownVersion { found: self.format_version, @@ -500,7 +554,12 @@ impl BackupManifest { if self.completeness != CompletenessState::Complete { return Err(BackupError::incomplete_bundle("completeness marker records an in-progress bundle")); } - self.verify_digest()?; + Ok(()) + } + + /// Semantic validation of everything except version, completeness, and + /// digest integrity. + fn validate_content(&self) -> Result<(), BackupError> { require_non_empty("backup_id", &self.backup_id)?; require_non_empty("rustfs_version", &self.rustfs_version)?; require_non_empty("deployment_identity", &self.deployment_identity)?; @@ -645,6 +704,29 @@ fn map_serde_error(error: serde_json::Error) -> BackupError { } } +/// Rebuild a JSON value with object keys in bytewise-sorted order at every +/// nesting level (array element order is preserved). +/// +/// `serde_json`'s map keeps keys sorted by default but preserves insertion +/// order when the `preserve_order` feature is unified into the build by any +/// other crate. Digest bytes must not depend on that, so the ordering is +/// imposed explicitly here instead of being inherited from the map type. +fn canonicalize_value(value: serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut entries: Vec<(String, serde_json::Value)> = map.into_iter().collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + let mut sorted = serde_json::Map::with_capacity(entries.len()); + for (key, entry) in entries { + sorted.insert(key, canonicalize_value(entry)); + } + serde_json::Value::Object(sorted) + } + serde_json::Value::Array(items) => serde_json::Value::Array(items.into_iter().map(canonicalize_value).collect()), + other => other, + } +} + #[cfg(test)] mod tests { use super::*; @@ -736,7 +818,7 @@ mod tests { /// canonical form and frozen. If serialization layout or field order /// changes, this value changes and the fixture test fails — which is the /// point: that is a format-version bump, not a patch. - const FIXTURE_DIGEST_HEX: &str = "01accb3e2bc51e12d17d1efc52ad6ab9441c50bec52c3fb29e0a9f92b725cdaa"; + const FIXTURE_DIGEST_HEX: &str = "a8b104d61c358cd75cc9a7691d2f7d2d96f73c53653bef8940a49e96dbdf7775"; fn fixture() -> String { FIXTURE.replace("SEALED_DIGEST_HEX", FIXTURE_DIGEST_HEX) @@ -1010,12 +1092,39 @@ mod tests { let with_discovery = fixture().replace("\"completeness\"", "\"capability_discovery\": [1], \"completeness\""); expect_corrupted(BackupManifest::decode(with_discovery.as_bytes()), "reserved"); - // Explicit null carries no data and is tolerated as absence; digest - // verification still passes because null slots are skipped on - // serialization. + // An explicit null slot decodes as absence at the schema layer, but + // sealed bundles never contain the key (`skip_serializing_if`), so + // inserting one after sealing is a byte-level modification and the + // raw-bytes digest check rejects it. let with_null = fixture().replace("\"completeness\"", "\"key_versions\": null, \"completeness\""); - let decoded = BackupManifest::decode(with_null.as_bytes()).expect("null reserved slot should decode"); - assert_eq!(decoded.key_versions, None); + expect_corrupted(BackupManifest::decode(with_null.as_bytes()), "digest mismatch"); + } + + #[test] + fn digest_verification_survives_non_round_tripping_timestamp_spellings() { + // A legacy `created_at` spelling (no time zone annotation) parses via + // the compat fallback and re-serializes differently ("+00:00[UTC]"), + // and host-dependent zone spellings can do the same. Digest + // verification therefore operates on the raw stored bytes and must + // never re-serialize parsed fields. + let sealed = seal(local_manifest_unsealed()); + let mut value = serde_json::to_value(&sealed).expect("manifest should convert to a value"); + value["created_at"] = serde_json::Value::String("2026-07-30T00:00:00+00:00".to_string()); + value["manifest_digest"]["hex"] = serde_json::Value::String(String::new()); + let digest = BackupManifest::digest_of_canonical_value(value.clone(), DigestAlgorithm::Sha256) + .expect("canonical digest should compute"); + value["manifest_digest"]["hex"] = serde_json::Value::String(digest.hex); + let bytes = serde_json::to_vec(&value).expect("manifest bytes"); + + let decoded = BackupManifest::decode(&bytes).expect("a non-round-tripping timestamp spelling must not break decoding"); + + // Precondition: the spelling really does not survive a typed + // round-trip — otherwise this test is vacuous. + let reserialized = serde_json::to_value(&decoded).expect("decoded manifest should convert to a value"); + assert_ne!(reserialized["created_at"], value["created_at"]); + // Which is exactly why the producer-side (in-memory) digest check + // cannot be used on decoded manifests. + assert!(decoded.verify_digest().is_err()); } #[test] diff --git a/crates/kms/src/backup/mod.rs b/crates/kms/src/backup/mod.rs index 59cdba350..d3f9134e9 100644 --- a/crates/kms/src/backup/mod.rs +++ b/crates/kms/src/backup/mod.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Backup/restore contract types for KMS state. +//! Backup/restore contracts and backup production for KMS state. //! -//! This module is contract-only: it defines the versioned backup manifest, -//! the per-backend responsibility matrix, typed failure modes, and the -//! restore dry-run report. Nothing here is wired into handlers or backends; -//! backup export, restore orchestration, and the admin API build on these -//! types in follow-up changes. +//! The contract side defines the versioned backup manifest, the per-backend +//! responsibility matrix, typed failure modes, and the restore dry-run +//! report. [`local_export`] implements the producer side for the Local +//! backend as a crate-internal API; restore orchestration and the admin API +//! build on these pieces in follow-up changes. //! //! # Bundle model //! @@ -50,6 +50,7 @@ mod capability; mod dry_run; mod error; +pub mod local_export; mod manifest; pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; @@ -57,6 +58,10 @@ pub use dry_run::{ ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, }; pub use error::BackupError; +pub use local_export::{ + BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup, + read_local_bundle_manifest, +}; pub use manifest::{ AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, From 04c59218506e012824a0a7720528f379f59cb332 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 1 Aug 2026 01:52:00 +0800 Subject: [PATCH 6/8] fix(s3): allow CopyObject COPY request metadata (#5514) * fix(s3): allow CopyObject COPY request metadata Co-Authored-By: heihutu * fix(kms): remove stale KmsClient import Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .../e2e_test/src/copy_object_metadata_test.rs | 33 ++++++++++--------- crates/kms/src/backup/local_export.rs | 1 - rustfs/src/app/object_usecase.rs | 13 -------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/crates/e2e_test/src/copy_object_metadata_test.rs b/crates/e2e_test/src/copy_object_metadata_test.rs index cfd257f06..f56fe7f41 100644 --- a/crates/e2e_test/src/copy_object_metadata_test.rs +++ b/crates/e2e_test/src/copy_object_metadata_test.rs @@ -117,9 +117,14 @@ mod tests { .key("assets/explicit-copy.js") .copy_source(format!("{bucket}/{key}")) .metadata_directive(MetadataDirective::Copy) + .customize() + .mutate_request(|request| { + request.headers_mut().insert("content-type", "application/octet-stream"); + request.headers_mut().insert("x-amz-meta-request-only", "ignored"); + }) .send() .await - .expect("explicit COPY directive failed"); + .expect("explicit COPY directive with request metadata failed"); let explicit_copy_head = client .head_object() .bucket(bucket) @@ -128,6 +133,18 @@ mod tests { .await .expect("HEAD failed after explicit COPY"); assert_eq!(explicit_copy_head.cache_control(), Some("max-age=60")); + assert_eq!(explicit_copy_head.content_type(), Some("text/javascript; charset=utf-8")); + assert_eq!( + explicit_copy_head.metadata().and_then(|metadata| metadata.get("mtime")), + Some(&"1777992333".to_string()) + ); + assert_eq!( + explicit_copy_head + .metadata() + .and_then(|metadata| metadata.get("request-only")), + None, + "COPY must ignore request metadata" + ); assert_eq!( explicit_copy_head.website_redirect_location(), None, @@ -571,20 +588,6 @@ mod tests { Some("InvalidArgument") ); - let ignored_replacement = client - .copy_object() - .bucket(bucket) - .key(key) - .copy_source(format!("{bucket}/{key}")) - .content_type("application/ignored") - .send() - .await - .expect_err("Replacement fields without REPLACE should be rejected"); - assert_eq!( - ignored_replacement.as_service_error().and_then(|error| error.code()), - Some("InvalidRequest") - ); - let unchanged = client .get_object() .bucket(bucket) diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs index d3905b31f..00bb905e8 100644 --- a/crates/kms/src/backup/local_export.rs +++ b/crates/kms/src/backup/local_export.rs @@ -560,7 +560,6 @@ async fn fsync_dir(path: &Path) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::backends::KmsClient; use crate::config::LocalConfig; use std::sync::Arc; use tempfile::TempDir; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 8d46feadd..806b07ff9 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -6072,19 +6072,6 @@ impl DefaultObjectUsecase { )); } }; - let has_replacement_metadata = metadata.is_some() - || cache_control.is_some() - || content_disposition.is_some() - || content_encoding.is_some() - || content_language.is_some() - || content_type.is_some() - || expires.is_some(); - if has_replacement_metadata && !replaces_metadata { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Replacement metadata requires the REPLACE metadata directive".to_string(), - )); - } let replacement_metadata = if replaces_metadata { validate_archive_content_encoding(&key, content_type.as_deref(), content_encoding.as_deref())?; let mut replacement_metadata = metadata.unwrap_or_default(); From 74019845c44f04c2e0c44df672060f8a6ad45d07 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 04:24:04 +0800 Subject: [PATCH 7/8] fix(kms): drop stale KmsClient trait import in local_export tests (#5519) The KmsClient trait was folded into KmsBackend (#5501), but the backup export tests merged afterwards (#5499) still imported it, breaking the crate's test build; create_key is an inherent LocalKmsClient method, so the import is simply unused. From 792f2ef2047bc8e147aaec2225a1b8b0aef0f791 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 05:28:10 +0800 Subject: [PATCH 8/8] docs(kms): add observability dashboard, alert rules and runbook (#5517) Add a Grafana dashboard for the four KMS backend operation metrics emitted at the policy choke point, Prometheus alert rules with conservative default thresholds (pending staging baseline calibration), and an operations runbook documenting the metric contract and the response procedure for each alert. Register the new alert rules file in the observability READMEs. Refs rustfs/backlog#1584 (part of rustfs/backlog#1562) --- .docker/observability/README.md | 10 + .docker/observability/README_ZH.md | 10 + .../prometheus-rules/rustfs-kms-alerts.yml | 188 +++++ deploy/observability/README.md | 2 + .../grafana/rustfs-kms-observability.json | 793 ++++++++++++++++++ docs/operations/kms-observability-runbook.md | 130 +++ 6 files changed, 1133 insertions(+) create mode 100644 .docker/observability/prometheus-rules/rustfs-kms-alerts.yml create mode 100644 deploy/observability/grafana/rustfs-kms-observability.json create mode 100644 docs/operations/kms-observability-runbook.md diff --git a/.docker/observability/README.md b/.docker/observability/README.md index ff5e0bbdb..cb4b9a881 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -60,6 +60,16 @@ The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-con | `CodecStreamingFallbackSpike` | Warning | Codec streaming fallback > 10x baseline for 10m | | `IoQueueSaturation` | Warning | IO queue utilization > 90% for 5m | +The file `prometheus-rules/rustfs-kms-alerts.yml` contains alerting rules for the KMS backend operation metrics. Thresholds are conservative defaults pending staging baseline calibration; response procedures live in `docs/operations/kms-observability-runbook.md`, and the matching dashboard is `deploy/observability/grafana/rustfs-kms-observability.json`. + +| Alert | Severity | Condition | +|-------|----------|-----------| +| `KmsBackendFatalErrors` | Critical | Fatal (non-retryable) attempt failures > 0 for 5m | +| `KmsBackendHighErrorRate` | Critical | Non-success operation ratio > 5% for 10m (with traffic guard) | +| `KmsBackendP99LatencyHigh` | Warning | Operation p99 duration (incl. retries) > 2s for 10m | +| `KmsBackendAttemptFailureSpike` | Warning | Attempt failure rate > 0.5/s for 10m | +| `KmsBackendRetryBudgetExhausted` | Warning | budget_exhausted / deadline_exceeded outcomes > 0.05/s for 10m | + ### Enabling Alert Rules Add the alert rules file to your Prometheus configuration: diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md index 97253f20c..f42674f6c 100644 --- a/.docker/observability/README_ZH.md +++ b/.docker/observability/README_ZH.md @@ -60,6 +60,16 @@ | `CodecStreamingFallbackSpike` | 警告 | Codec streaming 回退 > 10x 基线，持续 10 分钟 | | `IoQueueSaturation` | 警告 | IO 队列利用率 > 90%，持续 5 分钟 | +文件 `prometheus-rules/rustfs-kms-alerts.yml` 包含 KMS 后端操作指标的告警规则。阈值为保守默认值，待 staging 基线校准；响应流程见 `docs/operations/kms-observability-runbook.md`，配套仪表盘为 `deploy/observability/grafana/rustfs-kms-observability.json`。 + +| 告警 | 级别 | 条件 | +|------|------|------| +| `KmsBackendFatalErrors` | 严重 | fatal（不可重试）尝试失败 > 0，持续 5 分钟 | +| `KmsBackendHighErrorRate` | 严重 | 非 success 操作占比 > 5%，持续 10 分钟（含流量下限保护） | +| `KmsBackendP99LatencyHigh` | 警告 | 操作 p99 耗时（含重试）> 2s，持续 10 分钟 | +| `KmsBackendAttemptFailureSpike` | 警告 | 尝试失败率 > 0.5/s，持续 10 分钟 | +| `KmsBackendRetryBudgetExhausted` | 警告 | budget_exhausted / deadline_exceeded 结果 > 0.05/s，持续 10 分钟 | + ### 启用告警规则 在 Prometheus 配置中添加告警规则文件： diff --git a/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml b/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml new file mode 100644 index 000000000..e4f1faf03 --- /dev/null +++ b/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml @@ -0,0 +1,188 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# RustFS KMS backend — Prometheus alerting rules +# ============================================================================= +# +# Metric source: the KMS operation-policy choke point in +# crates/kms/src/policy.rs. All label values are static enum strings +# (operation, op_class, outcome, error_class); key identifiers, key material, +# and tokens never appear in labels. +# +# Response procedures: docs/operations/kms-observability-runbook.md +# +# IMPORTANT — threshold status: every numeric threshold below is a +# conservative default chosen without a production baseline. Calibrate against +# a staging baseline before relying on these alerts for paging, and prefer +# loosening over tightening until the baseline exists. Formal SLO targets are +# deliberately not encoded here (see rustfs/backlog#1584). +# +# NOTE: prometheus.yml loads /etc/prometheus/rules/*.yml — keep the .yml +# extension or the file is silently ignored by the docker-compose stack. +# +# Validate: promtool check rules rustfs-kms-alerts.yml +# ============================================================================= + +groups: + # ========================================================================== + # Critical alerts — immediate action required + # ========================================================================== + - name: rustfs-kms-critical + interval: 30s + rules: + # ------------------------------------------------------------------ + # 1. KmsBackendFatalErrors + # Any attempt failure classified as fatal (non-retryable): auth + # or permission errors, malformed requests, missing keys. The + # policy never retries these, so even a low rate means real + # operations are failing right now. + # ------------------------------------------------------------------ + - alert: KmsBackendFatalErrors + expr: | + sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m])) > 0 + for: 5m + labels: + severity: critical + component: kms + annotations: + summary: "KMS backend fatal errors on operation {{ $labels.operation }}" + description: >- + Attempt failures classified as fatal are occurring at + {{ $value | printf "%.3f" }}/s on operation + {{ $labels.operation }}. Fatal failures are not retried: + each one is a KMS backend call that failed permanently + (authentication, permissions, malformed request, or a + missing key/version). + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendfatalerrors" + + # ------------------------------------------------------------------ + # 2. KmsBackendHighErrorRate + # Sustained share of operations terminating without success + # (fatal, budget_exhausted, deadline_exceeded). The cancelled + # outcome is excluded because shutdowns legitimately produce it. + # The traffic guard keeps a single failure on a near-idle + # cluster from firing the alert. + # Threshold: 5% for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendHighErrorRate + expr: | + ( + sum(rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m])) + / + clamp_min(sum(rate(rustfs_kms_backend_operations_total[5m])), 1e-9) + ) > 0.05 + and + sum(rate(rustfs_kms_backend_operations_total[5m])) > 0.02 + for: 10m + labels: + severity: critical + component: kms + annotations: + summary: "KMS backend non-success ratio above 5% for 10m" + description: >- + {{ $value | humanizePercentage }} of KMS backend operations + are terminating in fatal, budget_exhausted, or + deadline_exceeded. Object encryption and decryption paths + depending on the KMS are degraded or failing. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate" + + # ========================================================================== + # Warning alerts — investigation needed + # ========================================================================== + - name: rustfs-kms-warning + interval: 30s + rules: + # ------------------------------------------------------------------ + # 3. KmsBackendP99LatencyHigh + # p99 wall-clock duration of whole operations (attempts plus + # backoff) is sustained above 2 seconds. Because the histogram + # includes retries, a high p99 usually means the retry policy + # is absorbing backend failures, not that every call is slow. + # Threshold: 2s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendP99LatencyHigh + expr: | + histogram_quantile(0.99, + sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])) + ) > 2 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend operation p99 latency above 2s for 10m" + description: >- + The 99th-percentile KMS backend operation duration is + {{ $value | humanizeDuration }}, including retries and + backoff. Encryption and decryption latency is leaking into + S3 request latency. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendp99latencyhigh" + + # ------------------------------------------------------------------ + # 4. KmsBackendAttemptFailureSpike + # Aggregate attempt-failure rate (all error classes) sustained + # above an absolute floor. An absolute threshold is used instead + # of an offset-1d baseline ratio because fresh deployments have + # no baseline and an empty offset vector would keep a ratio + # alert from ever firing; switch to a baseline-relative form + # (see rustfs-get-optimization-alerts.yaml for the pattern) + # once a stable staging baseline exists. + # Threshold: 0.5/s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendAttemptFailureSpike + expr: | + sum(rate(rustfs_kms_backend_attempt_failures_total[5m])) > 0.5 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend attempt failures above 0.5/s for 10m" + description: >- + KMS backend attempts are failing at + {{ $value | printf "%.2f" }}/s across all error classes. + The retry policy may still be masking these from callers — + check the error-class breakdown before it stops absorbing + them. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendattemptfailurespike" + + # ------------------------------------------------------------------ + # 5. KmsBackendRetryBudgetExhausted + # Operations are running out of retry budget (budget_exhausted) + # or operation deadline (deadline_exceeded). These surface to + # callers as failed KMS operations even though every individual + # failure was retryable — the backend is unhealthy for longer + # than the policy can bridge. + # Threshold: 0.05/s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendRetryBudgetExhausted + expr: | + sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m])) > 0.05 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend operations exhausting retry budget ({{ $labels.outcome }})" + description: >- + KMS backend operations are terminating as + {{ $labels.outcome }} at {{ $value | printf "%.3f" }}/s. + Retryable failures are outlasting the retry budget, so + callers are seeing hard failures. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted" diff --git a/deploy/observability/README.md b/deploy/observability/README.md index d06deb7b3..9c19a9bd7 100644 --- a/deploy/observability/README.md +++ b/deploy/observability/README.md @@ -9,3 +9,5 @@ Import `grafana/rustfs-node-observability.json` into Grafana and select a Promet The dashboard uses the RustFS `server` metric label introduced with the node-local observability updates. `server` represents the RustFS node identity and is preferred for RustFS node comparisons. Prometheus `instance` still identifies the scrape target and remains useful for scrape/debugging views, but dashboards that compare RustFS nodes should group and filter by `server`. During a rolling upgrade, older nodes may still emit metrics without the `server` label. Complete the rollout before using this dashboard for node-by-node comparisons. + +`grafana/rustfs-kms-observability.json` covers the KMS backend operation metrics emitted at the operation-policy choke point. Unlike the node dashboard, KMS metrics do not carry the `server` label — use `job`/`instance` or promoted OTel resource attributes to split by node. Matching Prometheus alert rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`, and the alert response procedures are documented in `docs/operations/kms-observability-runbook.md`. diff --git a/deploy/observability/grafana/rustfs-kms-observability.json b/deploy/observability/grafana/rustfs-kms-observability.json new file mode 100644 index 000000000..0810a3e4a --- /dev/null +++ b/deploy/observability/grafana/rustfs-kms-observability.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "KMS backend operation metrics emitted at the operation-policy choke point (crates/kms/src/policy.rs). All label values are static enum strings; key identifiers, key material, and tokens never appear in labels. These metrics do not carry the RustFS `server` label — use your scrape topology (job/instance or promoted OTel resource attributes) to split by node. Alert response procedures: docs/operations/kms-observability-runbook.md.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Terminal outcomes of KMS backend operations. `fatal` means a non-retryable failure ended the operation on first observation; `budget_exhausted` and `deadline_exceeded` mean retries ran out; `cancelled` is normal during shutdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Backend Operation Rate by Outcome", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Operation names are static per-call-site identifiers (e.g. vault_kv2_read_key_version, vault_transit_encrypt, vault_login). `op_class` distinguishes read_idempotent, mutating_non_idempotent, and auth operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation, op_class) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{operation}} ({{op_class}})", + "range": true, + "refId": "A" + } + ], + "title": "Backend Operation Rate by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of operations that terminated in fatal, budget_exhausted, or deadline_exceeded. The cancelled outcome is plotted separately because shutdown windows legitimately spike it. The ratio is meaningless at near-zero traffic — read it together with the operation rate panels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome!~\"success|cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "non-success (excl. cancelled)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome=\"cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "cancelled", + "range": true, + "refId": "B" + } + ], + "title": "Non-Success Outcome Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-attempt failures by retry classification. retryable_conn covers connection-level failures, retryable_status covers retryable backend status codes, attempt_timeout covers attempts cut off by the per-attempt timeout, and fatal covers non-retryable failures (auth, permission, malformed request). Sustained fatal traffic is always actionable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{error_class}}", + "range": true, + "refId": "A" + } + ], + "title": "Attempt Failure Rate by Error Class", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Wall-clock duration of whole operations, including retries and backoff sleeps. A rising p99 with a flat p50 usually means a slow retry tail (backend degradation), not a uniform slowdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "refId": "B" + } + ], + "title": "Operation Duration p50 / p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "p99 duration split by operation. Auth operations (vault_login, vault_token_renew) and mutating writes are expected to sit higher than idempotent reads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operation Duration p99 by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Attempts one operation used before completing. Healthy operations average close to 1. A rising average or p99 means the retry policy is absorbing backend failures — read together with the attempt-failure panel to see the failure class.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(rustfs_kms_backend_operation_attempts_sum{operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum by (operation) (rate(rustfs_kms_backend_operation_attempts_count{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "{{operation}} avg", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_attempts_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p99 (all operations)", + "range": true, + "refId": "B" + } + ], + "title": "Operation Attempts Distribution", + "type": "timeseries" + }, + { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Placeholder for KMS metrics planned by sibling changes of rustfs/backlog#1584 that have **not landed yet**. Do not add panels for these names until the emitting code is merged, or the panels will render empty and mask real gaps.\n\n- TODO: key-cache effectiveness — hit/miss counters, entry gauge, eviction counter (replaces the former hardcoded-zero miss stat).\n- TODO: key lifecycle — pending-deletion/tombstone gauges from the deletion worker sweep, aggregate rotation-age gauge.\n- TODO: Vault credentials — token TTL remaining and fail-closed state gauges.\n- TODO: synthetic backend probe — probe outcome/failure-class metrics feeding the readiness cache.\n\nWhen a family lands, replace one bullet with a real panel and keep this list in sync with docs/operations/kms-observability-runbook.md (Coverage gaps).", + "mode": "markdown" + }, + "title": "Planned Panels (TODO — metrics not landed yet)", + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "rustfs", + "observability", + "kms" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_kms_backend_operations_total, operation)", + "hide": 0, + "includeAll": true, + "label": "KMS operation", + "multi": true, + "name": "operation", + "options": [], + "query": "label_values(rustfs_kms_backend_operations_total, operation)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RustFS KMS Observability", + "uid": "rustfs-kms-observability", + "version": 1, + "weekStart": "" +} diff --git a/docs/operations/kms-observability-runbook.md b/docs/operations/kms-observability-runbook.md new file mode 100644 index 000000000..310df6cbe --- /dev/null +++ b/docs/operations/kms-observability-runbook.md @@ -0,0 +1,130 @@ +# KMS observability runbook + +This runbook covers the KMS backend operation metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md). + +## Metric reference + +All four metrics are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through. Label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression. + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `rustfs_kms_backend_operations_total` | counter | `operation`, `op_class`, `outcome` | Operations executed under the operation policy, counted once per terminal outcome | +| `rustfs_kms_backend_attempt_failures_total` | counter | `operation`, `error_class` | Individual failed attempts, including attempts the retry policy later absorbed | +| `rustfs_kms_backend_operation_duration_seconds` | histogram | `operation`, `outcome` | Wall-clock duration of a whole operation, including retries and backoff sleeps | +| `rustfs_kms_backend_operation_attempts` | histogram | `operation`, `outcome` | Number of attempts one operation used before completing | + +Label values: + +- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `cancelled` (shutdown or caller cancellation). +- `op_class`: `read_idempotent` (safe to retry), `mutating_non_idempotent` (never replayed — a retryable failure terminates after a single attempt because the server may have processed the request), `auth` (login and token renewal). +- `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version). +- `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`. + +Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. These metrics do not carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`). + +Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of KMS series on a cluster using those backends is expected, not an outage. + +## Dashboard + +Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` (Prometheus data source) and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed there. + +The dashboard's "Planned Panels (TODO)" text panel lists metric families designed under rustfs/backlog#1584 but not yet landed (key-cache effectiveness, lifecycle gauges, token TTL, synthetic probe). Do not add panels or alerts for those names until the emitting code is merged; keep that panel and the [Coverage gaps](#coverage-gaps-and-planned-metrics) section below in sync as they land. + +## Alert rules + +The rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. The docker-compose Prometheus loads `/etc/prometheus/rules/*.yml`, so the `.yml` extension is load-bearing. Validate edits with `promtool check rules rustfs-kms-alerts.yml`. + +Every threshold in that file is a conservative default chosen without a production baseline; see [Threshold calibration](#threshold-calibration) before treating a firing alert as an SLO breach or a quiet one as health. + +## Alert response procedures + +### KmsBackendFatalErrors + +Meaning: attempts are failing with `error_class="fatal"` — failures the policy never retries. Each one is a KMS backend call that failed permanently (authentication, permissions, malformed request, or a missing key/version), so callers are seeing errors right now. This is the highest-signal KMS alert: fatal failures do not appear as background noise in a healthy system. + +Investigation: + +1. Break the rate down by operation: `sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m]))`. +2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs. +3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the minimal policy in [KMS backend security properties](kms-backend-security.md) (a policy that drifted or was re-scoped produces 403s that classify as fatal), and check the Vault audit log for the corresponding denied requests. +4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the rotation retention preconditions in [KMS backend security properties](kms-backend-security.md) and verify nobody destroyed version records under the key subtree. +5. Confirm blast radius with the outcome view: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="fatal"}[5m]))`. + +Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operation Rate by Outcome" dashboard panels; Vault server audit and server logs; S3-level 5xx on encrypted buckets. + +### KmsBackendHighErrorRate + +Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, or `deadline_exceeded`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page. + +Investigation: + +1. Break the failures down by outcome: `sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))`. +2. If `fatal` dominates, follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). +3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted) — the backend is unavailable or too slow for longer than the retry policy can bridge. +4. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured. + +Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts in this runbook. + +### KmsBackendP99LatencyHigh + +Meaning: the p99 wall-clock duration of KMS operations is sustained above 2s. The histogram includes retries and backoff sleeps, so a high p99 with a healthy p50 usually means a slow retry tail (a subset of calls failing and being retried), not a uniform slowdown. + +Investigation: + +1. Compare p50 and p99 on the "Operation Duration p50 / p99" panel. Flat p50 with elevated p99 points at retries; both elevated points at the backend or the network path being uniformly slow. +2. Split by operation with `histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])))` to see whether one backend call or all of them regressed. +3. Check the attempts histogram: an average meaningfully above 1 confirms the latency is retry-driven; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the failure classes. +4. If latency is not retry-driven, check the network path to Vault (TLS handshakes, DNS, proxies) and Vault's own telemetry (storage backend latency, load). +5. Remember that this latency sits inside S3 request latency for encrypted objects: sustained p99 near the operation deadline will start converting into `deadline_exceeded` outcomes. + +Related signals: the "Operation Duration p99 by Operation" and "Operation Attempts Distribution" panels; `KMS backend attempt failed with a retryable error; backing off before retry` warnings (fields: `operation`, `attempt`, `error_class`, `backoff`). + +### KmsBackendAttemptFailureSpike + +Meaning: individual attempts are failing at a sustained rate across all error classes. The retry policy may still be absorbing these — operations can keep succeeding while this alert fires — but the system is burning retry budget and running degraded, and a small further degradation will surface to callers. + +Investigation: + +1. Break the rate down by class: `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total[5m]))`. +2. `retryable_conn`: network-level failures — check connectivity, TLS, DNS, and whether Vault is down or restarting. +3. `retryable_status`: the backend answered with a retryable error — check Vault health and seal status (a sealed Vault returns 503, which lands here), and Vault-side rate limiting. +4. `attempt_timeout`: attempts are being cut off by the per-attempt timeout — either the backend is slow (correlate with [KmsBackendP99LatencyHigh](#kmsbackendp99latencyhigh)) or the configured attempt timeout is too tight for the deployment's network path. +5. `fatal`: follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). +6. Grep RustFS logs for `KMS backend attempt failed with a retryable error; backing off before retry` — the structured fields (`operation`, `attempt`, `error_class`, `backoff`) identify which call sites are cycling. + +Related signals: the "Attempt Failure Rate by Error Class" panel; the attempts histogram average rising above 1. + +### KmsBackendRetryBudgetExhausted + +Meaning: operations are terminating as `budget_exhausted` or `deadline_exceeded` — every individual failure was retryable, but the backend stayed unhealthy for longer than the retry policy could bridge, so callers received hard failures. + +Investigation: + +1. Identify the failing operations: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m]))`. +2. Establish how long the underlying failure has persisted from the attempt-failure rate history; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the class-specific diagnosis. +3. Note the by-design case: `mutating_non_idempotent` operations (e.g. `vault_kv2_cas_write_key`, `vault_transit_create_key`) are never replayed, so a single retryable failure terminates them as `budget_exhausted` after one attempt. A spike confined to mutating operations means write-path failures, not an exhausted retry loop. +4. `deadline_exceeded` clustering with duration p99 near the operation deadline means the budget is being spent on slow attempts rather than fast failures — treat as a latency problem first. +5. Confirm client impact and, if the backend outage is confirmed external (Vault down), coordinate recovery there; RustFS will resume without intervention once the backend recovers. + +Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings in RustFS logs; Vault availability monitoring. + +## Threshold calibration + +Every numeric threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584). + +## Coverage gaps and planned metrics + +The following families are designed under rustfs/backlog#1584 but land in separate changes; they are intentionally absent from the dashboard and alert rules, and referencing their names before the emitting code merges would produce permanently-empty panels and never-firing alerts: + +- Key-cache effectiveness: hit/miss counters, entry gauge, eviction counter (replacing the former hardcoded-zero miss statistic). +- Key lifecycle: pending-deletion/tombstone gauges from the deletion-worker sweep and an aggregate rotation-age gauge. +- Vault credentials: token TTL remaining and fail-closed state gauges (today only the log warnings quoted above exist). +- Synthetic backend probe: probe outcome and failure-class metrics feeding the readiness cache. + +When one of these lands, add its panels, extend the alert rules, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel, and update this section. + +## Related documents + +- [KMS backend security properties](kms-backend-security.md) — backend trust boundaries, minimal Vault policies, rotation retention preconditions. +- [Vault KMS authentication runbook](vault-kms-authentication.md) — credential sources, refresh behavior, and the fail-closed window. +- `deploy/observability/README.md` — dashboard import notes for all RustFS dashboards.