From 04c59218506e012824a0a7720528f379f59cb332 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 1 Aug 2026 01:52:00 +0800 Subject: [PATCH 01/12] fix(s3): allow CopyObject COPY request metadata (#5514) * fix(s3): allow CopyObject COPY request metadata Co-Authored-By: heihutu * fix(kms): remove stale KmsClient import Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .../e2e_test/src/copy_object_metadata_test.rs | 33 ++++++++++--------- crates/kms/src/backup/local_export.rs | 1 - rustfs/src/app/object_usecase.rs | 13 -------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/crates/e2e_test/src/copy_object_metadata_test.rs b/crates/e2e_test/src/copy_object_metadata_test.rs index cfd257f06..f56fe7f41 100644 --- a/crates/e2e_test/src/copy_object_metadata_test.rs +++ b/crates/e2e_test/src/copy_object_metadata_test.rs @@ -117,9 +117,14 @@ mod tests { .key("assets/explicit-copy.js") .copy_source(format!("{bucket}/{key}")) .metadata_directive(MetadataDirective::Copy) + .customize() + .mutate_request(|request| { + request.headers_mut().insert("content-type", "application/octet-stream"); + request.headers_mut().insert("x-amz-meta-request-only", "ignored"); + }) .send() .await - .expect("explicit COPY directive failed"); + .expect("explicit COPY directive with request metadata failed"); let explicit_copy_head = client .head_object() .bucket(bucket) @@ -128,6 +133,18 @@ mod tests { .await .expect("HEAD failed after explicit COPY"); assert_eq!(explicit_copy_head.cache_control(), Some("max-age=60")); + assert_eq!(explicit_copy_head.content_type(), Some("text/javascript; charset=utf-8")); + assert_eq!( + explicit_copy_head.metadata().and_then(|metadata| metadata.get("mtime")), + Some(&"1777992333".to_string()) + ); + assert_eq!( + explicit_copy_head + .metadata() + .and_then(|metadata| metadata.get("request-only")), + None, + "COPY must ignore request metadata" + ); assert_eq!( explicit_copy_head.website_redirect_location(), None, @@ -571,20 +588,6 @@ mod tests { Some("InvalidArgument") ); - let ignored_replacement = client - .copy_object() - .bucket(bucket) - .key(key) - .copy_source(format!("{bucket}/{key}")) - .content_type("application/ignored") - .send() - .await - .expect_err("Replacement fields without REPLACE should be rejected"); - assert_eq!( - ignored_replacement.as_service_error().and_then(|error| error.code()), - Some("InvalidRequest") - ); - let unchanged = client .get_object() .bucket(bucket) diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs index d3905b31f..00bb905e8 100644 --- a/crates/kms/src/backup/local_export.rs +++ b/crates/kms/src/backup/local_export.rs @@ -560,7 +560,6 @@ async fn fsync_dir(path: &Path) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::backends::KmsClient; use crate::config::LocalConfig; use std::sync::Arc; use tempfile::TempDir; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 8d46feadd..806b07ff9 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -6072,19 +6072,6 @@ impl DefaultObjectUsecase { )); } }; - let has_replacement_metadata = metadata.is_some() - || cache_control.is_some() - || content_disposition.is_some() - || content_encoding.is_some() - || content_language.is_some() - || content_type.is_some() - || expires.is_some(); - if has_replacement_metadata && !replaces_metadata { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Replacement metadata requires the REPLACE metadata directive".to_string(), - )); - } let replacement_metadata = if replaces_metadata { validate_archive_content_encoding(&key, content_type.as_deref(), content_encoding.as_deref())?; let mut replacement_metadata = metadata.unwrap_or_default(); From 74019845c44f04c2e0c44df672060f8a6ad45d07 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 04:24:04 +0800 Subject: [PATCH 02/12] fix(kms): drop stale KmsClient trait import in local_export tests (#5519) The KmsClient trait was folded into KmsBackend (#5501), but the backup export tests merged afterwards (#5499) still imported it, breaking the crate's test build; create_key is an inherent LocalKmsClient method, so the import is simply unused. From 792f2ef2047bc8e147aaec2225a1b8b0aef0f791 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 05:28:10 +0800 Subject: [PATCH 03/12] docs(kms): add observability dashboard, alert rules and runbook (#5517) Add a Grafana dashboard for the four KMS backend operation metrics emitted at the policy choke point, Prometheus alert rules with conservative default thresholds (pending staging baseline calibration), and an operations runbook documenting the metric contract and the response procedure for each alert. Register the new alert rules file in the observability READMEs. Refs rustfs/backlog#1584 (part of rustfs/backlog#1562) --- .docker/observability/README.md | 10 + .docker/observability/README_ZH.md | 10 + .../prometheus-rules/rustfs-kms-alerts.yml | 188 +++++ deploy/observability/README.md | 2 + .../grafana/rustfs-kms-observability.json | 793 ++++++++++++++++++ docs/operations/kms-observability-runbook.md | 130 +++ 6 files changed, 1133 insertions(+) create mode 100644 .docker/observability/prometheus-rules/rustfs-kms-alerts.yml create mode 100644 deploy/observability/grafana/rustfs-kms-observability.json create mode 100644 docs/operations/kms-observability-runbook.md diff --git a/.docker/observability/README.md b/.docker/observability/README.md index ff5e0bbdb..cb4b9a881 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -60,6 +60,16 @@ The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-con | `CodecStreamingFallbackSpike` | Warning | Codec streaming fallback > 10x baseline for 10m | | `IoQueueSaturation` | Warning | IO queue utilization > 90% for 5m | +The file `prometheus-rules/rustfs-kms-alerts.yml` contains alerting rules for the KMS backend operation metrics. Thresholds are conservative defaults pending staging baseline calibration; response procedures live in `docs/operations/kms-observability-runbook.md`, and the matching dashboard is `deploy/observability/grafana/rustfs-kms-observability.json`. + +| Alert | Severity | Condition | +|-------|----------|-----------| +| `KmsBackendFatalErrors` | Critical | Fatal (non-retryable) attempt failures > 0 for 5m | +| `KmsBackendHighErrorRate` | Critical | Non-success operation ratio > 5% for 10m (with traffic guard) | +| `KmsBackendP99LatencyHigh` | Warning | Operation p99 duration (incl. retries) > 2s for 10m | +| `KmsBackendAttemptFailureSpike` | Warning | Attempt failure rate > 0.5/s for 10m | +| `KmsBackendRetryBudgetExhausted` | Warning | budget_exhausted / deadline_exceeded outcomes > 0.05/s for 10m | + ### Enabling Alert Rules Add the alert rules file to your Prometheus configuration: diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md index 97253f20c..f42674f6c 100644 --- a/.docker/observability/README_ZH.md +++ b/.docker/observability/README_ZH.md @@ -60,6 +60,16 @@ | `CodecStreamingFallbackSpike` | 警告 | Codec streaming 回退 > 10x 基线，持续 10 分钟 | | `IoQueueSaturation` | 警告 | IO 队列利用率 > 90%，持续 5 分钟 | +文件 `prometheus-rules/rustfs-kms-alerts.yml` 包含 KMS 后端操作指标的告警规则。阈值为保守默认值，待 staging 基线校准；响应流程见 `docs/operations/kms-observability-runbook.md`，配套仪表盘为 `deploy/observability/grafana/rustfs-kms-observability.json`。 + +| 告警 | 级别 | 条件 | +|------|------|------| +| `KmsBackendFatalErrors` | 严重 | fatal（不可重试）尝试失败 > 0，持续 5 分钟 | +| `KmsBackendHighErrorRate` | 严重 | 非 success 操作占比 > 5%，持续 10 分钟（含流量下限保护） | +| `KmsBackendP99LatencyHigh` | 警告 | 操作 p99 耗时（含重试）> 2s，持续 10 分钟 | +| `KmsBackendAttemptFailureSpike` | 警告 | 尝试失败率 > 0.5/s，持续 10 分钟 | +| `KmsBackendRetryBudgetExhausted` | 警告 | budget_exhausted / deadline_exceeded 结果 > 0.05/s，持续 10 分钟 | + ### 启用告警规则 在 Prometheus 配置中添加告警规则文件： diff --git a/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml b/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml new file mode 100644 index 000000000..e4f1faf03 --- /dev/null +++ b/.docker/observability/prometheus-rules/rustfs-kms-alerts.yml @@ -0,0 +1,188 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# RustFS KMS backend — Prometheus alerting rules +# ============================================================================= +# +# Metric source: the KMS operation-policy choke point in +# crates/kms/src/policy.rs. All label values are static enum strings +# (operation, op_class, outcome, error_class); key identifiers, key material, +# and tokens never appear in labels. +# +# Response procedures: docs/operations/kms-observability-runbook.md +# +# IMPORTANT — threshold status: every numeric threshold below is a +# conservative default chosen without a production baseline. Calibrate against +# a staging baseline before relying on these alerts for paging, and prefer +# loosening over tightening until the baseline exists. Formal SLO targets are +# deliberately not encoded here (see rustfs/backlog#1584). +# +# NOTE: prometheus.yml loads /etc/prometheus/rules/*.yml — keep the .yml +# extension or the file is silently ignored by the docker-compose stack. +# +# Validate: promtool check rules rustfs-kms-alerts.yml +# ============================================================================= + +groups: + # ========================================================================== + # Critical alerts — immediate action required + # ========================================================================== + - name: rustfs-kms-critical + interval: 30s + rules: + # ------------------------------------------------------------------ + # 1. KmsBackendFatalErrors + # Any attempt failure classified as fatal (non-retryable): auth + # or permission errors, malformed requests, missing keys. The + # policy never retries these, so even a low rate means real + # operations are failing right now. + # ------------------------------------------------------------------ + - alert: KmsBackendFatalErrors + expr: | + sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m])) > 0 + for: 5m + labels: + severity: critical + component: kms + annotations: + summary: "KMS backend fatal errors on operation {{ $labels.operation }}" + description: >- + Attempt failures classified as fatal are occurring at + {{ $value | printf "%.3f" }}/s on operation + {{ $labels.operation }}. Fatal failures are not retried: + each one is a KMS backend call that failed permanently + (authentication, permissions, malformed request, or a + missing key/version). + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendfatalerrors" + + # ------------------------------------------------------------------ + # 2. KmsBackendHighErrorRate + # Sustained share of operations terminating without success + # (fatal, budget_exhausted, deadline_exceeded). The cancelled + # outcome is excluded because shutdowns legitimately produce it. + # The traffic guard keeps a single failure on a near-idle + # cluster from firing the alert. + # Threshold: 5% for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendHighErrorRate + expr: | + ( + sum(rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m])) + / + clamp_min(sum(rate(rustfs_kms_backend_operations_total[5m])), 1e-9) + ) > 0.05 + and + sum(rate(rustfs_kms_backend_operations_total[5m])) > 0.02 + for: 10m + labels: + severity: critical + component: kms + annotations: + summary: "KMS backend non-success ratio above 5% for 10m" + description: >- + {{ $value | humanizePercentage }} of KMS backend operations + are terminating in fatal, budget_exhausted, or + deadline_exceeded. Object encryption and decryption paths + depending on the KMS are degraded or failing. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate" + + # ========================================================================== + # Warning alerts — investigation needed + # ========================================================================== + - name: rustfs-kms-warning + interval: 30s + rules: + # ------------------------------------------------------------------ + # 3. KmsBackendP99LatencyHigh + # p99 wall-clock duration of whole operations (attempts plus + # backoff) is sustained above 2 seconds. Because the histogram + # includes retries, a high p99 usually means the retry policy + # is absorbing backend failures, not that every call is slow. + # Threshold: 2s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendP99LatencyHigh + expr: | + histogram_quantile(0.99, + sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])) + ) > 2 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend operation p99 latency above 2s for 10m" + description: >- + The 99th-percentile KMS backend operation duration is + {{ $value | humanizeDuration }}, including retries and + backoff. Encryption and decryption latency is leaking into + S3 request latency. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendp99latencyhigh" + + # ------------------------------------------------------------------ + # 4. KmsBackendAttemptFailureSpike + # Aggregate attempt-failure rate (all error classes) sustained + # above an absolute floor. An absolute threshold is used instead + # of an offset-1d baseline ratio because fresh deployments have + # no baseline and an empty offset vector would keep a ratio + # alert from ever firing; switch to a baseline-relative form + # (see rustfs-get-optimization-alerts.yaml for the pattern) + # once a stable staging baseline exists. + # Threshold: 0.5/s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendAttemptFailureSpike + expr: | + sum(rate(rustfs_kms_backend_attempt_failures_total[5m])) > 0.5 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend attempt failures above 0.5/s for 10m" + description: >- + KMS backend attempts are failing at + {{ $value | printf "%.2f" }}/s across all error classes. + The retry policy may still be masking these from callers — + check the error-class breakdown before it stops absorbing + them. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendattemptfailurespike" + + # ------------------------------------------------------------------ + # 5. KmsBackendRetryBudgetExhausted + # Operations are running out of retry budget (budget_exhausted) + # or operation deadline (deadline_exceeded). These surface to + # callers as failed KMS operations even though every individual + # failure was retryable — the backend is unhealthy for longer + # than the policy can bridge. + # Threshold: 0.05/s for 10m — conservative default, calibrate + # against a staging baseline. + # ------------------------------------------------------------------ + - alert: KmsBackendRetryBudgetExhausted + expr: | + sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m])) > 0.05 + for: 10m + labels: + severity: warning + component: kms + annotations: + summary: "KMS backend operations exhausting retry budget ({{ $labels.outcome }})" + description: >- + KMS backend operations are terminating as + {{ $labels.outcome }} at {{ $value | printf "%.3f" }}/s. + Retryable failures are outlasting the retry budget, so + callers are seeing hard failures. + runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted" diff --git a/deploy/observability/README.md b/deploy/observability/README.md index d06deb7b3..9c19a9bd7 100644 --- a/deploy/observability/README.md +++ b/deploy/observability/README.md @@ -9,3 +9,5 @@ Import `grafana/rustfs-node-observability.json` into Grafana and select a Promet The dashboard uses the RustFS `server` metric label introduced with the node-local observability updates. `server` represents the RustFS node identity and is preferred for RustFS node comparisons. Prometheus `instance` still identifies the scrape target and remains useful for scrape/debugging views, but dashboards that compare RustFS nodes should group and filter by `server`. During a rolling upgrade, older nodes may still emit metrics without the `server` label. Complete the rollout before using this dashboard for node-by-node comparisons. + +`grafana/rustfs-kms-observability.json` covers the KMS backend operation metrics emitted at the operation-policy choke point. Unlike the node dashboard, KMS metrics do not carry the `server` label — use `job`/`instance` or promoted OTel resource attributes to split by node. Matching Prometheus alert rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`, and the alert response procedures are documented in `docs/operations/kms-observability-runbook.md`. diff --git a/deploy/observability/grafana/rustfs-kms-observability.json b/deploy/observability/grafana/rustfs-kms-observability.json new file mode 100644 index 000000000..0810a3e4a --- /dev/null +++ b/deploy/observability/grafana/rustfs-kms-observability.json @@ -0,0 +1,793 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "KMS backend operation metrics emitted at the operation-policy choke point (crates/kms/src/policy.rs). All label values are static enum strings; key identifiers, key material, and tokens never appear in labels. These metrics do not carry the RustFS `server` label — use your scrape topology (job/instance or promoted OTel resource attributes) to split by node. Alert response procedures: docs/operations/kms-observability-runbook.md.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Terminal outcomes of KMS backend operations. `fatal` means a non-retryable failure ended the operation on first observation; `budget_exhausted` and `deadline_exceeded` mean retries ran out; `cancelled` is normal during shutdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (outcome) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Backend Operation Rate by Outcome", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Operation names are static per-call-site identifiers (e.g. vault_kv2_read_key_version, vault_transit_encrypt, vault_login). `op_class` distinguishes read_idempotent, mutating_non_idempotent, and auth operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation, op_class) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{operation}} ({{op_class}})", + "range": true, + "refId": "A" + } + ], + "title": "Backend Operation Rate by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of operations that terminated in fatal, budget_exhausted, or deadline_exceeded. The cancelled outcome is plotted separately because shutdown windows legitimately spike it. The ratio is meaningless at near-zero traffic — read it together with the operation rate panels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome!~\"success|cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "non-success (excl. cancelled)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome=\"cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "cancelled", + "range": true, + "refId": "B" + } + ], + "title": "Non-Success Outcome Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-attempt failures by retry classification. retryable_conn covers connection-level failures, retryable_status covers retryable backend status codes, attempt_timeout covers attempts cut off by the per-attempt timeout, and fatal covers non-retryable failures (auth, permission, malformed request). Sustained fatal traffic is always actionable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total{operation=~\"$operation\"}[$__rate_interval]))", + "legendFormat": "{{error_class}}", + "range": true, + "refId": "A" + } + ], + "title": "Attempt Failure Rate by Error Class", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Wall-clock duration of whole operations, including retries and backoff sleeps. A rising p99 with a flat p50 usually means a slow retry tail (backend degradation), not a uniform slowdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "refId": "B" + } + ], + "title": "Operation Duration p50 / p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "p99 duration split by operation. Auth operations (vault_login, vault_token_renew) and mutating writes are expected to sit higher than idempotent reads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Operation Duration p99 by Operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Attempts one operation used before completing. Healthy operations average close to 1. A rising average or p99 means the retry policy is absorbing backend failures — read together with the attempt-failure panel to see the failure class.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(rustfs_kms_backend_operation_attempts_sum{operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum by (operation) (rate(rustfs_kms_backend_operation_attempts_count{operation=~\"$operation\"}[$__rate_interval])), 1e-9)", + "legendFormat": "{{operation}} avg", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_attempts_bucket{operation=~\"$operation\"}[$__rate_interval])))", + "legendFormat": "p99 (all operations)", + "range": true, + "refId": "B" + } + ], + "title": "Operation Attempts Distribution", + "type": "timeseries" + }, + { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Placeholder for KMS metrics planned by sibling changes of rustfs/backlog#1584 that have **not landed yet**. Do not add panels for these names until the emitting code is merged, or the panels will render empty and mask real gaps.\n\n- TODO: key-cache effectiveness — hit/miss counters, entry gauge, eviction counter (replaces the former hardcoded-zero miss stat).\n- TODO: key lifecycle — pending-deletion/tombstone gauges from the deletion worker sweep, aggregate rotation-age gauge.\n- TODO: Vault credentials — token TTL remaining and fail-closed state gauges.\n- TODO: synthetic backend probe — probe outcome/failure-class metrics feeding the readiness cache.\n\nWhen a family lands, replace one bullet with a real panel and keep this list in sync with docs/operations/kms-observability-runbook.md (Coverage gaps).", + "mode": "markdown" + }, + "title": "Planned Panels (TODO — metrics not landed yet)", + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "rustfs", + "observability", + "kms" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs_kms_backend_operations_total, operation)", + "hide": 0, + "includeAll": true, + "label": "KMS operation", + "multi": true, + "name": "operation", + "options": [], + "query": "label_values(rustfs_kms_backend_operations_total, operation)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RustFS KMS Observability", + "uid": "rustfs-kms-observability", + "version": 1, + "weekStart": "" +} diff --git a/docs/operations/kms-observability-runbook.md b/docs/operations/kms-observability-runbook.md new file mode 100644 index 000000000..310df6cbe --- /dev/null +++ b/docs/operations/kms-observability-runbook.md @@ -0,0 +1,130 @@ +# KMS observability runbook + +This runbook covers the KMS backend operation metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md). + +## Metric reference + +All four metrics are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through. Label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression. + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `rustfs_kms_backend_operations_total` | counter | `operation`, `op_class`, `outcome` | Operations executed under the operation policy, counted once per terminal outcome | +| `rustfs_kms_backend_attempt_failures_total` | counter | `operation`, `error_class` | Individual failed attempts, including attempts the retry policy later absorbed | +| `rustfs_kms_backend_operation_duration_seconds` | histogram | `operation`, `outcome` | Wall-clock duration of a whole operation, including retries and backoff sleeps | +| `rustfs_kms_backend_operation_attempts` | histogram | `operation`, `outcome` | Number of attempts one operation used before completing | + +Label values: + +- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `cancelled` (shutdown or caller cancellation). +- `op_class`: `read_idempotent` (safe to retry), `mutating_non_idempotent` (never replayed — a retryable failure terminates after a single attempt because the server may have processed the request), `auth` (login and token renewal). +- `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version). +- `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`. + +Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. These metrics do not carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`). + +Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of KMS series on a cluster using those backends is expected, not an outage. + +## Dashboard + +Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` (Prometheus data source) and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed there. + +The dashboard's "Planned Panels (TODO)" text panel lists metric families designed under rustfs/backlog#1584 but not yet landed (key-cache effectiveness, lifecycle gauges, token TTL, synthetic probe). Do not add panels or alerts for those names until the emitting code is merged; keep that panel and the [Coverage gaps](#coverage-gaps-and-planned-metrics) section below in sync as they land. + +## Alert rules + +The rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. The docker-compose Prometheus loads `/etc/prometheus/rules/*.yml`, so the `.yml` extension is load-bearing. Validate edits with `promtool check rules rustfs-kms-alerts.yml`. + +Every threshold in that file is a conservative default chosen without a production baseline; see [Threshold calibration](#threshold-calibration) before treating a firing alert as an SLO breach or a quiet one as health. + +## Alert response procedures + +### KmsBackendFatalErrors + +Meaning: attempts are failing with `error_class="fatal"` — failures the policy never retries. Each one is a KMS backend call that failed permanently (authentication, permissions, malformed request, or a missing key/version), so callers are seeing errors right now. This is the highest-signal KMS alert: fatal failures do not appear as background noise in a healthy system. + +Investigation: + +1. Break the rate down by operation: `sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m]))`. +2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs. +3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the minimal policy in [KMS backend security properties](kms-backend-security.md) (a policy that drifted or was re-scoped produces 403s that classify as fatal), and check the Vault audit log for the corresponding denied requests. +4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the rotation retention preconditions in [KMS backend security properties](kms-backend-security.md) and verify nobody destroyed version records under the key subtree. +5. Confirm blast radius with the outcome view: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="fatal"}[5m]))`. + +Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operation Rate by Outcome" dashboard panels; Vault server audit and server logs; S3-level 5xx on encrypted buckets. + +### KmsBackendHighErrorRate + +Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, or `deadline_exceeded`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page. + +Investigation: + +1. Break the failures down by outcome: `sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))`. +2. If `fatal` dominates, follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). +3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted) — the backend is unavailable or too slow for longer than the retry policy can bridge. +4. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured. + +Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts in this runbook. + +### KmsBackendP99LatencyHigh + +Meaning: the p99 wall-clock duration of KMS operations is sustained above 2s. The histogram includes retries and backoff sleeps, so a high p99 with a healthy p50 usually means a slow retry tail (a subset of calls failing and being retried), not a uniform slowdown. + +Investigation: + +1. Compare p50 and p99 on the "Operation Duration p50 / p99" panel. Flat p50 with elevated p99 points at retries; both elevated points at the backend or the network path being uniformly slow. +2. Split by operation with `histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])))` to see whether one backend call or all of them regressed. +3. Check the attempts histogram: an average meaningfully above 1 confirms the latency is retry-driven; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the failure classes. +4. If latency is not retry-driven, check the network path to Vault (TLS handshakes, DNS, proxies) and Vault's own telemetry (storage backend latency, load). +5. Remember that this latency sits inside S3 request latency for encrypted objects: sustained p99 near the operation deadline will start converting into `deadline_exceeded` outcomes. + +Related signals: the "Operation Duration p99 by Operation" and "Operation Attempts Distribution" panels; `KMS backend attempt failed with a retryable error; backing off before retry` warnings (fields: `operation`, `attempt`, `error_class`, `backoff`). + +### KmsBackendAttemptFailureSpike + +Meaning: individual attempts are failing at a sustained rate across all error classes. The retry policy may still be absorbing these — operations can keep succeeding while this alert fires — but the system is burning retry budget and running degraded, and a small further degradation will surface to callers. + +Investigation: + +1. Break the rate down by class: `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total[5m]))`. +2. `retryable_conn`: network-level failures — check connectivity, TLS, DNS, and whether Vault is down or restarting. +3. `retryable_status`: the backend answered with a retryable error — check Vault health and seal status (a sealed Vault returns 503, which lands here), and Vault-side rate limiting. +4. `attempt_timeout`: attempts are being cut off by the per-attempt timeout — either the backend is slow (correlate with [KmsBackendP99LatencyHigh](#kmsbackendp99latencyhigh)) or the configured attempt timeout is too tight for the deployment's network path. +5. `fatal`: follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). +6. Grep RustFS logs for `KMS backend attempt failed with a retryable error; backing off before retry` — the structured fields (`operation`, `attempt`, `error_class`, `backoff`) identify which call sites are cycling. + +Related signals: the "Attempt Failure Rate by Error Class" panel; the attempts histogram average rising above 1. + +### KmsBackendRetryBudgetExhausted + +Meaning: operations are terminating as `budget_exhausted` or `deadline_exceeded` — every individual failure was retryable, but the backend stayed unhealthy for longer than the retry policy could bridge, so callers received hard failures. + +Investigation: + +1. Identify the failing operations: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m]))`. +2. Establish how long the underlying failure has persisted from the attempt-failure rate history; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the class-specific diagnosis. +3. Note the by-design case: `mutating_non_idempotent` operations (e.g. `vault_kv2_cas_write_key`, `vault_transit_create_key`) are never replayed, so a single retryable failure terminates them as `budget_exhausted` after one attempt. A spike confined to mutating operations means write-path failures, not an exhausted retry loop. +4. `deadline_exceeded` clustering with duration p99 near the operation deadline means the budget is being spent on slow attempts rather than fast failures — treat as a latency problem first. +5. Confirm client impact and, if the backend outage is confirmed external (Vault down), coordinate recovery there; RustFS will resume without intervention once the backend recovers. + +Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings in RustFS logs; Vault availability monitoring. + +## Threshold calibration + +Every numeric threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584). + +## Coverage gaps and planned metrics + +The following families are designed under rustfs/backlog#1584 but land in separate changes; they are intentionally absent from the dashboard and alert rules, and referencing their names before the emitting code merges would produce permanently-empty panels and never-firing alerts: + +- Key-cache effectiveness: hit/miss counters, entry gauge, eviction counter (replacing the former hardcoded-zero miss statistic). +- Key lifecycle: pending-deletion/tombstone gauges from the deletion-worker sweep and an aggregate rotation-age gauge. +- Vault credentials: token TTL remaining and fail-closed state gauges (today only the log warnings quoted above exist). +- Synthetic backend probe: probe outcome and failure-class metrics feeding the readiness cache. + +When one of these lands, add its panels, extend the alert rules, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel, and update this section. + +## Related documents + +- [KMS backend security properties](kms-backend-security.md) — backend trust boundaries, minimal Vault policies, rotation retention preconditions. +- [Vault KMS authentication runbook](vault-kms-authentication.md) — credential sources, refresh behavior, and the fail-closed window. +- `deploy/observability/README.md` — dashboard import notes for all RustFS dashboards. From c09d11ff3be36070f9cc929bd19c030eb6c1ec5d Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 1 Aug 2026 08:33:46 +0800 Subject: [PATCH 04/12] fix(config): fence persisted config updates and reloads (#5512) * fix(config): fence persisted config updates and reloads * fix(ci): unblock config and e2e checks Co-Authored-By: heihutu --------- Co-authored-by: houseme Co-authored-by: heihutu --- crates/ecstore/src/api/mod.rs | 13 +- crates/ecstore/src/config/com.rs | 695 ++++++++++++++++-- crates/ecstore/src/set_disk/mod.rs | 225 +++++- crates/notify/src/config_manager.rs | 201 ++--- crates/notify/src/lib.rs | 5 +- crates/notify/src/storage_api.rs | 36 +- .../admin/handlers/audit_runtime_config.rs | 187 ++++- rustfs/src/admin/handlers/config_admin.rs | 343 ++++----- rustfs/src/admin/service/config.rs | 446 +++++++++-- rustfs/src/admin/storage_api.rs | 25 +- scripts/e2e-run.sh | 14 +- 11 files changed, 1711 insertions(+), 479 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 15aebe9d9..cfba93f88 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -267,12 +267,13 @@ pub mod config { pub mod com { pub use crate::config::com::{ COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS, - ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs, - read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate, - read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config, - save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock, - save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock, - with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock, + ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, + is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata, + read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock, + read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config, + save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation, + server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock, + with_server_config_read_lock, with_server_config_write_lock, }; } diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 539464173..c62aa74d8 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -53,12 +53,14 @@ use std::sync::LazyLock; use std::sync::{Arc, RwLock}; use tokio::sync::{OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; use tracing::{debug, error, info, instrument, warn}; +use uuid::Uuid; pub const CONFIG_PREFIX: &str = "config"; const SERVER_CONFIG_OBJECT: &str = "config/config.json"; +const CONFIG_TRANSACTION_LOCK_SUFFIX: &str = ".transaction.lock"; -// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock -// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order. +// Server-config lock order: SERVER_CONFIG_LOCK -> transaction lock -> +// SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order. static SERVER_CONFIG_LOCK: LazyLock>> = LazyLock::new(|| Arc::new(AsyncRwLock::new(()))); fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error { @@ -76,8 +78,11 @@ where T: Send + 'static, { tokio::spawn(async move { - // Lock order: SERVER_CONFIG_LOCK -> namespace write lock. + // Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock. let _local_guard = SERVER_CONFIG_LOCK.write().await; + let transaction_lock = server_config_transaction_lock_path(); + let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; + let _transaction_guard = transaction_lock.get_write_lock(get_lock_acquire_timeout()).await?; let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?; Ok(operation().await) @@ -96,8 +101,11 @@ where T: Send + 'static, { tokio::spawn(async move { - // Lock order: SERVER_CONFIG_LOCK -> namespace read lock. + // Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock. let _local_guard = SERVER_CONFIG_LOCK.read().await; + let transaction_lock = server_config_transaction_lock_path(); + let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; + let _transaction_guard = transaction_lock.get_read_lock(get_lock_acquire_timeout()).await?; let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?; Ok(operation().await) @@ -567,6 +575,21 @@ where } pub async fn save_config_with_opts(api: Arc, file: &str, data: Vec, opts: &ObjectOptions) -> Result<()> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + save_config_with_opts_and_metadata(api, file, data, opts).await.map(|_| ()) +} + +async fn save_config_with_opts_and_metadata(api: Arc, file: &str, data: Vec, opts: &ObjectOptions) -> Result where S: ObjectIO< Error = Error, @@ -579,11 +602,13 @@ where >, { let mut put_data = PutObjReader::from_vec(data); - if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await { - error!("save_config_with_opts: err: {:?}, file: {}", err, file); - return Err(err); + match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await { + Ok(object_info) => Ok(object_info), + Err(err) => { + error!("save_config_with_opts: err: {:?}, file: {}", err, file); + Err(err) + } } - Ok(()) } fn new_server_config() -> Config { @@ -594,8 +619,12 @@ async fn new_and_save_server_config(api: Arc) -> Result where S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking, { + let snapshot = read_server_config_snapshot(api.clone()).await?; + if snapshot.object_exists() { + return Ok(snapshot.config.clone()); + } let cfg = new_server_config(); - save_server_config(api, &cfg).await?; + save_server_config_snapshot(api, &cfg, &snapshot).await?; Ok(cfg) } @@ -617,6 +646,10 @@ pub fn server_config_path() -> String { SERVER_CONFIG_OBJECT.to_string() } +fn server_config_transaction_lock_path() -> String { + format!("{}{CONFIG_TRANSACTION_LOCK_SUFFIX}", server_config_path()) +} + fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS { let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| { let mut section = HashMap::new(); @@ -819,6 +852,9 @@ fn apply_external_scalar_config_map( let Some(config_value) = root.get(descriptor.subsystem_key) else { return Ok(false); }; + if descriptor.subsystem_key == HEAL_SUB_SYS && config_value.is_null() { + return Ok(false); + } let overrides = decode_scalar_config_value(config_value, descriptor)?; if overrides.is_empty() { @@ -1463,21 +1499,142 @@ fn build_audit_object(cfg: &Config) -> Map { build_target_object(cfg, &audit_target_descriptors()) } +fn sync_rendered_target_instance(existing: Value, rendered: Option<&Value>, valid_keys: &[&str]) -> Option { + match existing { + Value::Object(mut instance) => { + for key in valid_keys { + instance.remove(*key); + } + if let Some(Value::Object(rendered)) = rendered { + instance.extend(rendered.clone()); + } + (!instance.is_empty()).then_some(Value::Object(instance)) + } + Value::Array(entries) => { + let mut pending = rendered + .and_then(Value::as_object) + .map(|rendered| { + rendered + .iter() + .filter_map(|(key, value)| parse_target_scalar_value(key, value).map(|value| (key.clone(), value))) + .collect::>() + }) + .unwrap_or_default(); + let mut updated = Vec::with_capacity(entries.len().saturating_add(pending.len())); + for entry in entries { + let Some(entry_obj) = entry.as_object() else { + updated.push(entry); + continue; + }; + let Some(key) = entry_obj.get("key").and_then(Value::as_str) else { + updated.push(entry); + continue; + }; + if !valid_keys.contains(&key) { + updated.push(entry); + continue; + } + let Some(value) = pending.remove(key) else { + continue; + }; + let mut entry_obj = entry_obj.clone(); + entry_obj.insert("value".to_string(), Value::String(value)); + updated.push(Value::Object(entry_obj)); + } + updated.extend(rendered_scalar_config_kvs_entries(&pending)); + (!updated.is_empty()).then_some(Value::Array(updated)) + } + value if rendered.is_none() => Some(value), + _ => rendered.cloned(), + } +} + fn sync_rendered_target_object( target_obj: &mut Map, rendered_target: &Map, descriptors: &[TargetConfigDescriptor], ) { for descriptor in descriptors { - match rendered_target.get(descriptor.external_key) { - Some(Value::Object(v)) => { - target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone())); - target_obj.remove(descriptor.subsystem_key); + let existing = target_obj.remove(descriptor.external_key); + let alias = target_obj.remove(descriptor.subsystem_key); + let mut section = existing + .or(alias) + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + let rendered = rendered_target.get(descriptor.external_key).and_then(Value::as_object); + + if is_target_instance_shorthand(§ion, descriptor.valid_keys) { + let has_named_instances = rendered.is_some_and(|instances| instances.keys().any(|name| name != "default")); + if !has_named_instances { + if let Some(section) = sync_rendered_target_instance( + Value::Object(section), + rendered.and_then(|instances| instances.get("default")), + descriptor.valid_keys, + ) { + target_obj.insert(descriptor.external_key.to_string(), section); + } + continue; } - _ => { - target_obj.remove(descriptor.external_key); - target_obj.remove(descriptor.subsystem_key); + + let mut nested = Map::new(); + if let Some(default) = sync_rendered_target_instance( + Value::Object(section), + rendered.and_then(|instances| instances.get("default")), + descriptor.valid_keys, + ) { + nested.insert("default".to_string(), default); } + if let Some(rendered) = rendered { + for (instance_name, instance) in rendered { + if instance_name != "default" { + nested.insert(instance_name.clone(), instance.clone()); + } + } + } + if !nested.is_empty() { + target_obj.insert(descriptor.external_key.to_string(), Value::Object(nested)); + } + continue; + } + + if let Some(default_alias) = section.remove(DEFAULT_DELIMITER) { + if let Some(default) = section.get_mut("default") { + if let Some(alias) = sync_rendered_target_instance(default_alias, None, descriptor.valid_keys) { + match (default, alias) { + (Value::Object(default), Value::Object(alias)) => { + for (key, value) in alias { + default.entry(key).or_insert(value); + } + } + (Value::Array(default), Value::Array(alias)) => default.extend(alias), + _ => {} + } + } + } else { + section.insert("default".to_string(), default_alias); + } + } + + let mut merged = Map::new(); + for (instance_name, instance) in section { + if let Some(instance) = sync_rendered_target_instance( + instance, + rendered.and_then(|instances| instances.get(&instance_name)), + descriptor.valid_keys, + ) { + merged.insert(instance_name, instance); + } + } + if let Some(rendered) = rendered { + for (instance_name, instance) in rendered { + if !merged.contains_key(instance_name) { + merged.insert(instance_name.clone(), instance.clone()); + } + } + } + + if !merged.is_empty() { + target_obj.insert(descriptor.external_key.to_string(), Value::Object(merged)); } } } @@ -1496,6 +1653,14 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result v, _ => Map::new(), }; + for key in [ + storageclass::CLASS_STANDARD, + storageclass::CLASS_RRS, + storageclass::OPTIMIZE, + storageclass::INLINE_BLOCK, + ] { + sc_obj.remove(key); + } for (k, v) in build_storageclass_object(cfg) { sc_obj.insert(k, v); } @@ -1503,7 +1668,10 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result bool { matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty()) && matches!(root.get("storageclass"), Some(Value::Object(_))) && !root.contains_key("storage_class") + && !matches!(root.get(HEAL_SUB_SYS), Some(Value::Null)) } fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool { @@ -1593,7 +1762,7 @@ where FileInfo = FileInfo, ObjectToDelete = ObjectToDelete, DeletedObject = DeletedObject, - >, + > + NamespaceLocking, { if let Some(decrypt) = &decrypt_fn { register_server_config_decrypt_fn(decrypt.clone()); @@ -1601,14 +1770,7 @@ where let config_file = server_config_path(); match api - .get_object_info( - RUSTFS_META_BUCKET, - &config_file, - &ObjectOptions { - no_lock: true, - ..Default::default() - }, - ) + .get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default()) .await { Ok(_) => { @@ -1624,7 +1786,6 @@ where let opts = ObjectOptions { max_parity: true, - no_lock: true, ..Default::default() }; @@ -1677,7 +1838,33 @@ where } }; - match save_config(api, &config_file, normalized).await { + let snapshot = match read_server_config_snapshot(api.clone()).await { + Ok(snapshot) => snapshot, + Err(err) => { + warn!("recheck target server config failed, skip migration: {:?}", err); + return; + } + }; + if snapshot.object_exists() { + debug!("server config was created while legacy migration was preparing, skip migration"); + return; + } + + match save_config_with_opts( + api, + &config_file, + normalized, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { Ok(()) => { info!("Migrated compatible server config from legacy metadata bucket"); } @@ -1769,8 +1956,13 @@ where { let config_file = server_config_path(); - // Try to read the configuration file - match read_config_no_lock(api.clone(), &config_file).await { + // Try to read the configuration file. + let data = if namespace_lock_held { + read_config_no_lock(api.clone(), &config_file).await + } else { + read_config(api.clone(), &config_file).await + }; + match data { Ok(data) => read_server_config(api, &data, namespace_lock_held).await, Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await, Err(err) => handle_config_read_error(err, &config_file), @@ -1787,7 +1979,12 @@ where warn!("Received empty configuration data, try to reread from '{}'", config_file); // Try to read the configuration again - match read_config_no_lock(api.clone(), &config_file).await { + let data = if namespace_lock_held { + read_config_no_lock(api.clone(), &config_file).await + } else { + read_config(api.clone(), &config_file).await + }; + match data { Ok(cfg_data) => { let cfg = decode_persisted_server_config(&cfg_data)?; return Ok(cfg.merge()); @@ -2036,11 +2233,16 @@ pub struct ServerConfigSnapshot { raw: Option>, seed: Option>, etag: Option, + generation: Option, _local_guard: OwnedRwLockWriteGuard<()>, _guard: rustfs_lock::NamespaceLockGuard, } impl ServerConfigSnapshot { + pub fn object_exists(&self) -> bool { + self.raw.is_some() + } + pub fn ensure_lock_held(&self) -> Result<()> { if self._guard.is_lock_lost() { return Err(Error::other("server config transaction lock was lost")); @@ -2051,12 +2253,34 @@ impl ServerConfigSnapshot { pub fn is_lock_lost(&self) -> bool { self._guard.is_lock_lost() } + + pub fn generation(&self) -> Option { + self.generation + } } -/// Read a server config transaction snapshot while holding the same local and -/// distributed write locks used by every other server-config writer. Internal -/// reads and the later conditional write use no-lock object I/O; the guards -/// remain live until the snapshot is dropped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerConfigSaveResult { + persisted: bool, + generation: Option, +} + +impl ServerConfigSaveResult { + pub fn persisted(&self) -> bool { + self.persisted + } + + pub fn generation(&self) -> Option { + self.generation + } +} + +/// Read a server config transaction snapshot while holding a dedicated +/// transaction lock. The config object's normal namespace lock remains +/// available to fence reads and the conditional write at commit time. +/// The transaction guard remains live until the snapshot is dropped, +/// serializing persistence and history ordering across admin nodes. Runtime +/// state is reloaded from the durable object after this guard is released. pub async fn read_server_config_snapshot(api: Arc) -> Result where S: ObjectIO< @@ -2071,12 +2295,10 @@ where { let config_file = server_config_path(); let local_guard = SERVER_CONFIG_LOCK.clone().write_owned().await; - let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &config_file).await?; + let transaction_lock = server_config_transaction_lock_path(); + let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?; let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?; - let read_options = ObjectOptions { - no_lock: true, - ..Default::default() - }; + let read_options = ObjectOptions::default(); match read_config_with_metadata_inner(api, &config_file, &read_options, true).await { Ok((raw, object_info)) => { let (config, seed) = decode_persisted_server_config_with_seed(&raw)?; @@ -2085,6 +2307,7 @@ where raw: Some(raw), seed: Some(seed), etag: object_info.etag, + generation: object_info.data_dir.filter(|generation| !generation.is_nil()), _local_guard: local_guard, _guard: guard, }) @@ -2094,6 +2317,7 @@ where raw: None, seed: None, etag: None, + generation: None, _local_guard: local_guard, _guard: guard, }), @@ -2108,6 +2332,27 @@ where /// lock, so a concurrent update or transaction lease loss cannot commit an /// unfenced overwrite. pub async fn save_server_config_snapshot(api: Arc, cfg: &Config, snapshot: &ServerConfigSnapshot) -> Result +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + NamespaceLocking, +{ + save_server_config_snapshot_with_generation(api, cfg, snapshot) + .await + .map(|result| result.persisted()) +} + +pub async fn save_server_config_snapshot_with_generation( + api: Arc, + cfg: &Config, + snapshot: &ServerConfigSnapshot, +) -> Result where S: ObjectIO< Error = Error, @@ -2126,13 +2371,19 @@ where && configs_semantically_equal(&snapshot.config, cfg) { debug!("server config unchanged and already in standard object shape, skip write"); - return Ok(false); + return Ok(ServerConfigSaveResult { + persisted: false, + generation: snapshot.generation(), + }); } let data = encode_server_config_blob(cfg, snapshot.seed.as_deref())?; if snapshot.raw.as_deref().is_some_and(|current| current == data.as_slice()) { debug!("server config bytes unchanged after encode, skip write"); - return Ok(false); + return Ok(ServerConfigSaveResult { + persisted: false, + generation: snapshot.generation(), + }); } let http_preconditions = if snapshot.raw.is_some() { @@ -2152,19 +2403,22 @@ where } }; - save_config_with_opts( + snapshot.ensure_lock_held()?; + let object_info = save_config_with_opts_and_metadata( api, &config_file, data, &ObjectOptions { max_parity: true, - no_lock: true, http_preconditions: Some(http_preconditions), ..Default::default() }, ) .await?; - Ok(true) + Ok(ServerConfigSaveResult { + persisted: true, + generation: object_info.data_dir.filter(|generation| !generation.is_nil()), + }) } /// Saves the server config while an upper layer holds the namespace write @@ -2301,8 +2555,9 @@ mod tests { use super::{ SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config, - lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate, - read_server_config_snapshot, save_server_config, save_server_config_snapshot, server_config_path, storage_class_kvs_mut, + lookup_configs, new_and_save_server_config, read_config, read_config_preserve_empty, read_config_with_metadata, + read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot, + save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut, }; use crate::config::{audit, heal, notify, oidc, scanner}; use crate::disk::endpoint::Endpoint; @@ -2311,7 +2566,9 @@ mod tests { use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::runtime::sources as runtime_sources; use crate::set_disk::SetDisks; - use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, range::HTTPRangeSpec}; + use crate::storage_api_contracts::{ + admin::StorageAdminApi, namespace::NamespaceLocking as _, object::HTTPPreconditions, range::HTTPRangeSpec, + }; use http::HeaderMap; use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; use rustfs_config::notify::{ @@ -3104,6 +3361,85 @@ mod tests { ); } + #[test] + fn root_heal_null_decodes_as_no_override_and_is_canonicalized_on_save() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "heal":null, + "future_root":{"mode":"keep"}, + "openid":{"default":{ + "config_url":"https://issuer.example/.well-known/openid-configuration", + "client_id":"console", + "client_secret":"oidc-secret", + "future_provider_control":"keep" + }}, + "notify":{"webhook":{"primary":{ + "enable":true, + "endpoint":"https://notify.example/hook", + "auth_token":"notify-secret", + "future_notify_control":"keep" + }}}, + "logger":{"webhook":{"primary":{ + "enable":true, + "endpoint":"https://audit.example/hook", + "auth_token":"audit-secret", + "future_audit_control":"keep" + }}} + }"#; + + let cfg = decode_server_config_blob(seed).expect("root heal null should mean no persisted override"); + assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none()); + assert!(!is_standard_object_server_config(seed)); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("legacy seed should canonicalize on an authorized save"); + let value: Value = serde_json::from_slice(&encoded).expect("canonical config should be valid JSON"); + assert!(value.get(HEAL_SUB_SYS).is_none()); + assert_eq!(value["future_root"]["mode"].as_str(), Some("keep")); + assert_eq!(value["openid"]["default"]["client_secret"].as_str(), Some("oidc-secret")); + assert_eq!(value["openid"]["default"]["future_provider_control"].as_str(), Some("keep")); + assert_eq!(value["notify"]["webhook"]["primary"]["auth_token"].as_str(), Some("notify-secret")); + assert_eq!(value["notify"]["webhook"]["primary"]["future_notify_control"].as_str(), Some("keep")); + assert_eq!(value["logger"]["webhook"]["primary"]["auth_token"].as_str(), Some("audit-secret")); + assert_eq!(value["logger"]["webhook"]["primary"]["future_audit_control"].as_str(), Some("keep")); + assert!(is_standard_object_server_config(&encoded)); + } + + #[test] + fn invalid_scalar_and_nested_null_config_shapes_remain_rejected() { + let invalid_sections = [ + r#""scanner":null"#, + r#""heal":"""#, + r#""heal":false"#, + r#""heal":0"#, + r#""heal":{"default":null}"#, + r#""heal":{"_":null}"#, + r#""heal":{"bitrot_cycle":null}"#, + r#""heal":[{"key":"bitrot_cycle","value":null}]"#, + ]; + + for section in invalid_sections { + let input = format!(r#"{{"version":"33","storageclass":{{"standard":"","rrs":""}},{section}}}"#); + let err = decode_server_config_blob(input.as_bytes()).expect_err("invalid scalar shape must remain rejected"); + assert!( + err.to_string().contains("expected"), + "invalid section {section} returned an unrelated error: {err}" + ); + } + } + + #[test] + fn valid_heal_object_and_kvs_array_shapes_remain_accepted() { + let empty_object = br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":{}}"#; + let cfg = decode_server_config_blob(empty_object).expect("empty heal object should decode as no override"); + assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none()); + + let kvs_array = + br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":[{"key":"bitrot_cycle","value":"off"}]}"#; + let cfg = decode_server_config_blob(kvs_array).expect("heal KVS array should decode"); + assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_some()); + } + #[test] fn scanner_update_preserves_unknown_root_and_oidc_provider_fields() { let seed = br#"{ @@ -3131,6 +3467,171 @@ mod tests { assert_eq!(value["openid"]["default"]["client_id"].as_str(), Some("console")); } + #[test] + fn storageclass_reset_removes_stale_inline_block_from_seed() { + let seed = br#"{ + "version":"33", + "storageclass":{ + "standard":"EC:2", + "rrs":"EC:1", + "optimize":"availability", + "inline_block":"64KiB", + "future_storage_control":"keep" + } + }"#; + + let encoded = encode_server_config_blob(&Config::new(), Some(seed)).expect("storageclass reset should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let storageclass = value["storageclass"].as_object().expect("storageclass object"); + + assert!(storageclass.get(crate::config::storageclass::INLINE_BLOCK).is_none()); + assert_eq!(storageclass["future_storage_control"].as_str(), Some("keep")); + } + + #[test] + fn target_update_preserves_unknown_fields_without_restoring_removed_instances() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{ + "primary":{ + "enable":true, + "endpoint":"https://notify.example/old", + "auth_token":"notify-secret", + "future_control":"keep" + }, + "removed":{"enable":true,"endpoint":"https://notify.example/removed"}, + "retained":{"enable":true,"endpoint":"https://notify.example/retained","future_control":"keep"}, + "enable":{"enable":true,"endpoint":"https://notify.example/named-enable","future_control":"keep"} + }} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("target seed should decode"); + let webhook = cfg + .0 + .get_mut(NOTIFY_WEBHOOK_SUB_SYS) + .expect("notify webhook subsystem should exist"); + webhook + .get_mut("primary") + .expect("primary target should exist") + .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string()); + webhook.remove("removed"); + webhook.remove("retained"); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let webhook = value["notify"]["webhook"].as_object().expect("webhook section"); + + assert_eq!(webhook["primary"]["endpoint"].as_str(), Some("https://notify.example/new")); + assert_eq!(webhook["primary"]["auth_token"].as_str(), Some("notify-secret")); + assert_eq!(webhook["primary"]["future_control"].as_str(), Some("keep")); + assert!(webhook.get("removed").is_none()); + assert_eq!(webhook["retained"]["future_control"].as_str(), Some("keep")); + assert!(webhook["retained"].get("enable").is_none()); + assert!(webhook["retained"].get("endpoint").is_none()); + assert_eq!(webhook["enable"]["endpoint"].as_str(), Some("https://notify.example/named-enable")); + assert_eq!(webhook["enable"]["future_control"].as_str(), Some("keep")); + } + + #[test] + fn shorthand_target_update_preserves_shape_and_unknown_nested_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{ + "enable":true, + "endpoint":"https://notify.example/old", + "future_control":{"endpoint":"leave-untouched","mode":"keep"} + }} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("shorthand target should decode"); + cfg.0 + .get_mut(NOTIFY_WEBHOOK_SUB_SYS) + .and_then(|targets| targets.get_mut(DEFAULT_DELIMITER)) + .expect("default webhook target should exist") + .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string()); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("shorthand target update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let webhook = value["notify"]["webhook"].as_object().expect("webhook shorthand object"); + + assert_eq!(webhook["endpoint"].as_str(), Some("https://notify.example/new")); + assert!(webhook.get("default").is_none()); + assert_eq!(webhook["future_control"]["endpoint"].as_str(), Some("leave-untouched")); + assert_eq!(webhook["future_control"]["mode"].as_str(), Some("keep")); + let decoded = decode_server_config_blob(&encoded).expect("updated shorthand target should remain decodable"); + assert_eq!( + decoded + .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER) + .expect("updated default webhook target") + .get(rustfs_config::WEBHOOK_ENDPOINT), + "https://notify.example/new" + ); + } + + #[test] + fn target_kvs_update_preserves_unknown_entries_and_attributes() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{"primary":[ + {"key":"enable","value":"on","hidden_if_empty":false}, + {"key":"endpoint","value":"https://notify.example/old","future_attribute":"keep-endpoint"}, + {"key":"future_control","value":"keep","future_attribute":"keep-control"} + ]}} + }"#; + let mut cfg = decode_server_config_blob(seed).expect("target KVS seed should decode"); + cfg.0 + .get_mut(NOTIFY_WEBHOOK_SUB_SYS) + .and_then(|targets| targets.get_mut("primary")) + .expect("primary webhook target should exist") + .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string()); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target KVS update should encode"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let entries = value["notify"]["webhook"]["primary"] + .as_array() + .expect("target KVS shape should be preserved"); + let endpoint = entries + .iter() + .find(|entry| entry["key"].as_str() == Some(rustfs_config::WEBHOOK_ENDPOINT)) + .expect("endpoint entry should remain"); + let future = entries + .iter() + .find(|entry| entry["key"].as_str() == Some("future_control")) + .expect("unknown target entry should remain"); + + assert_eq!(endpoint["value"].as_str(), Some("https://notify.example/new")); + assert_eq!(endpoint["future_attribute"].as_str(), Some("keep-endpoint")); + assert_eq!(future["value"].as_str(), Some("keep")); + assert_eq!(future["future_attribute"].as_str(), Some("keep-control")); + } + + #[test] + fn target_default_alias_is_canonicalized_without_losing_unknown_fields() { + let seed = br#"{ + "version":"33", + "storageclass":{"standard":"","rrs":""}, + "notify":{"webhook":{ + "_":{"enable":false,"endpoint":"https://notify.example/alias","future_alias":"keep"}, + "default":{"enable":true,"endpoint":"https://notify.example/default","future_default":"keep"} + }} + }"#; + let cfg = decode_server_config_blob(seed).expect("dual default aliases should decode"); + let expected_endpoint = cfg + .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER) + .expect("default webhook target should exist") + .get(rustfs_config::WEBHOOK_ENDPOINT); + + let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("default alias should canonicalize"); + let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json"); + let webhook = value["notify"]["webhook"].as_object().expect("webhook section"); + + assert!(webhook.get(DEFAULT_DELIMITER).is_none()); + assert_eq!(webhook["default"]["endpoint"].as_str(), Some(expected_endpoint.as_str())); + assert_eq!(webhook["default"]["future_alias"].as_str(), Some("keep")); + assert_eq!(webhook["default"]["future_default"].as_str(), Some("keep")); + } + #[test] fn test_scanner_config_changes_are_semantically_significant() { let baseline = Config::new(); @@ -4124,6 +4625,7 @@ mod tests { /// What reads of the config object currently return. enum RecoveryReadState { + Missing, Blob(Vec), QuorumError, } @@ -4136,6 +4638,7 @@ mod tests { heal_calls: AtomicUsize, write_calls: AtomicUsize, last_put_no_lock: AtomicBool, + last_put_preconditions: Mutex>, revision: AtomicUsize, drive_counts: Vec, lock_manager: Arc, @@ -4150,6 +4653,7 @@ mod tests { heal_calls: AtomicUsize::new(0), write_calls: AtomicUsize::new(0), last_put_no_lock: AtomicBool::new(false), + last_put_preconditions: Mutex::new(None), revision: AtomicUsize::new(1), drive_counts: vec![2], lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()), @@ -4206,6 +4710,7 @@ mod tests { _opts: &ObjectOptions, ) -> Result { let data = match &*self.state.lock().expect("state lock poisoned") { + RecoveryReadState::Missing => return Err(Error::ConfigNotFound), RecoveryReadState::Blob(data) => data.clone(), RecoveryReadState::QuorumError => return Err(Error::ErasureReadQuorum), }; @@ -4213,6 +4718,9 @@ mod tests { size: data.len() as i64, actual_size: data.len() as i64, etag: Some(format!("config-{}", self.revision.load(Ordering::SeqCst))), + data_dir: Some(uuid::Uuid::from_u128( + u128::try_from(self.revision.load(Ordering::SeqCst)).expect("test revision should fit in u128"), + )), ..Default::default() }; Ok(GetObjectReader { @@ -4231,15 +4739,19 @@ mod tests { opts: &ObjectOptions, ) -> Result { let current_etag = format!("config-{}", self.revision.load(Ordering::SeqCst)); + let object_exists = matches!(&*self.state.lock().expect("state lock poisoned"), RecoveryReadState::Blob(_)); if let Some(preconditions) = &opts.http_preconditions - && (preconditions.if_match_value().is_some_and(|etag| etag != current_etag) - || preconditions.if_none_match_value() == Some("*")) + && (preconditions + .if_match_value() + .is_some_and(|etag| !object_exists || etag != current_etag) + || (object_exists && preconditions.if_none_match_value() == Some("*"))) { return Err(Error::PreconditionFailed); } let mut body = Vec::new(); data.stream.read_to_end(&mut body).await?; self.last_put_no_lock.store(opts.no_lock, Ordering::SeqCst); + *self.last_put_preconditions.lock().expect("preconditions lock poisoned") = opts.http_preconditions.clone(); self.write_calls.fetch_add(1, Ordering::SeqCst); *self.state.lock().expect("state lock poisoned") = RecoveryReadState::Blob(body.clone()); let revision = self.revision.fetch_add(1, Ordering::SeqCst) + 1; @@ -4247,6 +4759,7 @@ mod tests { size: i64::try_from(body.len()).expect("test config should fit in i64"), actual_size: i64::try_from(body.len()).expect("test config should fit in i64"), etag: Some(format!("config-{revision}")), + data_dir: Some(uuid::Uuid::from_u128(u128::try_from(revision).expect("test revision should fit in u128"))), ..Default::default() }) } @@ -4284,10 +4797,18 @@ mod tests { .expect("scanner-only config change should be persisted"); assert_eq!(store.write_calls.load(Ordering::SeqCst), 1); - assert!(store.last_put_no_lock.load(Ordering::SeqCst)); + assert!(!store.last_put_no_lock.load(Ordering::SeqCst)); + let preconditions = store + .last_put_preconditions + .lock() + .expect("preconditions lock poisoned") + .clone() + .expect("existing config update must be conditional"); + assert_eq!(preconditions.if_match_value(), Some("config-1")); + assert_eq!(preconditions.if_none_match_value(), None); assert_eq!( store.lock_resources.lock().expect("lock resources mutex poisoned").as_slice(), - &[server_config_path()] + &[server_config_transaction_lock_path()] ); let decoded = read_config_without_migrate(store) .await @@ -4301,6 +4822,73 @@ mod tests { ); } + #[tokio::test] + async fn server_config_snapshot_save_returns_committed_generation() { + let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + let snapshot = read_server_config_snapshot(store.clone()) + .await + .expect("server config snapshot"); + + let result = save_server_config_snapshot_with_generation(store, &config_with_scanner_cycle("61"), &snapshot) + .await + .expect("conditional config save"); + + assert!(result.persisted()); + assert_eq!(result.generation(), Some(uuid::Uuid::from_u128(2))); + } + + #[tokio::test] + async fn missing_server_config_is_created_with_if_none_match() { + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None)); + let cfg = config_with_scanner_cycle("61"); + + save_server_config(store.clone(), &cfg) + .await + .expect("missing config should be created conditionally"); + + assert_eq!(store.write_calls.load(Ordering::SeqCst), 1); + assert!(!store.last_put_no_lock.load(Ordering::SeqCst)); + let preconditions = store + .last_put_preconditions + .lock() + .expect("preconditions lock poisoned") + .clone() + .expect("missing config create must be conditional"); + assert_eq!(preconditions.if_match_value(), None); + assert_eq!(preconditions.if_none_match_value(), Some("*")); + let persisted = read_config_without_migrate(store) + .await + .expect("created config should reload"); + assert_eq!( + persisted + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("persisted scanner config") + .get(SCANNER_CYCLE), + "61" + ); + } + + #[tokio::test] + async fn missing_config_initialization_recheck_preserves_concurrent_config() { + let existing = config_with_scanner_cycle("73"); + let baseline = encode_server_config_blob(&existing, None).expect("existing config should encode"); + let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None)); + + let observed = new_and_save_server_config(store.clone()) + .await + .expect("initialization recheck should return the config created by another writer"); + + assert_eq!(store.write_calls.load(Ordering::SeqCst), 0); + assert_eq!( + observed + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("concurrent scanner config") + .get(SCANNER_CYCLE), + "73" + ); + } + #[tokio::test] async fn stale_server_config_snapshot_cannot_overwrite_newer_update() { let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode"); @@ -4357,7 +4945,7 @@ mod tests { let lock = rustfs_lock::NamespaceLock::new("server-config-lease-loss".to_string(), client.clone()); let guard = lock .lock_guard( - rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_path()), + rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_transaction_lock_path()), "server-config-lease-loss", std::time::Duration::from_secs(1), std::time::Duration::from_millis(120), @@ -4372,6 +4960,7 @@ mod tests { raw: Some(baseline.clone()), seed: None, etag: Some("config-0".to_string()), + generation: Some(uuid::Uuid::from_u128(1)), _local_guard: local_guard, _guard: guard, }; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 6ff7488a2..2a6e3895a 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -4714,6 +4714,7 @@ mod tests { use crate::layout::endpoints::SetupType; use crate::object_api::BLOCK_SIZE_V2; use crate::object_api::ObjectInfo; + use crate::set_disk::core::io_primitives::rename_fanout_barrier; use crate::storage_api_contracts::{ heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart, namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _, @@ -9565,6 +9566,15 @@ mod tests { make_local_bucket_test_set_disks_with_drive_count(2).await } + fn assert_exclusive_object_lock_held(set_disks: &SetDisks, bucket: &str, object: &str) { + let lock = set_disks + .local_lock_manager_for_test() + .get_lock_info(&ObjectKey::new(bucket, object)) + .expect("object lock should be visible while rename is paused"); + assert!(matches!(lock.mode, rustfs_lock::LockMode::Exclusive)); + assert_eq!(lock.owner.as_ref(), set_disks.locker_owner.as_str()); + } + async fn make_local_bucket_test_set_disks_with_drive_count(drive_count: usize) -> Arc { let format = FormatV3::new(1, drive_count); let mut endpoints = Vec::new(); @@ -9599,7 +9609,9 @@ mod tests { disks.push(Some(disk)); } - let set_disks = SetDisks::new( + let instance_ctx = Arc::new(InstanceContext::new()); + instance_ctx.update_erasure_type(SetupType::Erasure).await; + let set_disks = SetDisks::new_with_instance_ctx( "test-owner".to_string(), Arc::new(RwLock::new(disks)), drive_count, @@ -9609,6 +9621,7 @@ mod tests { endpoints, format, Vec::new(), + instance_ctx, ) .await; set_disks.set_test_storage_class_config( @@ -10262,6 +10275,216 @@ mod tests { )); } + #[tokio::test] + async fn conditional_replace_holds_object_lock_through_rename() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-conditional-replace-fence"; + let object = "config/conditional-replace.json"; + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let mut initial_reader = PutObjReader::from_vec(b"initial config".to_vec()); + let initial = set_disks + .put_object( + bucket, + object, + &mut initial_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("initial config should be written"); + let initial_etag = initial.etag.expect("initial config should have an ETag"); + + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer_store = set_disks.clone(); + let expected_etag = initial_etag.clone(); + let writer = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"replacement config".to_vec()); + writer_store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + preserve_etag: Some("replacement-etag".to_string()), + http_preconditions: Some(HTTPPreconditions { + if_match: Some(expected_etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + }); + + tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("conditional replace should reach the rename barrier"); + assert_exclusive_object_lock_held(&set_disks, bucket, object); + barrier.release(); + writer + .await + .expect("conditional writer task should finish") + .expect("matching conditional replace should commit"); + assert!( + set_disks + .local_lock_manager_for_test() + .get_lock_info(&ObjectKey::new(bucket, object)) + .is_none(), + "conditional replace should release the object lock after commit" + ); + let contender = set_disks + .new_ns_lock(bucket, object) + .await + .expect("contender namespace lock should be created"); + let contender_guard = contender + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("contender should acquire after conditional replace commits"); + drop(contender_guard); + + let mut stale_reader = PutObjReader::from_vec(b"stale config".to_vec()); + let err = set_disks + .put_object( + bucket, + object, + &mut stale_reader, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_match: Some(initial_etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect_err("the old ETag must fail after the fenced replacement commits"); + assert_eq!(err, StorageError::PreconditionFailed); + } + + #[tokio::test] + async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-write-generation"; + let object = "config/write-generation.json"; + let body = b"identical config body".to_vec(); + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let mut first_reader = PutObjReader::from_vec(body.clone()); + let first = set_disks + .put_object( + bucket, + object, + &mut first_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("first config body should be written"); + let mut second_reader = PutObjReader::from_vec(body); + let second = set_disks + .put_object( + bucket, + object, + &mut second_reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("identical config body should be rewritten"); + + assert_eq!(first.etag, second.etag, "content ETag should expose the ABA collision"); + assert_ne!(first.data_dir, second.data_dir, "each committed body write needs a unique generation"); + assert!(first.data_dir.is_some() && second.data_dir.is_some()); + } + + #[tokio::test] + async fn conditional_create_holds_object_lock_through_rename() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-conditional-create-fence"; + let object = "config/conditional-create.json"; + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + let writer_store = set_disks.clone(); + let writer = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"created config".to_vec()); + writer_store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + }); + + tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("conditional create should reach the rename barrier"); + assert_exclusive_object_lock_held(&set_disks, bucket, object); + barrier.release(); + writer + .await + .expect("conditional writer task should finish") + .expect("first conditional create should commit"); + assert!( + set_disks + .local_lock_manager_for_test() + .get_lock_info(&ObjectKey::new(bucket, object)) + .is_none(), + "conditional create should release the object lock after commit" + ); + let contender = set_disks + .new_ns_lock(bucket, object) + .await + .expect("contender namespace lock should be created"); + let contender_guard = contender + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("contender should acquire after conditional create commits"); + drop(contender_guard); + + let mut duplicate_reader = PutObjReader::from_vec(b"duplicate config".to_vec()); + let err = set_disks + .put_object( + bucket, + object, + &mut duplicate_reader, + &ObjectOptions { + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .expect_err("a second create-only write must not replace the committed config"); + assert_eq!(err, StorageError::PreconditionFailed); + } + #[tokio::test] async fn set_level_if_none_match_fails_closed_without_read_quorum() { let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await; diff --git a/crates/notify/src/config_manager.rs b/crates/notify/src/config_manager.rs index 91df663d0..617a7b186 100644 --- a/crates/notify/src/config_manager.rs +++ b/crates/notify/src/config_manager.rs @@ -19,7 +19,7 @@ use crate::{ resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, - with_notify_server_config_read_lock, with_notify_server_config_write_lock, + with_notify_server_config_read_lock, }; use rustfs_config::notify::{ NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, @@ -41,12 +41,32 @@ enum NotifyConfigStoreError { StorageNotAvailable, Read(String), Save(String), + Converge { persisted: bool, error: String }, +} + +fn notification_convergence_error(persisted: bool, error: impl std::fmt::Display) -> NotificationError { + let durable_state = if persisted { "persisted" } else { "unchanged" }; + NotificationError::Configuration(format!( + "configuration was {durable_state} but notification runtime convergence failed: {error}" + )) +} + +async fn supervise_config_update( + mutation: impl std::future::Future> + Send + 'static, +) -> Result +where + T: Send + 'static, +{ + tokio::spawn(mutation).await.map_err(|error| { + let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" }; + NotifyConfigStoreError::Lock(format!("notify config update task {outcome}")) + })? } async fn update_server_config( - modifier: F, + mut modifier: F, lifecycle: NotifyLifecycleCoordinator, -) -> Result, NotifyConfigStoreError> +) -> Result, NotifyConfigStoreError> where F: FnMut(&mut Config) -> bool + Send + 'static, { @@ -54,51 +74,31 @@ where return Err(NotifyConfigStoreError::StorageNotAvailable); }; - let store_for_read = store.clone(); - let store_for_save = store.clone(); - with_notify_server_config_write_lock(store, move || { - read_modify_write( - modifier, - move || async move { - crate::read_notify_server_config_without_migrate_no_lock(store_for_read) - .await - .map_err(NotifyConfigStoreError::Read) - }, - move |config| async move { - crate::save_notify_server_config_no_lock(store_for_save, &config) - .await - .map_err(NotifyConfigStoreError::Save) - }, - move |config| lifecycle.update_config(config), - ) + supervise_config_update(async move { + let snapshot = crate::read_notify_server_config_snapshot(store.clone()) + .await + .map_err(NotifyConfigStoreError::Read)?; + let mut config = snapshot.config.clone(); + if !modifier(&mut config) { + return Ok(None); + } + + let persisted = crate::save_notify_server_config_snapshot(store.clone(), &config, &snapshot) + .await + .map_err(NotifyConfigStoreError::Save)?; + drop(snapshot); + + let read_store = store.clone(); + with_notify_server_config_read_lock(store, move || async move { + let latest = crate::read_existing_notify_server_config_no_lock(read_store) + .await + .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?; + Ok::<_, NotifyConfigStoreError>(Some((lifecycle.update_config(latest), persisted))) + }) + .await + .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })? }) .await - .map_err(NotifyConfigStoreError::Lock)? -} - -async fn read_modify_write( - mut modifier: F, - read: R, - save: S, - publish: P, -) -> Result, NotifyConfigStoreError> -where - F: FnMut(&mut Config) -> bool, - R: FnOnce() -> RFut, - RFut: std::future::Future>, - S: FnOnce(Config) -> SFut, - SFut: std::future::Future>, - P: FnOnce(Config) -> T, -{ - let mut new_config = read().await?; - - if !modifier(&mut new_config) { - return Ok(None); - } - - save(new_config.clone()).await?; - - Ok(Some(publish(new_config))) } pub(crate) fn notify_configuration_hint() -> String { @@ -345,16 +345,18 @@ impl NotifyConfigManager { if self.lifecycle.state() == NotificationRuntimeState::Terminated { return Err(NotificationError::Initialization("Notification runtime has terminated".to_string())); } - let Some(transition) = update_server_config(modifier, self.lifecycle.clone()) - .await - .map_err(|err| match err { - NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err), - NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable( - "Failed to save target configuration: server storage not initialized".to_string(), - ), - NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err), - NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err), - })? + let Some((transition, persisted)) = + update_server_config(modifier, self.lifecycle.clone()) + .await + .map_err(|err| match err { + NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err), + NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable( + "Failed to save target configuration: server storage not initialized".to_string(), + ), + NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err), + NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err), + NotifyConfigStoreError::Converge { persisted, error } => notification_convergence_error(persisted, error), + })? else { debug!( event = EVENT_NOTIFY_CONFIG_UPDATE, @@ -367,23 +369,53 @@ impl NotifyConfigManager { return Ok(()); }; + let result = if persisted { "updated" } else { "unchanged" }; info!( event = EVENT_NOTIFY_CONFIG_UPDATE, component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, action = "reload_if_changed", - result = "updated", + result, "notify config update" ); - transition.wait().await + transition + .wait() + .await + .map_err(|err| notification_convergence_error(persisted, err)) } } #[cfg(test)] mod tests { - use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem}; + use super::{ + NotifyConfigManager, NotifyConfigStoreError, notification_convergence_error, runtime_target_id_for_subsystem, + supervise_config_update, + }; use crate::rules::RulesMap; use crate::{NotificationError, NotificationRuntimeState}; + + #[tokio::test] + async fn supervised_config_update_survives_waiter_cancellation() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); + + let waiter = tokio::spawn(async move { + supervise_config_update(async move { + let _ = started_tx.send(()); + let _ = release_rx.await; + let _ = completed_tx.send(()); + Ok::<_, NotifyConfigStoreError>(()) + }) + .await + }); + + started_rx.await.expect("config update should start"); + waiter.abort(); + release_tx.send(()).expect("config update should be released"); + let completed = tokio::time::timeout(std::time::Duration::from_secs(30), completed_rx).await; + assert!(matches!(completed, Ok(Ok(()))), "detached config update should complete"); + } use crate::{ integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, @@ -392,14 +424,11 @@ mod tests { NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; - use rustfs_config::server_config::{Config, KVS}; + use rustfs_config::server_config::Config; use rustfs_s3_types::EventName; use rustfs_targets::ReplayWorkerManager; use rustfs_targets::arn::TargetID; - use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }; + use std::sync::Arc; use tokio::sync::{RwLock, Semaphore}; fn build_manager() -> NotifyConfigManager { @@ -463,38 +492,6 @@ mod tests { assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. })); } - #[tokio::test] - async fn read_modify_write_publishes_only_after_save() { - let saved = Arc::new(AtomicBool::new(false)); - let saved_by_writer = saved.clone(); - let observed_by_publisher = saved.clone(); - - read_modify_write( - |config| { - config - .0 - .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string()) - .or_default() - .insert("primary".to_string(), KVS::default()); - true - }, - || async { Ok::<_, NotifyConfigStoreError>(Config::default()) }, - move |_config| async move { - saved_by_writer.store(true, Ordering::Release); - Ok::<_, NotifyConfigStoreError>(()) - }, - move |_config| { - assert!( - observed_by_publisher.load(Ordering::Acquire), - "publication must observe the completed save" - ); - }, - ) - .await - .expect("read-modify-write should succeed") - .expect("changed config should publish"); - } - #[test] fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() { let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary"); @@ -550,4 +547,16 @@ mod tests { assert_eq!(target_id.id, "audittrail"); assert_eq!(target_id.name, "postgres"); } + + #[test] + fn convergence_error_reports_durable_write_state() { + for (persisted, expected) in [(true, "configuration was persisted"), (false, "configuration was unchanged")] { + let error = notification_convergence_error(persisted, "injected failure"); + let NotificationError::Configuration(message) = error else { + panic!("convergence error should be a configuration error"); + }; + assert!(message.contains(expected), "unexpected convergence error: {message}"); + assert!(message.contains("injected failure")); + } + } } diff --git a/crates/notify/src/lib.rs b/crates/notify/src/lib.rs index df93c5a48..dcc421d88 100644 --- a/crates/notify/src/lib.rs +++ b/crates/notify/src/lib.rs @@ -57,7 +57,6 @@ pub use services::NotifyServices; pub use status_view::NotifyStatusView; pub use storage_api::NotifyStore; pub(crate) use storage_api::crate_boundary::{ - read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock, - resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock, - with_notify_server_config_write_lock, + read_existing_notify_server_config_no_lock, read_notify_server_config_snapshot, resolve_notify_object_store_handle, + save_notify_server_config_snapshot, with_notify_server_config_read_lock, }; diff --git a/crates/notify/src/storage_api.rs b/crates/notify/src/storage_api.rs index 9f4a44368..ba1ac98ea 100644 --- a/crates/notify/src/storage_api.rs +++ b/crates/notify/src/storage_api.rs @@ -15,11 +15,10 @@ use std::sync::Arc; use rustfs_ecstore::api::config::com::{ - read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock, read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock, - save_server_config_no_lock as save_notify_server_config_to_backend_no_lock, + read_server_config_snapshot as read_notify_server_config_snapshot_from_backend, + save_server_config_snapshot as save_notify_server_config_snapshot_to_backend, with_server_config_read_lock as with_notify_server_config_read_lock_from_backend, - with_server_config_write_lock as with_notify_server_config_write_lock_from_backend, }; use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend; pub use rustfs_ecstore::api::storage::ECStore as NotifyStore; @@ -28,10 +27,10 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option> { resolve_notify_object_store_handle_from_backend() } -pub(crate) async fn read_notify_server_config_without_migrate_no_lock( - store: Arc, -) -> Result { - read_notify_config_without_migrate_from_backend_no_lock(store) +pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot; + +pub(crate) async fn read_notify_server_config_snapshot(store: Arc) -> Result { + read_notify_server_config_snapshot_from_backend(store) .await .map_err(|err| err.to_string()) } @@ -44,22 +43,12 @@ pub(crate) async fn read_existing_notify_server_config_no_lock( .map_err(|err| err.to_string()) } -pub(crate) async fn save_notify_server_config_no_lock( +pub(crate) async fn save_notify_server_config_snapshot( store: Arc, config: &rustfs_config::server_config::Config, -) -> Result<(), String> { - save_notify_server_config_to_backend_no_lock(store, config) - .await - .map_err(|err| err.to_string()) -} - -pub(crate) async fn with_notify_server_config_write_lock(store: Arc, operation: F) -> Result -where - F: FnOnce() -> Fut + Send + 'static, - Fut: std::future::Future + Send + 'static, - T: Send + 'static, -{ - with_notify_server_config_write_lock_from_backend(store, operation) + snapshot: &NotifyServerConfigSnapshot, +) -> Result { + save_notify_server_config_snapshot_to_backend(store, config, snapshot) .await .map_err(|err| err.to_string()) } @@ -77,8 +66,7 @@ where pub(crate) mod crate_boundary { pub(crate) use super::{ - read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock, - resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock, - with_notify_server_config_write_lock, + read_existing_notify_server_config_no_lock, read_notify_server_config_snapshot, resolve_notify_object_store_handle, + save_notify_server_config_snapshot, with_notify_server_config_read_lock, }; } diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index b4d7e5f9f..1055d3bf3 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -15,13 +15,15 @@ use crate::admin::handlers::supervise_admin_mutation; use crate::admin::handlers::target_descriptor::AdminTargetSpec; use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context}; +use crate::admin::service::config::with_runtime_config_reload_lock; use crate::admin::storage_api::config::{ - read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_server_config_snapshot, + read_admin_config_without_migrate, read_admin_server_config_snapshot, read_existing_admin_server_config_no_lock, + save_admin_server_config_snapshot, with_admin_server_config_read_lock, }; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::server_config::Config; -use s3s::{S3Result, s3_error}; +use s3s::{S3Error, S3Result, s3_error}; use tracing::warn; pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result { @@ -48,6 +50,14 @@ fn has_any_audit_targets(specs: &[AdminTargetSpec], config: &Config) -> bool { }) } +fn audit_config_convergence_error(persisted: bool, error: impl std::fmt::Display) -> S3Error { + if persisted { + s3_error!(InternalError, "audit config persisted but runtime convergence failed: {}", error) + } else { + s3_error!(InternalError, "audit config unchanged but runtime convergence failed: {}", error) + } +} + pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config: Config) -> S3Result<()> { let has_targets = has_any_audit_targets(specs, &config); @@ -107,15 +117,23 @@ where return Ok(()); } - save_admin_server_config_snapshot(store, &config, &snapshot) + let persisted = save_admin_server_config_snapshot(store.clone(), &config, &snapshot) .await - .map(|_| ()) - .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; + .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))? + .persisted(); + drop(snapshot); - // Keep persistence and runtime publication in one detached, serialized - // mutation. Otherwise a cancelled caller or two concurrent updates can - // leave the persisted config and active audit generation disagreeing. - apply_audit_runtime_config(&specs, config).await + let read_store = store.clone(); + with_runtime_config_reload_lock(async move { + let latest = with_admin_server_config_read_lock(store, move || read_existing_admin_server_config_no_lock(read_store)) + .await + .map_err(|e| s3_error!(InternalError, "failed to lock server config for audit reload: {}", e))? + .map_err(|e| s3_error!(InternalError, "failed to read latest server config for audit reload: {}", e))?; + + apply_audit_runtime_config(&specs, latest).await + }) + .await + .map_err(|e| audit_config_convergence_error(persisted, e)) }) .await } @@ -164,3 +182,154 @@ pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsys }) .await } + +#[cfg(test)] +mod tests { + use super::*; + use crate::admin::handlers::target_descriptor::admin_target_spec_from_builtin; + use crate::admin::runtime_sources::{IamInterface, KmsInterface}; + use crate::admin::storage_api::config::save_admin_server_config; + use rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS; + use rustfs_config::server_config::KVS; + use rustfs_config::{ENABLE_KEY, EnableState, SCANNER_CYCLE, SCANNER_SUB_SYS, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR}; + use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; + use rustfs_kms::KmsServiceManager; + use rustfs_targets::catalog::builtin::builtin_audit_target_admin_descriptors; + use std::sync::Arc; + use std::time::Duration; + use tempfile::TempDir; + + struct TestIam; + + impl IamInterface for TestIam { + fn handle(&self) -> Arc> { + unreachable!("audit config tests do not use IAM") + } + + fn is_ready(&self) -> bool { + false + } + } + + struct TestKms; + + impl KmsInterface for TestKms { + fn handle(&self) -> Arc { + Arc::new(KmsServiceManager::new()) + } + } + + fn audit_specs() -> Vec { + builtin_audit_target_admin_descriptors() + .into_iter() + .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) + .collect() + } + + async fn wait_for_persisted_target(store: Arc, subsystem: &str, target: &str) { + tokio::time::timeout(Duration::from_secs(30), async { + let mut poll = tokio::time::interval(Duration::from_millis(10)); + loop { + poll.tick().await; + let config = read_admin_config_without_migrate(store.clone()) + .await + .expect("read persisted server config"); + if config.0.get(subsystem).is_some_and(|targets| targets.contains_key(target)) { + return; + } + } + }) + .await + .expect("config mutation should become durable"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial_test::serial] + async fn audit_reload_reads_latest_durable_config_after_releasing_write_snapshot() { + let temp_dir = TempDir::new().expect("audit config temp dir"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp_dir.path()) + .disk_count(1) + .init_bucket_metadata(false) + .build() + .await; + save_admin_server_config(env.ecstore.clone(), &Config::new()) + .await + .expect("persist baseline server config"); + let context = Arc::new(AppContext::new(env.ecstore.clone(), Arc::new(TestIam), Arc::new(TestKms))); + + let (locked_tx, locked_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let blocker = tokio::spawn(async move { + with_runtime_config_reload_lock(async move { + locked_tx.send(()).expect("signal runtime reload lock acquisition"); + release_rx.await.expect("release runtime reload lock"); + Ok(()) + }) + .await + .expect("runtime reload lock blocker"); + }); + locked_rx.await.expect("runtime reload lock should be held"); + + let older_context = context.clone(); + let older = tokio::spawn(async move { + let specs = audit_specs(); + update_audit_config_and_reload_for_context(Some(older_context.as_ref()), &specs, |config| { + config + .0 + .entry(SCANNER_SUB_SYS.to_string()) + .or_default() + .entry(DEFAULT_DELIMITER.to_string()) + .or_insert_with(KVS::new) + .insert(SCANNER_CYCLE.to_string(), "15s".to_string()); + true + }) + .await + }); + wait_for_persisted_target(env.ecstore.clone(), SCANNER_SUB_SYS, DEFAULT_DELIMITER).await; + assert!(!older.is_finished(), "audit runtime publication must wait for the shared reload lock"); + + let snapshot = read_admin_server_config_snapshot(env.ecstore.clone()) + .await + .expect("read newer server config snapshot"); + let mut latest = snapshot.config.clone(); + let mut latest_target = KVS::new(); + latest_target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + latest_target.insert(WEBHOOK_ENDPOINT.to_string(), "https://audit.invalid/hook".to_string()); + latest_target.insert( + WEBHOOK_QUEUE_DIR.to_string(), + temp_dir.path().join("audit-queue").to_string_lossy().into_owned(), + ); + latest + .0 + .entry(AUDIT_WEBHOOK_SUB_SYS.to_string()) + .or_default() + .insert("latest".to_string(), latest_target); + save_admin_server_config_snapshot(env.ecstore.clone(), &latest, &snapshot) + .await + .expect("persist newer audit config"); + drop(snapshot); + + release_tx.send(()).expect("release runtime reload blocker"); + blocker.await.expect("runtime reload blocker task"); + tokio::time::timeout(Duration::from_secs(30), older) + .await + .expect("older audit update should complete") + .expect("older audit update task should not panic") + .expect("older audit update should converge from the latest durable config"); + + let system = audit_system().expect("latest durable audit target should start the audit system"); + let targets = system.list_targets().await; + assert!(targets.iter().any(|target| target.contains("latest")), "active targets: {targets:?}"); + system.close().await.expect("audit system should stop after the test"); + } + + #[test] + fn audit_convergence_error_reports_durable_write_state() { + for (persisted, expected) in [(true, "audit config persisted"), (false, "audit config unchanged")] { + let error = audit_config_convergence_error(persisted, "injected failure"); + assert!(error.to_string().contains(expected), "unexpected convergence error: {error}"); + assert!(error.to_string().contains("injected failure")); + } + } +} diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index b9d409090..83641b017 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -17,18 +17,17 @@ use crate::admin::handlers::supervise_admin_mutation; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ current_action_credentials, current_app_context, current_object_store_handle_for_context, current_server_config_for_context, - publish_server_config, }; use crate::admin::service::config::{ - CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN, - LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, is_dynamic_config_subsystem, preflight_dynamic_config_reload, - prepare_server_config, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, signal_config_snapshot_reload, - signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked, + FULL_CONFIG_WORKER_SUBSYSTEMS, is_dynamic_config_subsystem, preflight_dynamic_config_reload, prepare_server_config, + publish_latest_runtime_config_snapshot, reload_dynamic_config_runtime_state, reload_runtime_config_snapshot, + signal_config_snapshot_reload, signal_config_snapshot_reload_checked, signal_dynamic_config_reload_checked, }; use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use crate::admin::storage_api::config::{ - AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, - read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, save_admin_server_config_snapshot, + AdminServerConfigSaveResult, AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, + read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, + save_admin_server_config_snapshot, }; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; @@ -766,7 +765,10 @@ async fn load_server_config_snapshot_from_store() -> S3Result S3Result { +async fn save_server_config_to_store( + config: &ServerConfig, + snapshot: &AdminServerConfigSnapshot, +) -> S3Result { let store = object_store()?; save_admin_server_config_snapshot(store, config, snapshot) .await @@ -987,8 +989,8 @@ fn decode_config_history_snapshot(data: &[u8]) -> S3Result { Ok(config) } -fn validate_restore_rollback_generation(current: &ServerConfig, restored: &ServerConfig) -> S3Result<()> { - if current == restored { +fn validate_restore_rollback_generation(current: Option, committed: Option) -> S3Result<()> { + if current.is_some() && current == committed { Ok(()) } else { Err(s3_error!( @@ -1004,12 +1006,12 @@ async fn save_server_config_history_snapshot(config: &ServerConfig) -> S3Result< save_server_config_history(&sealed).await } -async fn cleanup_failed_config_history_snapshot(restore_id: &str) { +async fn cleanup_config_history_snapshot(restore_id: &str) { if let Err(err) = delete_server_config_history(restore_id).await { warn!( restore_id, error = %err, - "Failed to remove config history snapshot for an uncommitted mutation" + "Failed to remove config history snapshot" ); } } @@ -1843,23 +1845,6 @@ async fn reject_lost_config_transaction(snapshot: AdminServerConfigSnapshot, )) } -fn publish_prepared_config_snapshots(config: ServerConfig, prepared: PreparedRuntimeConfig) -> S3Result<()> { - prepared.publish_storage_class()?; - publish_server_config(config); - Ok(()) -} - -/// Re-apply local mutable worker families after a full-config replacement. -/// Peers receive one full-snapshot signal after this returns; signaling each -/// family here as well would recreate audit/scanner targets twice per peer. -fn publish_notify_config_intent( - config: &ServerConfig, - sub_system: Option<&str>, -) -> Option { - (sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system))) - .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone())) -} - fn config_preflight_subsystems(sub_system: Option<&str>) -> Vec<&str> { if let Some(sub_system) = sub_system { return is_dynamic_config_subsystem(sub_system) @@ -1884,78 +1869,26 @@ async fn preflight_config_intent(sub_system: Option<&str>) -> S3Result<()> { Ok(()) } -async fn wait_notify_config_intent(transition: Option) -> S3Result { - let Some(transition) = transition else { - return Ok(false); - }; - transition.wait().await.map_err(|err| { - warn!(error = %err, "Failed to apply local notification config"); - s3_error!(InternalError, "failed to apply notification config") - })?; - Ok(true) -} - -async fn reload_non_notify_dynamic_subsystems() -> Vec { - let mut failures = Vec::new(); - for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { - if NOTIFY_SUB_SYSTEMS.contains(&sub_system) { - continue; - } - if reload_dynamic_config_runtime_state(sub_system).await.is_err() { - failures.push(format!("local {sub_system}")); - warn!( - event = EVENT_CONFIG_WORKER_RELOAD_FAILED, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_CONFIG, - config_subsystem = sub_system, - state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - reason = "apply_failed", - "Published server config but failed to reload a local worker subsystem" - ); - } - } - failures -} - -fn finish_config_reconciliation(errors: Vec) -> S3Result<()> { +fn finish_config_reconciliation(errors: Vec, persisted: bool) -> S3Result<()> { if errors.is_empty() { Ok(()) - } else { + } else if persisted { Err(s3_error!( InternalError, "server config persisted but runtime convergence failed: {}", errors.join("; ") )) + } else { + Err(s3_error!( + InternalError, + "server config was unchanged but runtime convergence failed: {}", + errors.join("; ") + )) } } -async fn reconcile_targeted_config( - sub_system: Option, - storage_class_applied: bool, - notify_transition: Option, -) -> S3Result { - let mut errors = Vec::new(); - let notify_applied = notify_transition.is_some(); - if let Err(err) = wait_notify_config_intent(notify_transition).await { - warn!(error = %err, "Local notification config failed to converge"); - errors.push("local notify".to_string()); - } - - let config_applied = if notify_applied { - if let Some(sub_system) = sub_system.as_deref() - && let Err(err) = signal_dynamic_config_reload_checked(sub_system).await - { - warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed"); - errors.push(format!("peer {sub_system}")); - } - true - } else if storage_class_applied { - if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await { - warn!(error = %err, "Peer storage-class reload failed"); - errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); - } - true - } else if let Some(sub_system) = sub_system.as_deref() +async fn reconcile_targeted_config(sub_system: Option, persisted: bool, mut errors: Vec) -> S3Result { + let config_applied = if let Some(sub_system) = sub_system.as_deref() && is_dynamic_config_subsystem(sub_system) { let config_applied = match reload_dynamic_config_runtime_state(sub_system).await { @@ -1979,21 +1912,19 @@ async fn reconcile_targeted_config( false }; - finish_config_reconciliation(errors)?; + finish_config_reconciliation(errors, persisted)?; Ok(config_applied) } -async fn reconcile_full_config(notify_transition: Option) -> S3Result<()> { - let mut errors = Vec::new(); - if let Err(err) = wait_notify_config_intent(notify_transition).await { - warn!(error = %err, "Local notification config failed to converge"); +async fn reconcile_full_config(persisted: bool, mut errors: Vec) -> S3Result<()> { + if let Err(err) = reload_dynamic_config_runtime_state(NOTIFY_WEBHOOK_SUB_SYS).await { + warn!(error = %err, "Local notification config failed to converge from durable config"); errors.push("local notify".to_string()); } if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await { warn!(error = %err, "Peer storage-class reload failed"); errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); } - errors.extend(reload_non_notify_dynamic_subsystems().await); if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await { warn!(error = %err, "Peer notification config reload failed"); errors.push("peer notify".to_string()); @@ -2002,91 +1933,82 @@ async fn reconcile_full_config(notify_transition: Option, - storage_class_applied: bool, - notify_transition: Option, + persisted: bool, + committed_generation: Option, } async fn persist_server_config_transaction( config: ServerConfig, - prepared: PreparedRuntimeConfig, snapshot: AdminServerConfigSnapshot, - sub_system: Option<&str>, ) -> S3Result { snapshot.ensure_lock_held().map_err(ApiError::from).map_err(S3Error::from)?; let previous_config = snapshot.config.clone(); let history_restore_id = save_server_config_history_snapshot(&previous_config).await?; if snapshot.is_lock_lost() { - cleanup_failed_config_history_snapshot(&history_restore_id).await; + cleanup_config_history_snapshot(&history_restore_id).await; return reject_lost_config_transaction(snapshot, "before persistence").await; } - let persisted = match save_server_config_to_store(&config, &snapshot).await { - Ok(persisted) => persisted, + let save_result = match save_server_config_to_store(&config, &snapshot).await { + Ok(result) => result, Err(err) => { - cleanup_failed_config_history_snapshot(&history_restore_id).await; + cleanup_config_history_snapshot(&history_restore_id).await; return Err(err); } }; + let persisted = save_result.persisted(); + let committed_generation = save_result.generation(); let history_restore_id = if persisted { Some(history_restore_id) } else { - cleanup_failed_config_history_snapshot(&history_restore_id).await; + cleanup_config_history_snapshot(&history_restore_id).await; None }; - if snapshot.is_lock_lost() { - return reject_lost_config_transaction(snapshot, "after persistence").await; - } - - let storage_class_applied = sub_system.is_none_or(|value| value == STORAGE_CLASS_SUB_SYS); - if storage_class_applied { - if let Err(err) = publish_prepared_config_snapshots(config.clone(), prepared) { - return Err(match history_restore_id.as_deref() { - Some(restore_id) => s3_error!( - InternalError, - "config persisted but runtime publish failed; recovery snapshot restoreId={}: {}", - restore_id, - err - ), - None => err, - }); - } - } else { - publish_server_config(config.clone()); - } - let notify_transition = publish_notify_config_intent(&config, sub_system); - if snapshot.is_lock_lost() { - return reject_lost_config_transaction(snapshot, "while publishing runtime snapshots").await; - } - drop(snapshot); Ok(PersistedConfigTransaction { previous_config, history_restore_id, - storage_class_applied, - notify_transition, + persisted, + committed_generation, }) } +async fn reconcile_committed_config(sub_system: Option, persisted: bool) -> S3Result { + let mut errors = Vec::new(); + let publication_result = match sub_system.as_deref() { + Some(sub_system) => publish_latest_runtime_config_snapshot(sub_system).await, + None => reload_runtime_config_snapshot().await, + }; + if let Err(err) = publication_result { + warn!(error = %err, "Failed to publish the latest durable server config"); + errors.push("local config snapshot".to_string()); + } + + if sub_system.is_none() { + reconcile_full_config(persisted, errors).await?; + return Ok(false); + } + + reconcile_targeted_config(sub_system, persisted, errors).await +} + async fn commit_server_config_transaction( config: ServerConfig, - prepared: PreparedRuntimeConfig, snapshot: AdminServerConfigSnapshot, sub_system: Option, ) -> S3Result { - let transaction = persist_server_config_transaction(config, prepared, snapshot, sub_system.as_deref()).await?; - - if sub_system.is_none() { - reconcile_full_config(transaction.notify_transition).await?; - return Ok(false); + let transaction = persist_server_config_transaction(config, snapshot).await?; + let result = reconcile_committed_config(sub_system, transaction.persisted).await; + match (result, transaction.history_restore_id.as_deref()) { + (Err(err), Some(restore_id)) => Err(s3_error!(InternalError, "{}; recovery snapshot restoreId={}", err, restore_id)), + (result, _) => result, } - - reconcile_targeted_config(sub_system, transaction.storage_class_applied, transaction.notify_transition).await } pub struct GetConfigKVHandler {} @@ -2127,8 +2049,8 @@ impl Operation for SetConfigKVHandler { let snapshot = load_server_config_snapshot_from_store().await?; let mut config = snapshot.config.clone(); apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, sub_system.as_deref()).await?; - commit_server_config_transaction(config, prepared, snapshot, sub_system).await + prepare_server_config(&config, sub_system.as_deref()).await?; + commit_server_config_transaction(config, snapshot, sub_system).await }) .await?; @@ -2155,8 +2077,8 @@ impl Operation for DelConfigKVHandler { let snapshot = load_server_config_snapshot_from_store().await?; let mut config = snapshot.config.clone(); apply_delete_directives(&mut config, &directives); - let prepared = prepare_server_config(&config, sub_system.as_deref()).await?; - commit_server_config_transaction(config, prepared, snapshot, sub_system).await + prepare_server_config(&config, sub_system.as_deref()).await?; + commit_server_config_transaction(config, snapshot, sub_system).await }) .await?; @@ -2239,15 +2161,19 @@ impl Operation for RestoreConfigHistoryKVHandler { supervise_admin_mutation("config mutation", async move { preflight_config_intent(None).await?; let snapshot = load_server_config_snapshot_from_store().await?; - let prepared = prepare_server_config(&config, None).await?; - let restored_config = config.clone(); - let transaction = persist_server_config_transaction(config, prepared, snapshot, None).await?; - let Err(restore_error) = reconcile_full_config(transaction.notify_transition).await else { + prepare_server_config(&config, None).await?; + let transaction = persist_server_config_transaction(config, snapshot).await?; + let Err(restore_error) = reconcile_committed_config(None, transaction.persisted).await else { return Ok(()); }; + if !transaction.persisted { + return Err(restore_error); + } + let previous_config = transaction.previous_config; let recovery_restore_id = transaction.history_restore_id; + let committed_generation = transaction.committed_generation; let recovery_reference = recovery_restore_id.as_deref().unwrap_or("not-created"); let rollback_snapshot = match load_server_config_snapshot_from_store().await { Ok(snapshot) => snapshot, @@ -2261,7 +2187,9 @@ impl Operation for RestoreConfigHistoryKVHandler { )); } }; - if let Err(rollback_error) = validate_restore_rollback_generation(&rollback_snapshot.config, &restored_config) { + if let Err(rollback_error) = + validate_restore_rollback_generation(rollback_snapshot.generation(), committed_generation) + { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", @@ -2271,12 +2199,20 @@ impl Operation for RestoreConfigHistoryKVHandler { )); } - let rollback_prepared = prepare_server_config(&previous_config, None).await?; - rollback_snapshot + if let Err(rollback_error) = prepare_server_config(&previous_config, None).await { + return Err(s3_error!( + InternalError, + "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", + restore_error, + rollback_error, + recovery_reference + )); + } + if let Err(rollback_error) = rollback_snapshot .ensure_lock_held() .map_err(ApiError::from) - .map_err(S3Error::from)?; - if let Err(rollback_error) = save_server_config_to_store(&previous_config, &rollback_snapshot).await { + .map_err(S3Error::from) + { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", @@ -2285,10 +2221,9 @@ impl Operation for RestoreConfigHistoryKVHandler { recovery_reference )); } - if rollback_snapshot.is_lock_lost() { - if let Err(rollback_error) = - reject_lost_config_transaction::<()>(rollback_snapshot, "after restore rollback persistence").await - { + let rollback_persisted = match save_server_config_to_store(&previous_config, &rollback_snapshot).await { + Ok(result) => result.persisted(), + Err(rollback_error) => { return Err(s3_error!( InternalError, "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", @@ -2297,46 +2232,20 @@ impl Operation for RestoreConfigHistoryKVHandler { recovery_reference )); } - return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly")); - } - - if let Err(rollback_error) = publish_prepared_config_snapshots(previous_config.clone(), rollback_prepared) { - return Err(s3_error!( - InternalError, - "config restore failed: {}; automatic rollback publish failed: {}; recovery snapshot restoreId={}", - restore_error, - rollback_error, - recovery_reference - )); - } - let rollback_transition = publish_notify_config_intent(&previous_config, None); - if rollback_snapshot.is_lock_lost() { - if let Err(rollback_error) = - reject_lost_config_transaction::<()>(rollback_snapshot, "while publishing restore rollback").await - { - return Err(s3_error!( - InternalError, - "config restore failed: {}; automatic rollback failed: {}; recovery snapshot restoreId={}", - restore_error, - rollback_error, - recovery_reference - )); - } - return Err(s3_error!(InternalError, "restore rollback lock-loss handling returned unexpectedly")); - } + }; drop(rollback_snapshot); - if let Err(rollback_error) = reconcile_full_config(rollback_transition).await { + if let Err(rollback_error) = reconcile_committed_config(None, rollback_persisted).await { return Err(s3_error!( InternalError, - "config restore failed: {}; persisted rollback did not converge: {}; recovery snapshot restoreId={}", + "config restore failed: {}; automatic rollback convergence failed: {}; recovery snapshot restoreId={}", restore_error, rollback_error, recovery_reference )); } - if let Some(recovery_restore_id) = recovery_restore_id.as_deref() { - cleanup_failed_config_history_snapshot(recovery_restore_id).await; + if rollback_persisted && let Some(recovery_restore_id) = recovery_restore_id.as_deref() { + cleanup_config_history_snapshot(recovery_restore_id).await; } Err(s3_error!(InternalError, "config restore failed and was rolled back: {}", restore_error)) }) @@ -2377,8 +2286,8 @@ impl Operation for SetConfigHandler { let snapshot = load_server_config_snapshot_from_store().await?; let mut config = ServerConfig::new(); apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, None).await?; - commit_server_config_transaction(config, prepared, snapshot, None).await?; + prepare_server_config(&config, None).await?; + commit_server_config_transaction(config, snapshot, None).await?; Ok(()) }) .await?; @@ -2408,6 +2317,23 @@ mod tests { ); } + #[test] + fn reconciliation_error_reports_whether_config_was_persisted() { + let persisted = finish_config_reconciliation(vec!["local config snapshot".to_string()], true) + .expect_err("committed config convergence failure must be reported"); + let unchanged = finish_config_reconciliation(vec!["local config snapshot".to_string()], false) + .expect_err("unchanged config convergence failure must be reported"); + + assert_eq!( + persisted.message(), + Some("server config persisted but runtime convergence failed: local config snapshot") + ); + assert_eq!( + unchanged.message(), + Some("server config was unchanged but runtime convergence failed: local config snapshot") + ); + } + #[test] fn tokenize_config_line_handles_quotes_and_escapes() { let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#) @@ -3191,23 +3117,32 @@ notify_webhook:secondary endpoint="https://secondary.example" auth_token="second } #[test] - fn restore_rollback_generation_rejects_concurrent_config_change() { - crate::admin::storage_api::config::init_admin_config_defaults(); - let restored = ServerConfig::new(); - let mut concurrent = restored.clone(); - apply_set_directives( - &mut concurrent, - &parse_config_directives(r#"identity_openid client_id="concurrent-client""#, false).expect("parse concurrent"), - ) - .expect("apply concurrent"); + fn restore_rollback_generation_accepts_committed_write_identity() { + let committed = Uuid::from_u128(1); + validate_restore_rollback_generation(Some(committed), Some(committed)).expect("matching committed generation"); + } - validate_restore_rollback_generation(&restored, &restored).expect("unchanged restore generation"); - let error = validate_restore_rollback_generation(&concurrent, &restored) - .expect_err("concurrent change must fence automatic rollback"); + #[test] + fn restore_rollback_generation_rejects_aba_with_matching_config_content() { + let error = validate_restore_rollback_generation(Some(Uuid::from_u128(2)), Some(Uuid::from_u128(1))) + .expect_err("a later generation must fence automatic rollback even when config content matches"); assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); assert!(error.to_string().contains("concurrent configuration change")); } + #[test] + fn restore_rollback_generation_rejects_missing_write_identity() { + for (current, committed) in [ + (None, Some(Uuid::from_u128(1))), + (Some(Uuid::from_u128(1)), None), + (None, None), + ] { + let error = validate_restore_rollback_generation(current, committed) + .expect_err("missing generation metadata must fence automatic rollback"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequest); + } + } + #[test] fn legacy_directive_history_is_rejected_without_being_applied_as_a_snapshot() { let error = decode_config_history_snapshot(b"identity_openid client_id=\"legacy\"") diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index 43e500d7c..c406dfafa 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -16,7 +16,9 @@ use crate::admin::runtime_sources::{ AppContext, current_app_context, current_notification_system_for_context, current_object_store_handle_for_context, publish_server_config, publish_storage_class_config, }; -use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass}; +use crate::admin::storage_api::config::{ + STORAGE_CLASS_SUB_SYS, read_existing_admin_server_config_no_lock, storageclass, with_admin_server_config_read_lock, +}; use crate::admin::storage_api::contract::admin::StorageAdminApi; use crate::admin::storage_api::runtime::ECStore; use crate::server::{ @@ -43,6 +45,25 @@ use url::Url; static RUNTIME_CONFIG_RELOAD_MUTEX: AsyncMutex<()> = AsyncMutex::const_new(()); +// Runtime publication lock order: reload mutex -> server-config local lock -> +// transaction lock -> config object lock. +pub(crate) async fn with_runtime_config_reload_lock(operation: Fut) -> S3Result +where + Fut: Future> + Send + 'static, + T: Send + 'static, +{ + let reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; + tokio::spawn(async move { + let _reload_guard = reload_guard; + operation.await + }) + .await + .map_err(|err| { + let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" }; + internal_error(format!("runtime config reload task {outcome}")) + })? +} + pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool { NOTIFY_SUB_SYSTEMS.contains(&sub_system) || matches!( @@ -121,11 +142,6 @@ impl PreparedRuntimeConfig { fn publish_storage_class_for_context(self, context: Option<&AppContext>) -> S3Result<()> { self.publish_storage_class_for_context_with(context, publish_storage_class_config) } - - pub(crate) fn publish_storage_class(self) -> S3Result<()> { - let context = current_app_context(); - self.publish_storage_class_for_context(context.as_deref()) - } } fn publish_server_config_for_context(context: Option<&AppContext>, config: ServerConfig) { @@ -396,7 +412,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste } pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> { - let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; + let context = context.cloned(); + let sub_system = sub_system.to_owned(); + with_runtime_config_reload_lock(async move { + reload_dynamic_config_runtime_state_under_reload_lock_for_context(context.as_ref(), &sub_system).await + }) + .await +} + +async fn reload_dynamic_config_runtime_state_under_reload_lock_for_context( + context: Option<&AppContext>, + sub_system: &str, +) -> S3Result<()> { if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM { let store = resolve_runtime_config_store_for_context(context)?; let notify_result = reconcile_event_notifier_from_store(store).await; @@ -414,18 +441,54 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap } let store = resolve_runtime_config_store_for_context(context)?; - let config = read_admin_config_without_migrate(store).await.map_err(|err| { - warn!("peer reload_dynamic_config: failed to load server config for {sub_system}: {err}"); - internal_error(format!("failed to load server config: {err}")) - })?; + let read_store = store.clone(); + let publication_context = context.cloned(); + let publication_sub_system = sub_system.to_owned(); + let notify_config = with_admin_server_config_read_lock(store, move || async move { + let config = read_existing_admin_server_config_no_lock(read_store).await.map_err(|err| { + warn!("peer reload_dynamic_config: failed to load server config for {publication_sub_system}: {err}"); + internal_error(format!("failed to load server config: {err}")) + })?; + let prepared = prepare_server_config_for_context(publication_context.as_ref(), &config, Some(&publication_sub_system)) + .await + .map_err(|err| { + if publication_sub_system == STORAGE_CLASS_SUB_SYS { + internal_error(format!("failed to apply storage class config: {err}")) + } else { + err + } + })?; - if matches!(sub_system, SCANNER_SUB_SYS | HEAL_SUB_SYS) { - validate_server_config_for_context(context, &config, Some(sub_system)).await?; - // Scanner cycles refresh from the process-wide server config before - // each pass. Publish the same validated snapshot first so that refresh - // cannot overwrite this peer's dynamic scanner update with stale data. - publish_server_config_for_context(context, config.clone()); - } + if publication_sub_system == STORAGE_CLASS_SUB_SYS { + prepared.publish_storage_class_for_context(publication_context.as_ref())?; + return Ok::, S3Error>(None); + } + if NOTIFY_SUB_SYSTEMS.contains(&publication_sub_system.as_str()) { + return Ok(Some(config)); + } + if matches!(publication_sub_system.as_str(), SCANNER_SUB_SYS | HEAL_SUB_SYS) { + publish_server_config_for_context(publication_context.as_ref(), config.clone()); + } + apply_dynamic_config_for_subsystem_for_context(publication_context.as_ref(), &config, &publication_sub_system) + .await + .inspect_err(|_| { + warn!( + config_subsystem = publication_sub_system, + reason = "apply_failed", + "Peer dynamic config apply failed" + ); + })?; + Ok(None) + }) + .await + .map_err(|err| { + warn!("peer reload_dynamic_config: failed to acquire server config publication fence for {sub_system}: {err}"); + internal_error(format!("failed to lock server config: {err}")) + })??; + + let Some(config) = notify_config else { + return Ok(()); + }; apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) .await .inspect_err(|_| { @@ -439,23 +502,16 @@ pub async fn reload_dynamic_config_runtime_state(sub_system: &str) -> S3Result<( reload_dynamic_config_runtime_state_for_context(context.as_deref(), sub_system).await } -async fn reload_runtime_config_snapshot_with( - read: ReadFuture, - prepare: Prepare, - publish: Publish, +async fn reload_runtime_config_snapshot_with( + publish_snapshot: PublishFuture, apply_workers: ApplyWorkers, ) -> S3Result<()> where - ReadFuture: Future>, - Prepare: FnOnce(ServerConfig) -> PrepareFuture, - PrepareFuture: Future>, - Publish: FnOnce(&ServerConfig, PreparedRuntimeConfig) -> S3Result<()>, + PublishFuture: Future>, ApplyWorkers: FnOnce(ServerConfig) -> ApplyWorkersFuture, ApplyWorkersFuture: Future>, { - let config = read.await?; - let (config, prepared) = prepare(config).await?; - publish(&config, prepared)?; + let config = publish_snapshot.await?; // Worker reloads mutate live state and have no rollback contract, so the // validated snapshots stay published. The RPC still reports convergence @@ -474,55 +530,105 @@ where Ok(()) } -pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> { - let _reload_guard = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; +async fn publish_latest_runtime_config_snapshot_under_reload_lock_for_context( + context: Option<&AppContext>, + sub_system: Option<&str>, + apply_workers: ApplyWorkers, +) -> S3Result<()> +where + ApplyWorkers: FnOnce(ServerConfig) -> ApplyWorkersFuture + Send + 'static, + ApplyWorkersFuture: Future> + Send + 'static, +{ let store = resolve_runtime_config_store_for_context(context)?; + let read_store = store.clone(); + let publication_context = context.cloned(); + let publication_sub_system = sub_system.map(str::to_owned); - reload_runtime_config_snapshot_with( - async move { - read_admin_config_without_migrate(store).await.map_err(|err| { - warn!("peer reload_runtime_config_snapshot: failed to load server config: {err}"); - internal_error(format!("failed to load server config: {err}")) - }) - }, - |config| async move { - let prepared = prepare_server_config_for_context(context, &config, None).await.map_err(|_| { - warn!("peer reload_runtime_config_snapshot: failed to prepare server config"); - internal_error("failed to prepare server config") - })?; - Ok((config, prepared)) - }, - |config, prepared| { - prepared.publish_storage_class_for_context(context)?; - publish_server_config_for_context(context, config.clone()); - Ok(()) - }, - |config| async move { - let mut failures = Vec::new(); - for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { - if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) - .await - .is_err() + with_admin_server_config_read_lock(store, move || async move { + let config = read_existing_admin_server_config_no_lock(read_store).await.map_err(|err| { + warn!("runtime config publication failed to load server config: {err}"); + internal_error(format!("failed to load server config: {err}")) + })?; + let prepared = + prepare_server_config_for_context(publication_context.as_ref(), &config, publication_sub_system.as_deref()) + .await + .map_err(|err| match publication_sub_system.as_deref() { + None => { + warn!("peer reload_runtime_config_snapshot: failed to prepare server config"); + internal_error("failed to prepare server config") + } + Some(STORAGE_CLASS_SUB_SYS) => internal_error(format!("failed to apply storage class config: {err}")), + Some(_) => err, + })?; + reload_runtime_config_snapshot_with( + async move { + if publication_sub_system + .as_deref() + .is_none_or(|sub_system| sub_system == STORAGE_CLASS_SUB_SYS) { - failures.push(sub_system); - warn!( - event = EVENT_CONFIG_WORKER_RELOAD_FAILED, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_CONFIG, - config_subsystem = sub_system, - state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - reason = "apply_failed", - "Peer runtime config snapshot was published but a subsystem worker reload failed" - ); + prepared.publish_storage_class_for_context(publication_context.as_ref())?; } + publish_server_config_for_context(publication_context.as_ref(), config.clone()); + Ok(config) + }, + apply_workers, + ) + .await + }) + .await + .map_err(|err| { + warn!("runtime config publication failed to acquire server config publication fence: {err}"); + internal_error(format!("failed to lock server config: {err}")) + })? +} + +pub(crate) async fn publish_latest_runtime_config_snapshot(sub_system: &str) -> S3Result<()> { + let context = current_app_context(); + let sub_system = sub_system.to_owned(); + with_runtime_config_reload_lock(async move { + publish_latest_runtime_config_snapshot_under_reload_lock_for_context(context.as_deref(), Some(&sub_system), |_| async { + Ok(()) + }) + .await + }) + .await +} + +pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppContext>) -> S3Result<()> { + let context = context.cloned(); + with_runtime_config_reload_lock(async move { + reload_runtime_config_snapshot_under_reload_lock_for_context(context.as_ref()).await + }) + .await +} + +async fn reload_runtime_config_snapshot_under_reload_lock_for_context(context: Option<&AppContext>) -> S3Result<()> { + let worker_context = context.cloned(); + publish_latest_runtime_config_snapshot_under_reload_lock_for_context(context, None, move |config| async move { + let mut failures = Vec::new(); + for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { + if apply_dynamic_config_for_subsystem_for_context(worker_context.as_ref(), &config, sub_system) + .await + .is_err() + { + failures.push(sub_system); + warn!( + event = EVENT_CONFIG_WORKER_RELOAD_FAILED, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_CONFIG, + config_subsystem = sub_system, + state = CONFIG_WORKER_RELOAD_FAILURE_STATE, + reason = "apply_failed", + "Peer runtime config snapshot was published but a subsystem worker reload failed" + ); } - if failures.is_empty() { - Ok(()) - } else { - Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; ")))) - } - }, - ) + } + if failures.is_empty() { + Ok(()) + } else { + Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; ")))) + } + }) .await } @@ -699,6 +805,77 @@ mod tests { } } + #[tokio::test] + async fn runtime_reload_waiter_cancellation_does_not_leave_queued_work() { + let _blocker = RUNTIME_CONFIG_RELOAD_MUTEX.lock().await; + let (operation_started_tx, mut operation_started_rx) = tokio::sync::oneshot::channel(); + let mut reload = Box::pin(with_runtime_config_reload_lock(async move { + let _ = operation_started_tx.send(()); + Ok(()) + })); + + poll_fn(|cx| match reload.as_mut().poll(cx) { + Poll::Pending => Poll::Ready(()), + Poll::Ready(_) => panic!("reload must wait while the mutex is held"), + }) + .await; + drop(reload); + + assert!( + matches!(operation_started_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Closed)), + "cancelling a queued reload must drop its work instead of detaching it" + ); + } + + #[tokio::test] + async fn runtime_reload_lock_survives_waiter_cancellation() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); + + let waiter = tokio::spawn(async move { + with_runtime_config_reload_lock(async move { + started_tx.send(()).expect("signal runtime reload start"); + release_rx.await.expect("release supervised runtime reload"); + completed_tx.send(()).expect("signal runtime reload completion"); + Ok(()) + }) + .await + }); + + started_rx.await.expect("supervised runtime reload started"); + waiter.abort(); + assert!(waiter.await.expect_err("reload waiter should be cancelled").is_cancelled()); + + let (contender_polled_tx, contender_polled_rx) = tokio::sync::oneshot::channel(); + let (contender_entered_tx, mut contender_entered_rx) = tokio::sync::oneshot::channel(); + let contender = tokio::spawn(async move { + let mut lock = Box::pin(RUNTIME_CONFIG_RELOAD_MUTEX.lock()); + let mut contender_polled_tx = Some(contender_polled_tx); + let _guard = poll_fn(|cx| { + if let Some(tx) = contender_polled_tx.take() { + let _ = tx.send(()); + } + lock.as_mut().poll(cx) + }) + .await; + let _ = contender_entered_tx.send(()); + }); + + contender_polled_rx.await.expect("contending reload should be polled"); + assert!( + matches!(contender_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)), + "a cancelled waiter must not release the reload mutex while its detached reload is still running" + ); + + release_tx.send(()).expect("release detached runtime reload"); + completed_rx.await.expect("detached runtime reload should complete"); + contender_entered_rx + .await + .expect("contending reload should enter after completion"); + contender.await.expect("contending reload task should not panic"); + } + #[tokio::test] async fn checked_scanner_reload_reports_unreachable_peer() { let temp_dir = TempDir::new().expect("scanner reload temp dir"); @@ -1221,6 +1398,123 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial_test::serial(storage_class_env)] + async fn full_reload_keeps_worker_apply_inside_durable_read_fence() { + temp_env::async_with_vars( + [ + (storageclass::STANDARD_ENV, None::<&str>), + (storageclass::RRS_ENV, None::<&str>), + (storageclass::OPTIMIZE_ENV, None::<&str>), + (storageclass::INLINE_BLOCK_ENV, None::<&str>), + ], + async { + let fixture = runtime_config_reload_fixture().await; + let older = scanner_server_config("41"); + save_admin_server_config(fixture.context.object_store(), &older) + .await + .expect("persist older scanner config"); + + let (worker_entered_tx, worker_entered_rx) = tokio::sync::oneshot::channel(); + let (release_worker_tx, release_worker_rx) = tokio::sync::oneshot::channel(); + let reload_context = fixture.context.clone(); + let reload = tokio::spawn(async move { + publish_latest_runtime_config_snapshot_under_reload_lock_for_context( + Some(&reload_context), + None, + move |config| async move { + worker_entered_tx.send(config).expect("signal runtime config worker entry"); + release_worker_rx.await.expect("release runtime config worker"); + Ok(()) + }, + ) + .await + }); + + let worker_config = tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, worker_entered_rx) + .await + .expect("worker apply should enter") + .expect("worker apply entry signal should be delivered"); + assert_eq!( + worker_config + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("older scanner config should be loaded") + .get(SCANNER_CYCLE), + "41" + ); + + let latest = scanner_server_config("71"); + let writer_store = fixture.context.object_store(); + let (writer_polled_tx, writer_polled_rx) = tokio::sync::oneshot::channel(); + let (writer_entered_tx, mut writer_entered_rx) = tokio::sync::oneshot::channel(); + let writer = tokio::spawn(async move { + let transaction_store = writer_store.clone(); + let transaction = with_admin_server_config_write_lock(writer_store, move || async move { + writer_entered_tx.send(()).expect("signal writer entry"); + save_admin_server_config_no_lock(transaction_store, &latest).await + }); + tokio::pin!(transaction); + let mut writer_polled_tx = Some(writer_polled_tx); + poll_fn(|cx| match transaction.as_mut().poll(cx) { + Poll::Pending => { + if let Some(tx) = writer_polled_tx.take() { + let _ = tx.send(()); + } + Poll::Ready(()) + } + Poll::Ready(_) => panic!("writer entered before the worker apply released its read fence"), + }) + .await; + transaction + .await + .expect("writer should acquire the server-config locks") + .expect("writer should persist the latest config"); + }); + + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, writer_polled_rx) + .await + .expect("writer should be polled") + .expect("writer poll signal should be delivered"); + assert!( + matches!(writer_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)), + "a server-config writer must wait until the old worker apply completes" + ); + + release_worker_tx.send(()).expect("release worker apply"); + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async { + reload + .await + .expect("full reload task should not panic") + .expect("full reload should finish"); + writer.await.expect("writer task should not panic"); + }) + .await + .expect("reload and writer should finish"); + + reload_runtime_config_snapshot_for_context(Some(&fixture.context)) + .await + .expect("a later full reload should publish the latest durable config"); + let snapshot = fixture + .server_snapshot + .lock() + .expect("server config result lock") + .clone() + .expect("latest server config should be published"); + assert_eq!( + snapshot + .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER) + .expect("latest scanner config should be present") + .get(SCANNER_CYCLE), + "71" + ); + assert_eq!(rustfs_scanner::scanner_runtime_config_status().cycle_interval_seconds.value, 71); + + rustfs_scanner::apply_scanner_runtime_config(&ServerConfig::new()).expect("restore scanner runtime defaults"); + }, + ) + .await; + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn peer_full_reload_rejects_later_pool_without_publishing() { @@ -1251,11 +1545,9 @@ mod tests { let worker_events = events.clone(); let err = reload_runtime_config_snapshot_with( - async { Ok(ServerConfig::new()) }, - |config| async { Ok((config, PreparedRuntimeConfig::default())) }, - move |_config, _prepared| { + async move { publish_events.lock().expect("reload event lock").push("publish"); - Ok(()) + Ok(ServerConfig::new()) }, move |_config| async move { let mut events = worker_events.lock().expect("reload event lock"); diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index c678b3e88..cd437819e 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -644,12 +644,17 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc) -> Resu ecstore_config::com::read_config_without_migrate(api).await } +pub(crate) async fn read_existing_admin_server_config_no_lock(api: Arc) -> Result { + ecstore_config::com::read_existing_server_config_no_lock(api).await +} + #[cfg(test)] pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc) -> Result { ecstore_config::com::read_config_without_migrate_no_lock(api).await } pub(crate) type AdminServerConfigSnapshot = ecstore_config::com::ServerConfigSnapshot; +pub(crate) type AdminServerConfigSaveResult = ecstore_config::com::ServerConfigSaveResult; pub(crate) async fn save_admin_config(api: Arc, file: &str, data: Vec) -> Result<()> { ecstore_config::com::save_config(api, file, data).await @@ -690,8 +695,17 @@ pub(crate) async fn save_admin_server_config_snapshot( api: Arc, cfg: &rustfs_config::server_config::Config, snapshot: &AdminServerConfigSnapshot, -) -> Result { - ecstore_config::com::save_server_config_snapshot(api, cfg, snapshot).await +) -> Result { + ecstore_config::com::save_server_config_snapshot_with_generation(api, cfg, snapshot).await +} + +pub(crate) async fn with_admin_server_config_read_lock(api: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + ecstore_config::com::with_server_config_read_lock(api, operation).await } pub(crate) fn init_admin_config_defaults() { @@ -793,9 +807,10 @@ pub(crate) mod cluster { pub(crate) mod config { pub(crate) use super::storageclass; pub(crate) use super::{ - AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, - read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, save_admin_config, - save_admin_server_config_snapshot, + AdminServerConfigSaveResult, AdminServerConfigSnapshot, RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, + init_admin_config_defaults, read_admin_config, read_admin_config_without_migrate, read_admin_server_config_snapshot, + read_existing_admin_server_config_no_lock, save_admin_config, save_admin_server_config_snapshot, + with_admin_server_config_read_lock, }; #[cfg(test)] pub(crate) use super::{ diff --git a/scripts/e2e-run.sh b/scripts/e2e-run.sh index ed66c1aed..645184736 100755 --- a/scripts/e2e-run.sh +++ b/scripts/e2e-run.sh @@ -40,7 +40,19 @@ export RUST_BACKTRACE=full "$BIN" --address "127.0.0.1:${RUSTFS_TEST_PORT}" "$VOLUME" > "${RUSTFS_TEST_LOG}" 2>&1 & RUSTFS_PID=$! -sleep 10 +for _ in {1..60}; do + if curl --noproxy '*' --silent --fail "http://127.0.0.1:${RUSTFS_TEST_PORT}/health/ready" >/dev/null; then + break + fi + + if ! kill -0 "${RUSTFS_PID}" 2>/dev/null; then + wait "${RUSTFS_PID}" + fi + + sleep 1 +done + +curl --noproxy '*' --silent --show-error --fail "http://127.0.0.1:${RUSTFS_TEST_PORT}/health/ready" >/dev/null export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY:-rustfsadmin}" export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" From ffc9de72cbe79dfc68c4c8c0ef0e6e6a37b9f04a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 09:31:03 +0800 Subject: [PATCH 05/12] docs: make validation proportional to change risk (#5527) --- AGENTS.md | 83 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1ae7c5fc0..5a13a84b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,33 +105,78 @@ CI) fails the build if anything is committed under `docs/superpowers/`, even via ## Verification Before PR -Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion. -Non-exempt changes must also pass Adversarial Validation (next section) before the checks below count as completion. +Convert changes into independently verifiable outcomes. This section controls +agent-run local validation; preparing a commit or PR does not by itself require +the broadest gate. Inspect only the final task-owned diff, classify it by +behavioral impact rather than line count or path alone, and run the smallest +set of checks that provides meaningful coverage. Do not let unrelated +worktree changes or a generic contributor checklist expand the scope. +Non-exempt changes must also pass Adversarial Validation (next section) before +the checks below count as completion. -For code changes, run and pass the following before opening a PR: +### Validation floor -```bash -make pre-pr -``` +- Every change that is not documentation-only must finish with + `cargo fmt --all --check` passing. An umbrella gate that runs this exact + check satisfies the requirement; do not run it twice. Use `cargo fmt --all` + only when formatting needs to be fixed. Run the configured formatter or + validator for other changed languages when one exists. +- Documentation-only or instruction-only means all task-owned changes are + prose or documentation assets and cannot affect runtime, builds, CI, + dependencies, generated code, or tests. Run `git diff --check` and any + relevant documentation guard, but skip Cargo formatting, compilation, + Clippy, tests, `make pre-commit`, and `make pre-pr`. +- Behavior changes require relevant existing or new tests. Prefer the most + focused test or affected package. A passing targeted test can also provide + sufficient compilation coverage when it builds every changed target and + feature involved; do not add a redundant `cargo check` in that case. +- `cargo check` supplements compilation coverage; it never substitutes for a + behavioral test. If a relevant test cannot reasonably be added or run, use + the narrowest compilation check and report the reason and remaining risk. -Before committing code changes, prefer focused verification for the touched -surface and use the faster local gate when a broad smoke check is needed: +### Validation tiers -```bash -make pre-commit -``` +1. **Documentation/instruction-only:** Apply the exemption above. Run a guard + such as `make doc-paths-check` only when it is relevant to the edited text. +2. **Non-behavioral source change:** For comments, formatting, or another + demonstrably non-executable change, run the formatting floor. Compilation, + Clippy, and tests may be skipped only when the edit cannot affect + compilation or runtime behavior; run targeted doctests if executable + documentation examples changed. +3. **Localized or bounded behavior change:** Run the formatting floor and the + narrowest relevant tests. Add package-scoped `cargo check` or Clippy only + for changed targets, features, APIs, error handling, async behavior, or + control flow not already covered. When several crates are affected but the + dependency set is identifiable, validate those packages and known + dependents instead of the whole workspace. Use `make pre-commit` only when + a repository-wide fast gate adds useful confidence beyond those checks. +4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage + cannot bound the impact, including: + - dependency, feature, build-script, procedural-macro, code-generation, + toolchain, or CI changes that alter compilation or the test matrix; + - cross-crate public APIs, shared foundational code, or broad refactors with + an unbounded dependent set; + - locking, storage durability or formats, erasure coding, replication, + RPC/protocol compatibility, IAM/KMS/auth, cryptography, or other + security-sensitive behavior; + - a targeted check that reveals wider impact, an explicit user request, or + a release policy that requires the full gate. -For migration batches, do not run the full `make pre-pr` gate before every -intermediate commit. Use focused tests and `make pre-commit` during -development, then reserve `make pre-pr` for the final PR-ready branch. +Documentation-only and non-behavioral classifications take precedence over +path-based triggers. A small diff can still be high-risk, while a CI comment, +manifest comment, or release-note edit does not require full validation. -Before pushing code changes, make sure formatting is clean: +`make pre-pr` includes `make pre-commit` coverage. Never run both for the same +unchanged diff, and do not repeat equivalent checks during PR preparation or +because a local hook already ran them. Rerun only checks whose scope is affected +by later edits. Full workspace checks do not replace a relevant integration or +E2E test for changed behavior; run that focused test when required and +available, or report why it was not run and the remaining risk. -- Run `cargo fmt --all`. -- Run `cargo fmt --all --check` and ensure no files are modified unexpectedly. +If `make` is unavailable, run the equivalent checks defined under +`.config/make/`. At handoff, list the checks actually run, checks intentionally +skipped, and the reason for the selected tier. -If `make` is unavailable, run the equivalent checks defined under `.config/make/`. -Documentation-only or instruction-only changes are exempt from the verification commands above (including the `.config/make/` equivalents), though any locally installed git pre-commit hooks may still run on commit unless explicitly skipped. After build-based verification completes, clean generated build artifacts before wrapping up to avoid unnecessary disk usage. Do not open a PR with code changes when the required checks fail. Make a failing check pass by fixing the cause, never by weakening the gate: From 98b20b4231dac7d297f839f58eb9f776b51270e5 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 09:31:21 +0800 Subject: [PATCH 06/12] test(s3): cover delete marker list visibility (#5503) --- .../delete_marker_migration_semantics_test.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs index aac225d7a..5a51c053a 100644 --- a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs +++ b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs @@ -56,6 +56,21 @@ mod tests { ); } + async fn assert_current_list_hides_delete_marker(client: &Client, bucket: &str, key: &str) { + let listed = client + .list_objects_v2() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("list current objects after delete marker"); + + assert!( + listed.contents().iter().all(|object| object.key() != Some(key)), + "ListObjectsV2 must hide an object whose latest version is a delete marker" + ); + } + #[tokio::test] #[serial] async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() { @@ -94,6 +109,7 @@ mod tests { assert_eq!(markers[0].version_id(), Some(delete_marker_version_id)); assert_eq!(markers[0].is_latest(), Some(true)); assert_current_get_is_delete_marker_not_found(&client, bucket, key).await; + assert_current_list_hides_delete_marker(&client, bucket, key).await; } #[tokio::test] @@ -118,6 +134,17 @@ mod tests { .await .expect("put historical version"); let data_version_id = put.version_id().expect("put should return data version id"); + let listed_before_delete = client + .list_objects_v2() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("list current object before creating delete marker"); + assert!( + listed_before_delete.contents().iter().any(|object| object.key() == Some(key)), + "ListObjectsV2 must include the current object before it is deleted" + ); let delete_marker = client .delete_object() @@ -145,6 +172,7 @@ mod tests { assert_eq!(markers[0].version_id(), Some(delete_marker_version_id)); assert_eq!(markers[0].is_latest(), Some(true)); assert_current_get_is_delete_marker_not_found(&client, bucket, key).await; + assert_current_list_hides_delete_marker(&client, bucket, key).await; let historical = client .get_object() From fdac60b0e27ea529c6f15035e98b83f6c674fc35 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 09:36:12 +0800 Subject: [PATCH 07/12] fix(kms): make Vault KV2 lifecycle writes check-and-set (#5518) * fix(kms): drop stale KmsClient trait import in local_export tests The KmsClient trait was folded into KmsBackend (#5501), but the backup export tests merged afterwards (#5499) still imported it, breaking the crate's test build; create_key is an inherent LocalKmsClient method, so the import is simply unused. * fix(kms): make Vault KV2 lifecycle writes check-and-set Every KV2 lifecycle write used to be a blind whole-record overwrite, so two nodes racing on the same key could lose updates: a disable racing a rotation wrote the pre-rotation record back (rolling back the version and material of a committed rotation), concurrent same-name creates let the later material win (orphaning DEKs wrapped under the earlier one), and a cancellation racing the deletion sweep could be overwritten by the tombstone (or resurrect an already tombstoned key). All lifecycle mutations now go through a bounded check-and-set read-modify-write loop: each attempt re-reads the record pinned to its KV2 secret version, re-runs the state gate against the fresh snapshot, and writes back check-and-set against exactly that version; after LIFECYCLE_CAS_ATTEMPTS lost races the typed conflict error surfaces. The loop composes with the operation policy's single-attempt rule for non-idempotent writes: each write is still sent at most once, only the whole read-gate-write cycle repeats. create_key becomes a create-only write (cas=0) so exactly one of two concurrent creates commits and the loser reports KeyAlreadyExists. The blind store_key_data primitive is now test-only. Reads and rotation additionally fail closed when the version history is inconsistent: resolving material through a version record above the current pointer is refused (that state only arises when a lost update rolled back a committed rotation), and rotation refuses to extend a history whose records reach more than one step past the current pointer (one step ahead is the footprint of an interrupted rotation and still recovers through the adopt path). Refs rustfs/backlog#1581 --- crates/kms/src/backends/scripted_vault.rs | 42 +- crates/kms/src/backends/vault.rs | 1061 ++++++++++++++++++--- 2 files changed, 938 insertions(+), 165 deletions(-) diff --git a/crates/kms/src/backends/scripted_vault.rs b/crates/kms/src/backends/scripted_vault.rs index 19453ea46..320b78c1a 100644 --- a/crates/kms/src/backends/scripted_vault.rs +++ b/crates/kms/src/backends/scripted_vault.rs @@ -61,7 +61,7 @@ impl ScriptedResponse { pub(crate) struct ScriptedVault { /// Base address (`http://127.0.0.1:port`) to point a Vault client at. pub(crate) address: String, - requests: Arc>>, + requests: Arc>>, } impl ScriptedVault { @@ -81,13 +81,10 @@ impl ScriptedVault { let Ok((mut stream, _)) = listener.accept().await else { return; }; - let Some(request_line) = read_request(&mut stream).await else { + let Some(request) = read_request(&mut stream).await else { continue; }; - recorded - .lock() - .expect("scripted vault request log poisoned") - .push(request_line); + recorded.lock().expect("scripted vault request log poisoned").push(request); let response = responses .next() .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted")); @@ -107,14 +104,32 @@ impl ScriptedVault { /// The `METHOD /path` lines of every request served so far, in order. pub(crate) fn requests(&self) -> Vec { - self.requests.lock().expect("scripted vault request log poisoned").clone() + self.requests + .lock() + .expect("scripted vault request log poisoned") + .iter() + .map(|(line, _)| line.clone()) + .collect() + } + + /// The request bodies, in the same order as [`Self::requests`]; empty for + /// bodyless requests. Lets tests assert what a write actually persisted + /// (record contents, check-and-set options), not just that a write happened. + pub(crate) fn request_bodies(&self) -> Vec { + self.requests + .lock() + .expect("scripted vault request log poisoned") + .iter() + .map(|(_, body)| body.clone()) + .collect() } } /// Read one HTTP/1.1 request (head plus content-length body) and return its -/// `METHOD /path` line. Draining the body before responding keeps the client -/// from seeing a connection reset while it is still writing. -async fn read_request(stream: &mut TcpStream) -> Option { +/// `METHOD /path` line together with the body. Draining the body before +/// responding keeps the client from seeing a connection reset while it is +/// still writing. +async fn read_request(stream: &mut TcpStream) -> Option<(String, String)> { let mut buffer = Vec::new(); let mut chunk = [0u8; 4096]; let head_end = loop { @@ -146,14 +161,17 @@ async fn read_request(stream: &mut TcpStream) -> Option { }) .next() .unwrap_or(0); - let mut remaining = content_length.saturating_sub(buffer.len() - head_end); + let mut body = buffer[head_end..].to_vec(); + let mut remaining = content_length.saturating_sub(body.len()); while remaining > 0 { let read = stream.read(&mut chunk).await.ok()?; if read == 0 { break; } + body.extend_from_slice(&chunk[..read]); remaining = remaining.saturating_sub(read); } + body.truncate(content_length); - Some(format!("{method} {path}")) + Some((format!("{method} {path}"), String::from_utf8_lossy(&body).into_owned())) } diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 24d9c9c39..287ad4eb0 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -113,6 +113,29 @@ struct VaultKeyVersionRecord { /// Sub-path (under each key path) reserved for immutable version records. const KEY_VERSIONS_SUBPATH: &str = "versions"; +/// Upper bound on read-modify-write attempts for a check-and-set lifecycle +/// mutation. A lost race means live contention on the key; each retry re-reads +/// and re-validates, and a small bound keeps a pathologically contended key +/// from spinning while still absorbing ordinary interleavings. +const LIFECYCLE_CAS_ATTEMPTS: u32 = 3; + +/// Typed error for a lifecycle write that lost its check-and-set race (and, in +/// the retry loop, kept losing it up to the attempt bound). +fn concurrent_modification(key_id: &str) -> KmsError { + KmsError::invalid_operation(format!("Concurrent modification of key {key_id} detected, retry the operation")) +} + +/// Decision returned by a [`VaultKmsClient::update_key_data_with_cas`] +/// mutation closure. +enum CasMutation { + /// Persist the mutated record with a check-and-set write, then yield the + /// value. + Write(T), + /// The freshly observed state already settles the operation; yield the + /// value without writing. + Skip(T), +} + /// Drop KV2 directory entries from a key listing. /// /// Once a key has version records, listing the key prefix returns both the key @@ -293,7 +316,23 @@ impl VaultKmsClient { let encrypted_material = if version == key_data.version { key_data.encrypted_key_material.clone() } else { - self.get_key_version_record(key_id, version).await?.encrypted_key_material + let record = self.get_key_version_record(key_id, version).await?; + if version > key_data.version { + // The requested version has an immutable record, yet the + // current pointer sits below it. Material for a version is + // only requested once an envelope references it, and + // envelopes are only stamped after the pointer switch + // committed — so the pointer must have regressed (a lost + // update rolled back a committed rotation). Fail closed: + // serving in this state would keep new encryptions on the + // rolled-back material. A version with no record at all still + // fails as KeyVersionNotFound above. + return Err(KmsError::internal_error(format!( + "current version {} of key {key_id} is behind existing version record {version}; refusing to use an inconsistent key record", + key_data.version + ))); + } + record.encrypted_key_material }; decode_stored_key_material(key_id, &encrypted_material).inspect_err(|error| { @@ -345,39 +384,84 @@ impl VaultKmsClient { Ok((cas, key_data)) } - /// Check-and-set write of the key record. + /// Check-and-set write of the key record, reporting a lost race as + /// `Ok(None)`. /// /// `cas` must match the KV2 secret version currently holding the record. - /// Returns the secret version created by this write so a caller can chain - /// further check-and-set writes. - async fn cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result { + /// On success returns the secret version created by this write so a caller + /// can chain further check-and-set writes. + async fn try_cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result> { let path = self.key_path(key_id); let path = path.as_str(); // Single attempt: replaying a lost-response write would double-apply - // the mutation, and a CAS conflict is a normal concurrency signal that - // must reach the caller untouched. + // the mutation, and a CAS conflict is a normal concurrency signal the + // caller resolves by re-reading, never by resending the same write. let written = self .run("vault_kv2_cas_write_key", OpClass::MutatingNonIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; - kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas }) - .await - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - if is_cas_conflict(&e) { - KmsError::invalid_operation(format!( - "Concurrent modification of key {key_id} detected, retry the rotation" - )) - } else { - KmsError::backend_error(format!("Failed to store key in Vault: {e}")) - } - }) - }) + match kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas }).await + { + Ok(written) => Ok(Some(written)), + Err(e) if is_cas_conflict(&e) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to store key in Vault: {e}")) + })), + } }) .await?; - u32::try_from(written.version) - .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32"))) + written + .map(|written| { + u32::try_from(written.version) + .map_err(|_| KmsError::backend_error(format!("KV2 secret version for key {key_id} exceeds u32"))) + }) + .transpose() + } + + /// Check-and-set write of the key record, surfacing a lost race as the + /// typed concurrent-modification error. + async fn cas_store_key_data(&self, key_id: &str, key_data: &VaultKeyData, cas: u32) -> Result { + self.try_cas_store_key_data(key_id, key_data, cas) + .await? + .ok_or_else(|| concurrent_modification(key_id)) + } + + /// Apply a lifecycle mutation to the key record as a check-and-set + /// read-modify-write loop. + /// + /// Every attempt re-reads the record pinned to its current KV2 secret + /// version, re-derives the mutation from that fresh snapshot — `mutate` + /// must re-run its state gate, so a transition that lost a race against + /// e.g. a rotation or a cancellation is re-validated against the committed + /// state instead of being replayed — and writes back check-and-set against + /// exactly the version it read. A conflict means another writer committed + /// in between; after [`LIFECYCLE_CAS_ATTEMPTS`] lost races the typed + /// conflict error is surfaced to the caller. + /// + /// This loop does not bypass the operation policy's single-attempt rule + /// for `MutatingNonIdempotent` writes: each individual write is still sent + /// at most once and never replayed on a lost response. Only the whole + /// read-gate-mutate-write cycle repeats, and every repeat is derived from + /// newly observed state, so the two layers compose instead of conflicting. + async fn update_key_data_with_cas(&self, key_id: &str, mut mutate: F) -> Result<(VaultKeyData, T)> + where + F: FnMut(&mut VaultKeyData) -> Result>, + { + for attempt in 1..=LIFECYCLE_CAS_ATTEMPTS { + let (cas, mut key_data) = self.get_key_data_versioned(key_id).await?; + match mutate(&mut key_data)? { + CasMutation::Skip(value) => return Ok((key_data, value)), + CasMutation::Write(value) => { + if self.try_cas_store_key_data(key_id, &key_data, cas).await?.is_some() { + return Ok((key_data, value)); + } + debug!(key_id, attempt, "Vault KV2 lifecycle write lost a check-and-set race; re-reading"); + } + } + } + + Err(concurrent_modification(key_id)) } /// Create-only write of an immutable version record (KV2 check-and-set of 0). @@ -405,14 +489,42 @@ impl VaultKmsClient { .await } - /// Store key data in Vault + /// Create-only write of the top-level key record (KV2 check-and-set of 0). + /// + /// Returns `Ok(true)` when this call created the record and `Ok(false)` + /// when a record already exists — i.e. a concurrent create committed + /// first. An existing record is never overwritten. + async fn try_create_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result { + let path = self.key_path(key_id); + let path = path.as_str(); + + // Single attempt: the create-only CAS makes a duplicate replay fail + // with a conflict, which create_key reports as the key already + // existing, so retrying here would only mask that signal. + self.run("vault_kv2_create_key", OpClass::MutatingNonIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::set_with_options(&vault.client, &self.kv_mount, path, key_data, SetSecretRequestOptions { cas: 0 }).await { + Ok(_) => Ok(true), + Err(e) if is_cas_conflict(&e) => Ok(false), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to store key in Vault: {e}")) + })), + } + }) + .await + } + + /// Blind, last-writer-wins overwrite of the key record. + /// + /// Test-only: production writes go through the create-only or + /// check-and-set paths so concurrent writers cannot silently clobber each + /// other. Kept for tests that need to inject corrupted or downgraded + /// records. + #[cfg(test)] async fn store_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> { let path = self.key_path(key_id); let path = path.as_str(); - // Single attempt: this is a whole-record overwrite without a CAS - // precondition, so a replay after a lost response could clobber a - // concurrent writer. self.run("vault_kv2_write_key", OpClass::MutatingNonIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; kv2::set(&vault.client, &self.kv_mount, path, key_data) @@ -431,40 +543,28 @@ impl VaultKmsClient { async fn store_key_metadata(&self, key_id: &str, request: &CreateKeyRequest) -> Result<()> { debug!("Storing key metadata for {}, input tags: {:?}", key_id, request.tags); - // Get existing key data to preserve encrypted_key_material and other fields - // This is called after create_key, so the key should already exist - let existing_key_data = self.get_key_data(key_id).await?; + // Read-modify-write under check-and-set: only the request-driven + // fields change, everything else — most importantly the key material, + // version and status — is carried over from the freshly read record, + // so a rotation or state transition landing in between is preserved + // instead of clobbered. + self.update_key_data_with_cas(key_id, |key_data| { + // A key that was just created must already carry material; an empty value means + // the create flow failed to persist it. Fail closed instead of minting replacement + // material: silently generating a new key here would mask the broken create and + // orphan any DEK already wrapped by a different copy of this key. + if key_data.encrypted_key_material.is_empty() { + warn!(key_id, "Vault KMS key metadata missing encrypted key material"); + return Err(KmsError::material_missing(key_id)); + } - // A key that was just created must already carry material; an empty value means - // the create flow failed to persist it. Fail closed instead of minting replacement - // material: silently generating a new key here would mask the broken create and - // orphan any DEK already wrapped by a different copy of this key. - if existing_key_data.encrypted_key_material.is_empty() { - warn!(key_id, "Vault KMS key metadata missing encrypted key material"); - return Err(KmsError::material_missing(key_id)); - } - - // Update only the metadata fields, preserving the encrypted_key_material - let key_data = VaultKeyData { - algorithm: existing_key_data.algorithm.clone(), - usage: request.key_usage.clone(), - created_at: existing_key_data.created_at, - status: existing_key_data.status, - version: existing_key_data.version, - description: request.description.clone(), - metadata: existing_key_data.metadata.clone(), - tags: request.tags.clone(), - deletion_date: existing_key_data.deletion_date.clone(), - encrypted_key_material: existing_key_data.encrypted_key_material.clone(), // Preserve the key material - baseline_version: existing_key_data.baseline_version, - }; - - debug!( - "VaultKeyData tags before storage: {:?}, encrypted_key_material length: {}", - key_data.tags, - key_data.encrypted_key_material.len() - ); - self.store_key_data(key_id, &key_data).await + key_data.usage = request.key_usage.clone(); + key_data.description = request.description.clone(); + key_data.tags = request.tags.clone(); + Ok(CasMutation::Write(())) + }) + .await?; + Ok(()) } /// Retrieve key data from Vault @@ -520,6 +620,53 @@ impl VaultKmsClient { } } + /// List the names of a key's immutable version records. + /// + /// `None` means the versions directory does not exist — the key was never + /// rotated and has no version records. + async fn list_key_version_records(&self, key_id: &str) -> Result>> { + let versions_dir = self.key_versions_dir(key_id); + let versions_dir = versions_dir.as_str(); + self.run("vault_kv2_list_key_versions", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::list(&vault.client, &self.kv_mount, versions_dir).await { + Ok(versions) => Ok(Some(versions)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to list key version records in Vault: {e}")) + })), + } + }) + .await + } + + /// Fail closed when the version history extends more than one step past + /// the current version. + /// + /// A record exactly one above current is the footprint of an interrupted + /// rotation (material persisted, pointer switch never committed) and is + /// recovered by the next rotation's adopt path. Anything further ahead + /// cannot come from the rotation protocol: it means the top-level record + /// regressed (for example a historical lost update rolled back committed + /// rotations), and extending the history from the rolled-back state would + /// re-mint version numbers that already have immutable records. + async fn ensure_current_version_not_behind(&self, key_id: &str, current_version: u32) -> Result<()> { + let max_recorded = self + .list_key_version_records(key_id) + .await? + .unwrap_or_default() + .iter() + .filter_map(|entry| entry.trim_end_matches('/').parse::().ok()) + .max(); + + match max_recorded { + Some(max_recorded) if max_recorded > current_version.saturating_add(1) => Err(KmsError::internal_error(format!( + "current version {current_version} of key {key_id} is behind existing version record {max_recorded}; refusing to extend an inconsistent version history" + ))), + _ => Ok(()), + } + } + /// Physically delete a key from Vault storage async fn delete_key(&self, key_id: &str) -> Result<()> { let path = self.key_path(key_id); @@ -531,18 +678,7 @@ impl VaultKmsClient { let versions_dir = self.key_versions_dir(key_id); let versions_dir = versions_dir.as_str(); // `None` means no version records exist (the key was never rotated). - let versions = self - .run("vault_kv2_list_key_versions", OpClass::ReadIdempotent, move || async move { - let vault = self.vault().map_err(AttemptError::fatal)?; - match kv2::list(&vault.client, &self.kv_mount, versions_dir).await { - Ok(versions) => Ok(Some(versions)), - Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), - Err(e) => Err(AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to list key version records in Vault: {e}")) - })), - } - }) - .await?; + let versions = self.list_key_version_records(key_id).await?; for version in versions.unwrap_or_default() { let version_path = format!("{versions_dir}/{version}"); let version_path = version_path.as_str(); @@ -768,8 +904,14 @@ impl VaultKmsClient { baseline_version: None, }; - // Store in Vault - self.store_key_data(key_id, &key_data).await?; + // Create-only write: the not-found pre-check above is only advisory — + // another node can create the same key in between — so the write + // itself must refuse to overwrite. Exactly one of two concurrent + // creates commits; the loser reports the key as already existing + // instead of adopting material it did not persist. + if !self.try_create_key_data(key_id, &key_data).await? { + return Err(KmsError::key_already_exists(key_id)); + } let master_key = MasterKeyInfo { key_id: key_id.to_string(), @@ -853,10 +995,12 @@ impl VaultKmsClient { pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Enabling key: {}", key_id); - let mut key_data = self.get_key_data(key_id).await?; - ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Enable)?; - key_data.status = KeyStatus::Active; - self.store_key_data(key_id, &key_data).await?; + self.update_key_data_with_cas(key_id, |key_data| { + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Enable)?; + key_data.status = KeyStatus::Active; + Ok(CasMutation::Write(())) + }) + .await?; debug!(key_id, "Vault KMS key enabled"); Ok(()) @@ -865,10 +1009,12 @@ impl VaultKmsClient { pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { debug!("Disabling key: {}", key_id); - let mut key_data = self.get_key_data(key_id).await?; - ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Disable)?; - key_data.status = KeyStatus::Disabled; - self.store_key_data(key_id, &key_data).await?; + self.update_key_data_with_cas(key_id, |key_data| { + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::Disable)?; + key_data.status = KeyStatus::Disabled; + Ok(CasMutation::Write(())) + }) + .await?; debug!(key_id, "Vault KMS key disabled"); Ok(()) @@ -901,6 +1047,11 @@ impl VaultKmsClient { decode_stored_key_material(key_id, &key_data.encrypted_key_material) .inspect_err(|error| warn!(key_id, %error, "Vault KMS key material failed validation"))?; + // A version history that already extends past what this rotation would + // commit means the current pointer regressed; fail closed instead of + // re-minting version numbers that have immutable records. + self.ensure_current_version_not_behind(key_id, key_data.version).await?; + // Step 1: freeze the baseline on first rotation. if key_data.baseline_version.is_none() { let baseline = VaultKeyVersionRecord { @@ -1028,31 +1179,27 @@ impl VaultKmsBackend { self.client.credentials.spawn_renewal_task() } - /// Update key metadata in Vault storage - async fn update_key_metadata_in_storage(&self, key_id: &str, metadata: &KeyMetadata) -> Result<()> { - // Get the current key data from Vault - let mut key_data = self.client.get_key_data(key_id).await?; - - // This is a read-modify-write of the whole VaultKeyData document. Refuse to write - // back a record whose key material is missing: persisting it would cement the - // empty-material state under a fresh document version. A damaged key must go - // through an explicit repair operation, not a metadata update. - if key_data.encrypted_key_material.is_empty() { - return Err(KmsError::material_missing(key_id)); - } - - // Update the status based on the new metadata - key_data.status = match metadata.key_state { - KeyState::Enabled => KeyStatus::Active, - KeyState::Disabled => KeyStatus::Disabled, - KeyState::PendingDeletion => KeyStatus::PendingDeletion, - KeyState::Unavailable => KeyStatus::Deleted, - KeyState::PendingImport => KeyStatus::Disabled, // Treat as disabled until import completes - }; - key_data.deletion_date = metadata.deletion_date.clone(); - - // Update the key data in Vault storage - self.client.store_key_data(key_id, &key_data).await?; + /// Mark a key `PendingDeletion` with the given deadline, under + /// check-and-set with per-attempt re-validation. + /// + /// The state gate re-runs on every attempt, so a transition that lost a + /// race (for example against a concurrent schedule or rotation) is + /// re-validated against the committed state. Refuses to write back a + /// record whose key material is missing: persisting it would cement the + /// empty-material state under a fresh document version — a damaged key + /// must go through an explicit repair operation, not a lifecycle update. + async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &Zoned) -> Result<()> { + self.client + .update_key_data_with_cas(key_id, |key_data| { + ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?; + if key_data.encrypted_key_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + key_data.status = KeyStatus::PendingDeletion; + key_data.deletion_date = Some(deletion_date.clone()); + Ok(CasMutation::Write(())) + }) + .await?; Ok(()) } } @@ -1183,12 +1330,22 @@ impl KmsBackend for VaultKmsBackend { if key_metadata.key_state == KeyState::PendingDeletion || key_metadata.key_state == KeyState::Unavailable { // Tombstone first: mark the record Deleted before removing it, // so a crash between the two steps leaves a key that is already - // unusable and whose removal can simply be re-run. - if key_metadata.key_state == KeyState::PendingDeletion { - let mut key_data = self.client.get_key_data(key_id).await?; - key_data.status = KeyStatus::Deleted; - self.client.store_key_data(key_id, &key_data).await?; - } + // unusable and whose removal can simply be re-run. Written + // check-and-set with re-validation: a concurrent cancellation + // that commits first wins and fails this removal instead of + // being overwritten. + self.client + .update_key_data_with_cas(key_id, |key_data| match key_data.status { + KeyStatus::Deleted => Ok(CasMutation::Skip(())), + KeyStatus::PendingDeletion => { + key_data.status = KeyStatus::Deleted; + Ok(CasMutation::Write(())) + } + KeyStatus::Active | KeyStatus::Disabled => { + Err(KmsError::invalid_key_state(format!("Key {key_id} is no longer pending deletion"))) + } + }) + .await?; // Force immediate deletion: physically delete the key from Vault storage self.client.delete_key(key_id).await?; @@ -1196,11 +1353,10 @@ impl KmsBackend for VaultKmsBackend { None } else { // For non-pending keys, mark as PendingDeletion + let marked_at = Zoned::now(); + self.mark_key_pending_deletion(key_id, &marked_at).await?; key_metadata.key_state = KeyState::PendingDeletion; - key_metadata.deletion_date = Some(Zoned::now()); - - // Update the key metadata in Vault storage to reflect the new state - self.update_key_metadata_in_storage(key_id, &key_metadata).await?; + key_metadata.deletion_date = Some(marked_at); None } @@ -1216,12 +1372,10 @@ impl KmsBackend for VaultKmsBackend { } let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400); + self.mark_key_pending_deletion(key_id, &deletion_date).await?; key_metadata.key_state = KeyState::PendingDeletion; key_metadata.deletion_date = Some(deletion_date.clone()); - // Update the key metadata in Vault storage to reflect the new state - self.update_key_metadata_in_storage(key_id, &key_metadata).await?; - Some(deletion_date.to_string()) }; @@ -1248,15 +1402,32 @@ impl KmsBackend for VaultKmsBackend { return Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); } + // Persist the reset state back to Vault. Without this the key stays PendingDeletion in + // storage and would still be reaped, so we must fail the request if the write fails + // rather than report a false success. Check-and-set with per-attempt + // re-validation: once the deletion sweep has tombstoned the record, the + // cancellation must fail instead of resurrecting a key whose material + // is about to be (or already is) destroyed. + self.client + .update_key_data_with_cas(key_id, |key_data| match key_data.status { + KeyStatus::PendingDeletion => { + if key_data.encrypted_key_material.is_empty() { + return Err(KmsError::material_missing(key_id)); + } + key_data.status = KeyStatus::Active; + key_data.deletion_date = None; + Ok(CasMutation::Write(())) + } + KeyStatus::Active | KeyStatus::Disabled | KeyStatus::Deleted => { + Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))) + } + }) + .await?; + // Cancel the deletion by resetting the state key_metadata.key_state = KeyState::Enabled; key_metadata.deletion_date = None; - // Persist the reset state back to Vault. Without this the key stays PendingDeletion in - // storage and would still be reaped, so we must fail the request if the write fails - // rather than report a false success. - self.update_key_metadata_in_storage(key_id, &key_metadata).await?; - Ok(CancelKeyDeletionResponse { key_id: key_id.clone(), key_metadata, @@ -1293,31 +1464,36 @@ impl KmsBackend for VaultKmsBackend { } async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result { - // Vault KV2 offers no compare-and-swap here, so a cancellation racing - // the read below can still lose; the window is a single read-write - // gap and the sweep re-reads on every pass. - let mut key_data = match self.client.get_key_data(key_id).await { - Ok(key_data) => key_data, - Err(KmsError::KeyNotFound { .. }) => return Ok(ExpiredKeyRemoval::Removed), - Err(error) => return Err(error), - }; - match key_data.status { - // Tombstone left by a crashed removal: complete it. - KeyStatus::Deleted => {} - KeyStatus::PendingDeletion => { - match &key_data.deletion_date { - Some(deadline) if deadline <= now => {} + // Tombstone under check-and-set with per-attempt re-validation: a + // cancellation landing between the read and the write makes the write + // conflict, and the re-read then observes the cancelled state and + // reports StateChanged instead of overwriting it. + let settled = self + .client + .update_key_data_with_cas(key_id, |key_data| match key_data.status { + // Tombstone left by a crashed removal: complete it. + KeyStatus::Deleted => Ok(CasMutation::Skip(None)), + KeyStatus::PendingDeletion => match &key_data.deletion_date { + Some(deadline) if deadline <= now => { + // Tombstone first: mark the record Deleted before + // removing it, so a crash between the two steps leaves + // a key that is already unusable and whose removal can + // simply be re-run. + key_data.status = KeyStatus::Deleted; + Ok(CasMutation::Write(None)) + } // Not yet due, or a legacy record without a persisted // deadline — never auto-remove those. - _ => return Ok(ExpiredKeyRemoval::NotExpired), - } - // Tombstone first: mark the record Deleted before removing it, - // so a crash between the two steps leaves a key that is - // already unusable and whose removal can simply be re-run. - key_data.status = KeyStatus::Deleted; - self.client.store_key_data(key_id, &key_data).await?; - } - KeyStatus::Active | KeyStatus::Disabled => return Ok(ExpiredKeyRemoval::StateChanged), + _ => Ok(CasMutation::Skip(Some(ExpiredKeyRemoval::NotExpired))), + }, + KeyStatus::Active | KeyStatus::Disabled => Ok(CasMutation::Skip(Some(ExpiredKeyRemoval::StateChanged))), + }) + .await; + match settled { + Ok((_, Some(outcome))) => return Ok(outcome), + Ok((_, None)) => {} + Err(KmsError::KeyNotFound { .. }) => return Ok(ExpiredKeyRemoval::Removed), + Err(error) => return Err(error), } match self.client.delete_key(key_id).await { @@ -1438,7 +1614,7 @@ mod tests { let (vault, client) = scripted_client(vec![ScriptedResponse::error(503, "sealed")]).await; let error = client - .store_key_data("wired-key", &healthy_key_data()) + .cas_store_key_data("wired-key", &healthy_key_data(), 1) .await .expect_err("the scripted 503 must fail the write"); assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); @@ -2167,10 +2343,12 @@ mod tests { let mut disabled = healthy_key_data(); disabled.status = KeyStatus::Disabled; let vault = ScriptedVault::serve(vec![ - // disable: read the Active record, persist it Disabled. + // disable: versioned read of the Active record, persist it Disabled. + ScriptedResponse::ok(kv2_metadata_read_data(1)), ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), ScriptedResponse::ok(kv2_write_ack()), - // enable: read the Disabled record, persist it Active. + // enable: versioned read of the Disabled record, persist it Active. + ScriptedResponse::ok(kv2_metadata_read_data(2)), ScriptedResponse::ok(kv2_read_data(&disabled)), ScriptedResponse::ok(kv2_write_ack()), ]) @@ -2192,8 +2370,585 @@ mod tests { .expect("KmsBackend::enable_key must persist through the client"); let requests = vault.requests(); - assert_eq!(requests.len(), 4, "each transition is one read plus one write: {requests:?}"); - assert!(requests[0].starts_with("GET ") && requests[2].starts_with("GET "), "{requests:?}"); - assert!(requests[1].starts_with("POST ") && requests[3].starts_with("POST "), "{requests:?}"); + assert_eq!( + requests.len(), + 6, + "each transition is one versioned read (metadata + data) plus one write: {requests:?}" + ); + assert!( + requests[0].starts_with("GET /v1/secret/metadata/") && requests[3].starts_with("GET /v1/secret/metadata/"), + "{requests:?}" + ); + assert!( + requests[1].starts_with("GET /v1/secret/data/") && requests[4].starts_with("GET /v1/secret/data/"), + "{requests:?}" + ); + assert!(requests[2].starts_with("POST ") && requests[5].starts_with("POST "), "{requests:?}"); + + // Both lifecycle writes must carry a check-and-set precondition pinned + // to the KV2 secret version they read. + let bodies = vault.request_bodies(); + for (index, cas) in [(2usize, 1u64), (5, 2)] { + let body: serde_json::Value = serde_json::from_str(&bodies[index]).expect("lifecycle write body must be JSON"); + assert_eq!( + body["options"]["cas"], + serde_json::json!(cas), + "write {index} must be check-and-set: {body}" + ); + } + } + + /// Parse a captured KV2 write body (`{"data": ..., "options": {"cas": N}}`). + fn parse_write_body(body: &str) -> serde_json::Value { + serde_json::from_str(body).expect("KV2 write body must be JSON") + } + + /// KV2 read payload for an immutable version record. + fn kv2_read_version_record_data(record: &VaultKeyVersionRecord) -> serde_json::Value { + serde_json::json!({ + "data": serde_json::to_value(record).expect("serialize version record"), + "metadata": { + "created_time": "2026-01-01T00:00:00Z", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1, + }, + }) + } + + const CAS_CONFLICT_MESSAGE: &str = "check-and-set parameter did not match the current version"; + + /// Base64 material distinct from `healthy_key_data`'s, standing in for the + /// material a concurrent rotation committed. + fn rotated_material() -> String { + general_purpose::STANDARD.encode([0x43u8; 32]) + } + + /// The issue's lost-update scenario: node A disables a key while node B's + /// rotation commits in between. The blind write this replaces would have + /// written A's stale snapshot back — rolling the key from version 2 to + /// version 1 and resurrecting the pre-rotation material, which is exactly + /// what the final-write assertions below reject. Under check-and-set the + /// stale write conflicts, A re-reads, re-passes the state gate against the + /// rotated record, and persists only the status change on top of it. + #[tokio::test] + async fn wired_disable_interleaved_with_rotate_preserves_committed_rotation() { + let pre_rotate = healthy_key_data(); + let mut rotated = healthy_key_data(); + rotated.version = 2; + rotated.baseline_version = Some(1); + rotated.encrypted_key_material = rotated_material(); + + let (vault, client) = scripted_client(vec![ + // Attempt 1: versioned read observes the pre-rotation record... + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&pre_rotate)), + // ...but the concurrent rotation committed KV2 versions 2 and 3 in + // between, so the check-and-set write loses. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // Attempt 2: the re-read observes the rotated record and the write + // pinned to it succeeds. + ScriptedResponse::ok(kv2_metadata_read_data(3)), + ScriptedResponse::ok(kv2_read_data(&rotated)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + client + .disable_key("wired-key", None) + .await + .expect("the disable must retry past the lost race and commit"); + + let requests = vault.requests(); + assert_eq!( + requests, + vec![ + "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(), + "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=1".to_string(), + "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(), + "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=3".to_string(), + "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + ], + "a conflict must trigger exactly one full re-read before the retry write" + ); + + let bodies = vault.request_bodies(); + let first_write = parse_write_body(&bodies[2]); + assert_eq!(first_write["options"]["cas"], serde_json::json!(1), "{first_write}"); + + // The committed write must be the *rotated* snapshot with only the + // status changed. A blind write would have persisted version 1 and the + // pre-rotation material here. + let committed = parse_write_body(&bodies[5]); + assert_eq!(committed["options"]["cas"], serde_json::json!(3), "{committed}"); + assert_eq!(committed["data"]["status"], serde_json::json!("Disabled"), "{committed}"); + assert_eq!( + committed["data"]["version"], + serde_json::json!(2), + "the rotation's version bump must survive: {committed}" + ); + assert_eq!(committed["data"]["baseline_version"], serde_json::json!(1), "{committed}"); + assert_eq!( + committed["data"]["encrypted_key_material"], + serde_json::json!(rotated_material()), + "the rotation's material must survive the disable: {committed}" + ); + } + + #[tokio::test] + async fn wired_create_key_write_is_create_only() { + let (vault, client) = scripted_client(vec![ + // Existence pre-check: not found. + ScriptedResponse::error(404, "not found"), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + let created = client + .create_key("wired-key", "AES_256", None) + .await + .expect("create against an absent key must succeed"); + assert_eq!(created.version, 1); + + let requests = vault.requests(); + assert_eq!( + requests, + vec![ + "GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(), + ] + ); + + // The write must be create-only (check-and-set of 0). A blind + // overwrite — the pre-CAS behavior — carries no options at all. + let body = parse_write_body(&vault.request_bodies()[1]); + assert_eq!(body["options"]["cas"], serde_json::json!(0), "create must be create-only: {body}"); + } + + /// Concurrent same-name create: both nodes pass the not-found pre-check, + /// exactly one create-only write commits, and the loser reports + /// KeyAlreadyExists instead of overwriting the winner's material (which + /// would permanently orphan every DEK the winner already wrapped). + #[tokio::test] + async fn wired_concurrent_create_loser_reports_key_already_exists() { + let (vault, client) = scripted_client(vec![ + // Existence pre-check: not found (the racing create has not + // committed yet). + ScriptedResponse::error(404, "not found"), + // The create-only write loses: the racing create committed first. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ]) + .await; + + let error = client + .create_key("wired-key", "AES_256", None) + .await + .expect_err("the losing create must fail"); + assert!(matches!(error, KmsError::KeyAlreadyExists { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "the loser must not retry or fall back to a blind write: {requests:?}"); + } + + /// Deletion sweep racing a cancellation: the sweep's tombstone write loses + /// its check-and-set race, the re-read observes the cancelled (Active) + /// record, and the sweep reports StateChanged without deleting anything. + /// The blind tombstone this replaces would have overwritten the committed + /// cancellation and destroyed the key. + #[tokio::test] + async fn wired_expired_key_sweep_yields_to_concurrent_cancellation() { + let now = Zoned::now() + Duration::from_secs(3600); + let mut pending = healthy_key_data(); + pending.status = KeyStatus::PendingDeletion; + pending.deletion_date = Some(Zoned::now()); + + let vault = ScriptedVault::serve(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&pending)), + // The cancellation commits between the read and the tombstone. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ScriptedResponse::ok(kv2_metadata_read_data(2)), + // The re-read observes the cancelled (Active again) record. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let outcome = backend + .remove_expired_key("wired-key", &now) + .await + .expect("the sweep must settle by observing the cancelled state"); + assert_eq!(outcome, ExpiredKeyRemoval::StateChanged); + + let requests = vault.requests(); + assert_eq!(requests.len(), 5, "{requests:?}"); + assert!( + !requests.iter().any(|line| line.starts_with("DELETE ")), + "a sweep that lost to a cancellation must not delete anything: {requests:?}" + ); + // The one write attempt was the check-and-set tombstone. + let body = parse_write_body(&vault.request_bodies()[2]); + assert_eq!(body["options"]["cas"], serde_json::json!(1), "{body}"); + assert_eq!(body["data"]["status"], serde_json::json!("Deleted"), "{body}"); + } + + /// The other half of the cancel × sweep interleaving: once the sweep has + /// tombstoned the record, a cancellation re-validates against the fresh + /// state and fails instead of resurrecting a key whose material is about + /// to be destroyed. + #[tokio::test] + async fn wired_cancel_deletion_after_sweep_tombstone_fails_closed() { + let mut pending = healthy_key_data(); + pending.status = KeyStatus::PendingDeletion; + pending.deletion_date = Some(Zoned::now()); + let mut tombstoned = healthy_key_data(); + tombstoned.status = KeyStatus::Deleted; + + let vault = ScriptedVault::serve(vec![ + // describe_key still observes the pre-sweep PendingDeletion state + // (one read for the key info, one for the stored metadata). + ScriptedResponse::ok(kv2_read_data(&pending)), + ScriptedResponse::ok(kv2_read_data(&pending)), + // The check-and-set update re-reads and observes the tombstone. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(kv2_read_data(&tombstoned)), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let error = backend + .cancel_key_deletion(CancelKeyDeletionRequest { + key_id: "wired-key".to_string(), + }) + .await + .expect_err("cancelling after the sweep tombstoned the key must fail"); + assert!( + matches!(&error, KmsError::InvalidOperation { message } if message.contains("not pending deletion")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert!( + !requests.iter().any(|line| line.starts_with("POST ")), + "a cancellation that lost to the sweep must not write anything: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_schedule_deletion_retries_after_cas_conflict() { + let vault = ScriptedVault::serve(vec![ + // describe_key: key info plus stored metadata. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // Attempt 1 loses its check-and-set race. + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // Attempt 2: the re-read re-passes the state gate and commits. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let response = backend + .delete_key(DeleteKeyRequest { + key_id: "wired-key".to_string(), + pending_window_in_days: Some(7), + force_immediate: Some(false), + }) + .await + .expect("the schedule must retry past the lost race and commit"); + assert!(response.deletion_date.is_some()); + + let requests = vault.requests(); + assert_eq!(requests.len(), 8, "{requests:?}"); + let committed = parse_write_body(&vault.request_bodies()[7]); + assert_eq!(committed["options"]["cas"], serde_json::json!(2), "{committed}"); + assert_eq!(committed["data"]["status"], serde_json::json!("PendingDeletion"), "{committed}"); + assert!( + !committed["data"]["deletion_date"].is_null(), + "the deadline must be persisted: {committed}" + ); + } + + /// Conflict semantics are re-read *and* re-gate: when the re-read after a + /// lost race shows the key was concurrently scheduled for deletion, the + /// state gate rejects the retry instead of blindly re-applying it. + #[tokio::test] + async fn wired_schedule_deletion_regates_after_conflict() { + let mut already_pending = healthy_key_data(); + already_pending.status = KeyStatus::PendingDeletion; + already_pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400)); + + let vault = ScriptedVault::serve(vec![ + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // A concurrent schedule committed first. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(kv2_read_data(&already_pending)), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let error = backend + .delete_key(DeleteKeyRequest { + key_id: "wired-key".to_string(), + pending_window_in_days: Some(7), + force_immediate: Some(false), + }) + .await + .expect_err("the retry must re-run the state gate against the fresh record"); + assert!( + matches!(&error, KmsError::InvalidOperation { message } if message.contains("pending deletion")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 7, "{requests:?}"); + assert_eq!( + requests.iter().filter(|line| line.starts_with("POST ")).count(), + 1, + "the rejected retry must not write again: {requests:?}" + ); + } + + /// The read-modify-write loop is bounded: persistent contention surfaces + /// the typed conflict error after `LIFECYCLE_CAS_ATTEMPTS` full + /// read-gate-write cycles instead of spinning or falling back to a blind + /// write. + #[tokio::test] + async fn wired_lifecycle_cas_retries_are_bounded() { + let mut responses = Vec::new(); + for secret_version in 1..=LIFECYCLE_CAS_ATTEMPTS as u64 { + responses.push(ScriptedResponse::ok(kv2_metadata_read_data(secret_version))); + responses.push(ScriptedResponse::ok(kv2_read_data(&healthy_key_data()))); + responses.push(ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE)); + } + let (vault, client) = scripted_client(responses).await; + + let error = client + .disable_key("wired-key", None) + .await + .expect_err("persistent contention must surface the typed conflict error"); + assert!( + matches!(&error, KmsError::InvalidOperation { message } if message.contains("Concurrent modification")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3 * LIFECYCLE_CAS_ATTEMPTS as usize, "{requests:?}"); + assert_eq!( + requests.iter().filter(|line| line.starts_with("POST ")).count(), + LIFECYCLE_CAS_ATTEMPTS as usize, + "every attempt must be a fresh read-gate-write cycle: {requests:?}" + ); + } + + /// The tags write-back after a create is a check-and-set read-modify-write + /// that carries the key material over from the freshly read record. + #[tokio::test] + async fn wired_create_key_tags_writeback_is_check_and_set() { + let vault = ScriptedVault::serve(vec![ + // create_key: existence pre-check misses, create-only write lands. + ScriptedResponse::error(404, "not found"), + ScriptedResponse::ok(kv2_write_ack()), + // store_key_metadata: versioned read plus check-and-set write. + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let response = backend + .create_key(CreateKeyRequest { + key_name: Some("wired-key".to_string()), + key_usage: KeyUsage::EncryptDecrypt, + tags: HashMap::from([("team".to_string(), "storage".to_string())]), + ..Default::default() + }) + .await + .expect("create with tags must succeed"); + assert_eq!(response.key_id, "wired-key"); + + let bodies = vault.request_bodies(); + let create = parse_write_body(&bodies[1]); + assert_eq!(create["options"]["cas"], serde_json::json!(0), "{create}"); + + let writeback = parse_write_body(&bodies[4]); + assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}"); + assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}"); + assert_eq!( + writeback["data"]["encrypted_key_material"], + serde_json::json!(healthy_key_data().encrypted_key_material), + "the write-back must preserve the material of the freshly read record: {writeback}" + ); + } + + /// A version record above the current pointer means the top-level record + /// regressed (a lost update rolled back a committed rotation). Resolving + /// material through such a record must fail closed instead of quietly + /// serving it while new encryptions keep using the rolled-back material. + #[tokio::test] + async fn wired_decrypt_fails_closed_when_current_version_regressed() { + let material_v2 = [0x43u8; 32]; + let record_v2 = VaultKeyVersionRecord { + version: 2, + encrypted_key_material: general_purpose::STANDARD.encode(material_v2), + created_at: Zoned::now(), + }; + // A well-formed envelope wrapped under version 2 — under a reverted + // guard this decrypt would *succeed*, which is exactly the masked + // rollback this test pins down. + let (encrypted_key, nonce) = AesDekCrypto::new() + .encrypt(&material_v2, b"dek-plaintext") + .await + .expect("wrap test DEK"); + let envelope = DataKeyEnvelope { + key_id: "dek".to_string(), + master_key_id: "wired-key".to_string(), + key_spec: "AES_256".to_string(), + encrypted_key, + nonce, + encryption_context: HashMap::new(), + created_at: Zoned::now(), + master_key_version: Some(2), + }; + let ciphertext = serde_json::to_vec(&envelope).expect("serialize envelope"); + + let (vault, client) = scripted_client(vec![ + // Top-level record: current version rolled back to 1. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // ...yet the immutable record for version 2 exists. + ScriptedResponse::ok(kv2_read_version_record_data(&record_v2)), + ]) + .await; + + let error = client + .decrypt( + &DecryptRequest { + ciphertext, + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("a version record above the current pointer must fail the decrypt"); + assert!( + matches!(&error, KmsError::InternalError { message } if message.contains("behind existing version record")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 2, "the inconsistency must be decided from the two reads: {requests:?}"); + } + + /// Rotation refuses to extend a version history whose records already + /// reach more than one step past the current pointer: that state cannot + /// come from the rotation protocol and re-minting those version numbers + /// would collide with immutable records. + #[tokio::test] + async fn wired_rotate_fails_closed_when_version_history_regressed() { + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(4)), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + // Version records reach 3 while the current pointer says 1. + ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2", "3"] })), + ]) + .await; + + let error = client + .rotate_key("wired-key", None) + .await + .expect_err("a regressed version history must fail the rotation"); + assert!( + matches!(&error, KmsError::InternalError { message } if message.contains("refusing to extend")), + "got {error:?}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "{requests:?}"); + assert!( + !requests.iter().any(|line| line.starts_with("POST ")), + "nothing may be written on a regressed history: {requests:?}" + ); + } + + /// A record exactly one past the current pointer is the footprint of an + /// interrupted rotation; the next rotation must adopt its persisted + /// material (the monotonicity guard must not misread it as a regression). + #[tokio::test] + async fn wired_rotate_adopts_interrupted_rotation_record() { + let mut key_data = healthy_key_data(); + key_data.baseline_version = Some(1); + let adopted_material = rotated_material(); + let record_v2 = VaultKeyVersionRecord { + version: 2, + encrypted_key_material: adopted_material.clone(), + created_at: Zoned::now(), + }; + + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(5)), + ScriptedResponse::ok(kv2_read_data(&key_data)), + // The interrupted rotation left a record for version 2. + ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })), + // The create-only write for version 2 conflicts... + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // ...so the rotation reads the persisted record back and adopts it. + ScriptedResponse::ok(kv2_read_version_record_data(&record_v2)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + let rotated = client + .rotate_key("wired-key", None) + .await + .expect("an interrupted rotation must be recoverable"); + assert_eq!(rotated.version, 2); + + // The pointer switch must commit the adopted (persisted) material, not + // freshly generated material that no record holds. + let committed = parse_write_body(&vault.request_bodies()[5]); + assert_eq!(committed["options"]["cas"], serde_json::json!(5), "{committed}"); + assert_eq!(committed["data"]["version"], serde_json::json!(2), "{committed}"); + assert_eq!( + committed["data"]["encrypted_key_material"], + serde_json::json!(adopted_material), + "{committed}" + ); } } From 8763cd0c67dc641d25c8b67e420c8eeec573a318 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 10:05:33 +0800 Subject: [PATCH 08/12] fix(kms): CAS Transit metadata writes and bound the metadata cache (#5520) * fix(kms): drop the KmsClient trait import removed by #5501 The local backup export tests (#5499) merged after #5501 folded the KmsClient trait into KmsBackend, leaving a dead trait import that breaks 'cargo test -p rustfs-kms' compilation on main. create_key is an inherent method on LocalKmsClient since #5501, so the import is unnecessary. * fix(kms): CAS Transit metadata writes and bound the metadata cache Transit KV metadata writes were whole-record overwrites with no precondition, so two nodes mutating the same key could silently clobber each other's lifecycle state, and the process-local metadata cache had neither a TTL nor a capacity bound, so a key disabled or scheduled for deletion on one node stayed usable on every other node until restart. - Replace write_metadata_to_kv with a versioned read (read_metadata_from_kv_versioned) plus a check-and-set write (cas_write_metadata_to_kv); mutate_key_metadata re-reads the authoritative record and re-runs the state gate on every attempt, and a lost CAS race retries with a fresh snapshot (bounded budget) instead of replaying the stale one. - Migrate every read-modify-write caller: enable, disable, schedule and cancel deletion, rotate version bump, the expired-key tombstone, and both create paths (create-only CAS that read-confirms the winner on a lost race). - Bound the metadata cache with moka (300s TTL, 1024 entries) and drop a key's entry when a transit data call reports it gone server-side. - Fail closed when the synthesized-metadata fallback cannot be read or persisted: the fabricated Enabled record is only served after a durable create-only CAS write, closing the gate weakening documented as a KNOWN RISK; the persistence fallback for pre-metadata keys is kept. Refs rustfs/backlog#1581 (part of rustfs/backlog#1562) --- crates/kms/src/backends/vault_transit.rs | 1107 +++++++++++++++++----- 1 file changed, 878 insertions(+), 229 deletions(-) diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 2b01e8478..2c8db47d7 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -27,25 +27,53 @@ use crate::types::*; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use jiff::Zoned; +use moka::future::Cache; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::sync::Arc; use std::time::Duration; -use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; use tracing::info; use vaultrs::{ + api::kv2::requests::SetSecretRequestOptions, api::transit::{ KeyType, requests::{ CreateKeyRequestBuilder, DecryptDataRequestBuilder, EncryptDataRequestBuilder, UpdateKeyConfigurationRequestBuilder, }, }, + error::ClientError, kv2, transit::{data, key}, }; +/// Attempt budget for metadata read-modify-write cycles: every check-and-set +/// conflict triggers a fresh read plus state-gate re-validation, never a blind +/// replay of the stale snapshot. +const METADATA_CAS_ATTEMPTS: usize = 3; + +/// TTL bound on cached metadata records. This caps how long one node can keep +/// acting on lifecycle state another node has since changed (disable, +/// schedule-deletion): the divergence window is one TTL instead of "until +/// process restart". Matches the manager-level `KmsCache` TTL. +const METADATA_CACHE_TTL: Duration = Duration::from_secs(300); + +/// Capacity bound on the metadata cache so an unbounded key namespace cannot +/// grow process memory without limit. +const METADATA_CACHE_CAPACITY: u64 = 1024; + +/// Whether a KV2 write failed its check-and-set precondition. +/// +/// Mirrors the helper of the same name in `vault.rs`; the two backends keep +/// separate copies because they share no private module. +fn is_cas_conflict(error: &ClientError) -> bool { + matches!( + error, + ClientError::APIError { code: 400, errors } if errors.iter().any(|message| message.contains("check-and-set")) + ) +} + #[derive(Debug, Clone)] struct TransitKeyMetadata { key_usage: KeyUsage, @@ -88,12 +116,16 @@ impl TransitKeyMetadata { } } - // KNOWN RISK (rustfs/backlog#1571, residual of rustfs/backlog#808): this - // fallback defaults to Enabled, so a key whose KV metadata read fails is - // treated as usable — a disabled or pending-deletion key can transiently - // "revive" on that path. State gates therefore only hold as strongly as - // metadata reads do. Changing the fallback is out of scope here; the - // synthesized_metadata_defaults_to_enabled test pins the current behavior. + // Fallback record for transit keys created before metadata persistence + // existed (rustfs#4256 / rustfs#4262): those keys have no KV record at + // all, and failing closed on the missing record would brick every one of + // them, so the record defaults to Enabled to match their pre-persistence + // behavior. The historical fail-open around it (rustfs/backlog#808, + // rustfs/backlog#1571: any metadata read failure yielded a usable Enabled + // key) is resolved for rustfs/backlog#1581: `get_key_metadata` only serves + // this record after durably persisting it with a create-only + // check-and-set, and any read or persist failure on that path fails + // closed. fn synthesized() -> Self { Self { key_usage: KeyUsage::EncryptDecrypt, @@ -148,7 +180,10 @@ pub struct VaultTransitKmsClient { metadata_kv_mount: String, /// Path prefix under metadata_kv_mount for storing transit key metadata records metadata_key_prefix: String, - metadata_cache: RwLock>, + /// Process-local metadata cache, TTL- and capacity-bounded (see + /// [`METADATA_CACHE_TTL`]): a lifecycle change made by another node + /// becomes visible here within one TTL window at the latest. + metadata_cache: Cache, /// Budgets wrapping every outbound Vault call (see `crate::policy`). retry: RetryPolicy, /// Cancellation point for the operation executor: aborts in-flight @@ -179,7 +214,10 @@ impl VaultTransitKmsClient { metadata_kv_mount: config.metadata_kv_mount.clone(), metadata_key_prefix: config.metadata_key_prefix.clone(), config, - metadata_cache: RwLock::new(HashMap::new()), + metadata_cache: Cache::builder() + .max_capacity(METADATA_CACHE_CAPACITY) + .time_to_live(METADATA_CACHE_TTL) + .build(), retry: RetryPolicy::from_config(kms_config), cancel: CancellationToken::new(), }) @@ -276,11 +314,7 @@ impl VaultTransitKmsClient { } data::encrypt(&vault.client, &self.config.mount_path, key_id, plaintext_b64, Some(&mut builder)) .await - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to encrypt data with Vault Transit key {key_id}: {e}")) - }) - }) + .map_err(|e| AttemptError::from_vaultrs(e, |e| Self::map_vault_error(key_id, e, "encrypt"))) }) .await?; @@ -305,11 +339,7 @@ impl VaultTransitKmsClient { } data::decrypt(&vault.client, &self.config.mount_path, key_id, ciphertext, Some(&mut builder)) .await - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to decrypt data with Vault Transit key {key_id}: {e}")) - }) - }) + .map_err(|e| AttemptError::from_vaultrs(e, |e| Self::map_vault_error(key_id, e, "decrypt"))) }) .await?; @@ -339,28 +369,87 @@ impl VaultTransitKmsClient { .await } - async fn write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { + /// Read the persisted metadata record together with the KV2 secret version + /// holding it, so a later write can be check-and-set against exactly this + /// snapshot. `None` means no record exists (a pre-persistence key). + async fn read_metadata_from_kv_versioned(&self, key_id: &str) -> Result> { + let path = self.metadata_key_path(key_id); + let path = path.as_str(); + + let kv_metadata = self + .run("vault_transit_read_metadata_version", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::read_metadata(&vault.client, &self.metadata_kv_mount, path).await { + Ok(metadata) => Ok(Some(metadata)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to read transit key metadata version from Vault KV: {e}")) + })), + } + }) + .await?; + let Some(kv_metadata) = kv_metadata else { + return Ok(None); + }; + let cas = u32::try_from(kv_metadata.current_version) + .map_err(|_| KmsError::backend_error(format!("KV2 secret version for transit key {key_id} metadata exceeds u32")))?; + + // Read the exact secret version named by the metadata so the + // (cas, record) pair stays consistent even if another writer lands in + // between the two reads. + let secret_version = kv_metadata.current_version; + let record: Option = self + .run("vault_transit_read_metadata_at_version", OpClass::ReadIdempotent, move || async move { + let vault = self.vault().map_err(AttemptError::fatal)?; + match kv2::read_version(&vault.client, &self.metadata_kv_mount, path, secret_version).await { + Ok(persisted) => Ok(Some(persisted)), + Err(ClientError::ResponseWrapError) | Err(ClientError::APIError { code: 404, .. }) => Ok(None), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to read transit key metadata from Vault KV: {e}")) + })), + } + }) + .await?; + + Ok(record.map(|persisted| (cas, persisted.into()))) + } + + /// Check-and-set write of the metadata record. + /// + /// `cas` must match the KV2 secret version currently holding the record + /// (0 = create-only). Returns `Ok(false)` when the precondition failed — a + /// concurrent writer landed first — so the caller re-reads instead of + /// clobbering. Single attempt: replaying a lost-response write would + /// double-apply the mutation, and a CAS conflict is a normal concurrency + /// signal, not a backend failure. + async fn cas_write_metadata_to_kv(&self, key_id: &str, metadata: &TransitKeyMetadata, cas: u32) -> Result { let path = self.metadata_key_path(key_id); let path = path.as_str(); let persisted: TransitKeyMetadataPersisted = metadata.clone().into(); let persisted = &persisted; - // Single attempt: this is a whole-record overwrite without a CAS - // precondition, so a replay after a lost response could clobber a - // concurrent writer. - self.run("vault_transit_write_metadata", OpClass::MutatingNonIdempotent, move || async move { + self.run("vault_transit_cas_write_metadata", OpClass::MutatingNonIdempotent, move || async move { let vault = self.vault().map_err(AttemptError::fatal)?; - kv2::set(&vault.client, &self.metadata_kv_mount, path, persisted) + match kv2::set_with_options(&vault.client, &self.metadata_kv_mount, path, persisted, SetSecretRequestOptions { cas }) .await - .map(|_| ()) - .map_err(|e| { - AttemptError::from_vaultrs(e, |e| { - KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}")) - }) - }) + { + Ok(_) => Ok(true), + Err(e) if is_cas_conflict(&e) => Ok(false), + Err(e) => Err(AttemptError::from_vaultrs(e, |e| { + KmsError::backend_error(format!("Failed to write transit key metadata to Vault KV: {e}")) + })), + } }) .await } + /// The error surfaced when a metadata read-modify-write exhausts its + /// [`METADATA_CAS_ATTEMPTS`] budget without winning a check-and-set write. + fn metadata_cas_conflict(key_id: &str) -> KmsError { + KmsError::invalid_operation(format!( + "Concurrent modification of transit key {key_id} metadata detected; retry the operation" + )) + } + async fn delete_metadata_from_kv(&self, key_id: &str) -> Result<()> { let path = self.metadata_key_path(key_id); let path = path.as_str(); @@ -414,45 +503,103 @@ impl VaultTransitKmsClient { } async fn get_key_metadata(&self, key_id: &str) -> Result { - // Check in-memory cache first. - if let Some(metadata) = self.metadata_cache.read().await.get(key_id).cloned() { + // Check in-memory cache first (TTL-bounded, so a stale entry can only + // survive one TTL window). + if let Some(metadata) = self.metadata_cache.get(key_id).await { return Ok(metadata); } - // On cache miss, try reading from the persistent KV store. - if let Some(persisted) = self.read_metadata_from_kv(key_id).await? { - self.metadata_cache - .write() - .await - .insert(key_id.to_string(), persisted.clone()); - return Ok(persisted); - } + for _ in 0..METADATA_CAS_ATTEMPTS { + // On cache miss, try reading from the persistent KV store. + if let Some(persisted) = self.read_metadata_from_kv(key_id).await? { + self.metadata_cache.insert(key_id.to_string(), persisted.clone()).await; + return Ok(persisted); + } - // Deliberate exemption from the "read paths never write" rule (rustfs#4256 / - // rustfs#4262): transit keys created before metadata persistence existed have no - // KV record at all, so failing closed here would brick every pre-existing transit - // key. The synthesised record only describes metadata — key material lives solely - // inside Vault's transit engine and is never generated or written by this path. - // - // Verify the transit key actually exists in Vault before synthesising. - self.read_transit_key(key_id).await?; - let metadata = TransitKeyMetadata::synthesized(); - // Persist the synthesised metadata so future cache misses pick it up (best - // effort: the KV write failing must not fail the read). - let _ = self.write_metadata_to_kv(key_id, &metadata).await; - self.metadata_cache.write().await.insert(key_id.to_string(), metadata.clone()); - Ok(metadata) + // Deliberate exemption from the "read paths never write" rule (rustfs#4256 / + // rustfs#4262): transit keys created before metadata persistence existed have no + // KV record at all, so failing closed here would brick every pre-existing transit + // key. The synthesised record only describes metadata — key material lives solely + // inside Vault's transit engine and is never generated or written by this path. + // + // Verify the transit key actually exists in Vault before synthesising. + self.read_transit_key(key_id).await?; + let metadata = TransitKeyMetadata::synthesized(); + // Fail closed on the persist (rustfs/backlog#1581): the synthesised record is + // only served once it is durable, so every node gates on the same stored + // state; a failed KV write must fail the read instead of minting a usable + // Enabled record out of thin air. The create-only check-and-set keeps two + // nodes from fabricating divergent records — losing that race loops back to + // re-read the winner's record. + if self.cas_write_metadata_to_kv(key_id, &metadata, 0).await? { + self.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + return Ok(metadata); + } + } + Err(Self::metadata_cas_conflict(key_id)) } - async fn store_key_metadata(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result<()> { - self.write_metadata_to_kv(key_id, metadata).await?; - self.metadata_cache.write().await.insert(key_id.to_string(), metadata.clone()); - Ok(()) + /// Create-only write of the metadata record (check-and-set of 0). + /// + /// Returns `Ok(false)` when a record already exists — a concurrent creator + /// won the race — and never overwrites it; the caller reconciles by + /// reading the stored record back. + async fn create_key_metadata(&self, key_id: &str, metadata: &TransitKeyMetadata) -> Result { + if self.cas_write_metadata_to_kv(key_id, metadata, 0).await? { + self.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + return Ok(true); + } + Ok(false) + } + + /// Read-modify-write of the persisted metadata record under KV2 + /// check-and-set. + /// + /// Every attempt re-reads the authoritative record, re-runs `apply` — + /// which owns state-gate validation — against that fresh snapshot, and + /// writes back with the snapshot's KV2 secret version as the check-and-set + /// precondition, so a concurrent writer is never clobbered blind. Losing + /// the race drops the (now stale) cache entry and retries with a fresh + /// read; exhausting the budget surfaces the conflict to the caller. + async fn mutate_key_metadata(&self, key_id: &str, mut apply: F) -> Result + where + F: FnMut(&mut TransitKeyMetadata) -> Result<()>, + { + for _ in 0..METADATA_CAS_ATTEMPTS { + let (cas, mut metadata) = match self.read_metadata_from_kv_versioned(key_id).await? { + Some(snapshot) => snapshot, + None => { + // Pre-persistence key without a KV record (see + // get_key_metadata): mutate the synthesised record and + // create it with a create-only check-and-set so two nodes + // cannot fabricate divergent records. + self.read_transit_key(key_id).await?; + (0, TransitKeyMetadata::synthesized()) + } + }; + apply(&mut metadata)?; + if self.cas_write_metadata_to_kv(key_id, &metadata, cas).await? { + self.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + return Ok(metadata); + } + self.metadata_cache.invalidate(key_id).await; + } + Err(Self::metadata_cas_conflict(key_id)) + } + + /// Drop the cached metadata record when a transit data-path call failed in + /// a way that signals the cached lifecycle state diverged from Vault (the + /// key is gone server-side), so the next state gate re-reads the + /// authoritative record instead of trusting the stale entry until its TTL. + async fn invalidate_metadata_on_state_error(&self, key_id: &str, error: &KmsError) { + if matches!(error, KmsError::KeyNotFound { .. }) { + self.metadata_cache.invalidate(key_id).await; + } } async fn delete_key_metadata(&self, key_id: &str) -> Result<()> { self.delete_metadata_from_kv(key_id).await?; - self.metadata_cache.write().await.remove(key_id); + self.metadata_cache.invalidate(key_id).await; Ok(()) } @@ -514,9 +661,16 @@ impl VaultTransitKmsClient { .await?; let plaintext_key = generate_key_material(&request.key_spec)?; - let encrypted_key = self + let encrypted_key = match self .transit_encrypt(&request.master_key_id, &plaintext_key, &request.encryption_context) - .await?; + .await + { + Ok(encrypted_key) => encrypted_key, + Err(error) => { + self.invalidate_metadata_on_state_error(&request.master_key_id, &error).await; + return Err(error); + } + }; let envelope = DataKeyEnvelope { key_id: uuid::Uuid::new_v4().to_string(), @@ -545,9 +699,16 @@ impl VaultTransitKmsClient { let metadata = self .ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt) .await?; - let ciphertext = self + let ciphertext = match self .transit_encrypt(&request.key_id, &request.plaintext, &request.encryption_context) - .await?; + .await + { + Ok(ciphertext) => ciphertext, + Err(error) => { + self.invalidate_metadata_on_state_error(&request.key_id, &error).await; + return Err(error); + } + }; Ok(EncryptResponse { ciphertext: ciphertext.into_bytes(), @@ -575,8 +736,16 @@ impl VaultTransitKmsClient { let encrypted_key = std::str::from_utf8(&envelope.encrypted_key) .map_err(|e| KmsError::cryptographic_error("utf8", format!("Invalid Transit ciphertext: {e}")))?; - self.transit_decrypt(&envelope.master_key_id, encrypted_key, &envelope.encryption_context) + match self + .transit_decrypt(&envelope.master_key_id, encrypted_key, &envelope.encryption_context) .await + { + Ok(plaintext) => Ok(plaintext), + Err(error) => { + self.invalidate_metadata_on_state_error(&envelope.master_key_id, &error).await; + Err(error) + } + } } /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. @@ -598,59 +767,68 @@ impl VaultTransitKmsClient { // this create would have produced; report it as the create result. // Anything else keeps failing. A failed pre-check read must fail the // create rather than fall through to re-creating over an unknown key. - match self.read_transit_key(key_id).await { - Ok(_) => { - let existing = self.get_key_metadata(key_id).await?; - return if existing.key_state == KeyState::Enabled && existing.key_usage == KeyUsage::EncryptDecrypt { - info!( - key_id, - "Vault Transit create found an identical enabled key; treating it as a recovered create" - ); - Ok(MasterKeyInfo { - key_id: key_id.to_string(), - version: existing.current_version, - algorithm: algorithm.to_string(), - usage: existing.key_usage, - status: KeyStatus::Active, - description: existing.description, - metadata: existing.tags.clone(), - created_at: existing.created_at, - rotated_at: None, - created_by: existing.created_by, - deletion_date: None, - }) - } else { - Err(KmsError::key_already_exists(key_id)) - }; + // + // Two passes: losing the create-only metadata check-and-set race loops + // back here so the pre-check read-confirms the winning record. + for _ in 0..2 { + match self.read_transit_key(key_id).await { + Ok(_) => { + let existing = self.get_key_metadata(key_id).await?; + return if existing.key_state == KeyState::Enabled && existing.key_usage == KeyUsage::EncryptDecrypt { + info!( + key_id, + "Vault Transit create found an identical enabled key; treating it as a recovered create" + ); + Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: existing.current_version, + algorithm: algorithm.to_string(), + usage: existing.key_usage, + status: KeyStatus::Active, + description: existing.description, + metadata: existing.tags.clone(), + created_at: existing.created_at, + rotated_at: None, + created_by: existing.created_by, + deletion_date: None, + }) + } else { + Err(KmsError::key_already_exists(key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } - Err(KmsError::KeyNotFound { .. }) => {} - Err(error) => return Err(error), + + self.create_transit_key(key_id).await?; + + let metadata = TransitKeyMetadata { + created_by: Some("vault-transit".to_string()), + ..TransitKeyMetadata::from_create_request(&CreateKeyRequest { + key_name: Some(key_id.to_string()), + ..Default::default() + }) + }; + if self.create_key_metadata(key_id, &metadata).await? { + return Ok(MasterKeyInfo { + key_id: key_id.to_string(), + version: metadata.current_version, + algorithm: algorithm.to_string(), + usage: metadata.key_usage, + status: KeyStatus::Active, + description: metadata.description, + metadata: metadata.tags, + created_at: metadata.created_at, + rotated_at: None, + created_by: metadata.created_by, + deletion_date: None, + }); + } + // A concurrent creator persisted metadata first; make sure the + // pre-check reads their record, not a stale cache entry. + self.metadata_cache.invalidate(key_id).await; } - - self.create_transit_key(key_id).await?; - - let metadata = TransitKeyMetadata { - created_by: Some("vault-transit".to_string()), - ..TransitKeyMetadata::from_create_request(&CreateKeyRequest { - key_name: Some(key_id.to_string()), - ..Default::default() - }) - }; - self.store_key_metadata(key_id, &metadata).await?; - - Ok(MasterKeyInfo { - key_id: key_id.to_string(), - version: metadata.current_version, - algorithm: algorithm.to_string(), - usage: metadata.key_usage, - status: KeyStatus::Active, - description: metadata.description, - metadata: metadata.tags, - created_at: metadata.created_at, - rotated_at: None, - created_by: metadata.created_by, - deletion_date: None, - }) + Err(KmsError::key_already_exists(key_id)) } /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. @@ -708,17 +886,26 @@ impl VaultTransitKmsClient { pub(crate) async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { // A pending deletion must be reverted through cancel_key_deletion, not - // silently by enabling, so the gate rejects PendingDeletion here. - let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Enable).await?; - metadata.key_state = KeyState::Enabled; - metadata.deletion_date = None; - self.store_key_metadata(key_id, &metadata).await + // silently by enabling, so the gate rejects PendingDeletion here. The + // gate runs inside the check-and-set loop against every fresh snapshot. + self.mutate_key_metadata(key_id, |metadata| { + ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::Enable)?; + metadata.key_state = KeyState::Enabled; + metadata.deletion_date = None; + Ok(()) + }) + .await + .map(|_| ()) } pub(crate) async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> { - let mut metadata = self.ensure_key_state_allows(key_id, StateGatedOperation::Disable).await?; - metadata.key_state = KeyState::Disabled; - self.store_key_metadata(key_id, &metadata).await + self.mutate_key_metadata(key_id, |metadata| { + ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::Disable)?; + metadata.key_state = KeyState::Disabled; + Ok(()) + }) + .await + .map(|_| ()) } /// Test-only lifecycle driver: the product path goes through [`KmsBackend`]. @@ -729,12 +916,15 @@ impl VaultTransitKmsClient { pending_window_days: u32, _context: Option<&OperationContext>, ) -> Result<()> { - let mut metadata = self - .ensure_key_state_allows(key_id, StateGatedOperation::ScheduleDeletion) - .await?; - metadata.key_state = KeyState::PendingDeletion; - metadata.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400)); - self.store_key_metadata(key_id, &metadata).await + let deletion_date = Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400); + self.mutate_key_metadata(key_id, |metadata| { + ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?; + metadata.key_state = KeyState::PendingDeletion; + metadata.deletion_date = Some(deletion_date.clone()); + Ok(()) + }) + .await + .map(|_| ()) } pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result { @@ -755,9 +945,15 @@ impl VaultTransitKmsClient { }) .await?; - let mut metadata = self.get_key_metadata(key_id).await?; - metadata.current_version += 1; - self.store_key_metadata(key_id, &metadata).await?; + let metadata = self + .mutate_key_metadata(key_id, |metadata| { + // The transit rotation above has already happened; recording + // the version bump must not be blocked by a concurrent + // lifecycle transition, so no state gate here. + metadata.current_version += 1; + Ok(()) + }) + .await?; Ok(MasterKeyInfo { key_id: key_id.to_string(), @@ -788,6 +984,15 @@ impl VaultTransitKmsClient { } } +#[cfg(test)] +impl VaultTransitKmsClient { + /// Rebuild the metadata cache with test-controlled bounds so TTL and + /// capacity behavior can be exercised without real sleeps. + fn rebuild_metadata_cache_for_tests(&mut self, capacity: u64, ttl: Duration) { + self.metadata_cache = Cache::builder().max_capacity(capacity).time_to_live(ttl).build(); + } +} + pub struct VaultTransitKmsBackend { client: VaultTransitKmsClient, } @@ -835,59 +1040,68 @@ impl KmsBackend for VaultTransitKmsBackend { // what this request would have written, report it as the create // result; any divergence keeps failing so a create can never adopt or // reshape a key it would not have produced. - match self.client.read_transit_key(&key_id).await { - Ok(_) => { - let existing = self.client.get_key_metadata(&key_id).await?; - let requested = TransitKeyMetadata::from_create_request(&request); - return if existing.key_state == KeyState::Enabled - && existing.key_usage == requested.key_usage - && existing.description == requested.description - && existing.tags == requested.tags - { - info!( - key_id, - "Vault Transit create found an identical enabled key; treating it as a recovered create" - ); - Ok(CreateKeyResponse { - key_id: key_id.clone(), - key_metadata: KeyMetadata { + // + // Two passes: losing the create-only metadata check-and-set race loops + // back here so the pre-check read-confirms the winning record. + for _ in 0..2 { + match self.client.read_transit_key(&key_id).await { + Ok(_) => { + let existing = self.client.get_key_metadata(&key_id).await?; + let requested = TransitKeyMetadata::from_create_request(&request); + return if existing.key_state == KeyState::Enabled + && existing.key_usage == requested.key_usage + && existing.description == requested.description + && existing.tags == requested.tags + { + info!( key_id, - key_state: existing.key_state, - key_usage: existing.key_usage, - description: existing.description, - creation_date: existing.created_at, - deletion_date: existing.deletion_date, - origin: existing.origin, - key_manager: "VAULT_TRANSIT".to_string(), - tags: existing.tags, - }, - }) - } else { - Err(KmsError::key_already_exists(&key_id)) - }; + "Vault Transit create found an identical enabled key; treating it as a recovered create" + ); + Ok(CreateKeyResponse { + key_id: key_id.clone(), + key_metadata: KeyMetadata { + key_id, + key_state: existing.key_state, + key_usage: existing.key_usage, + description: existing.description, + creation_date: existing.created_at, + deletion_date: existing.deletion_date, + origin: existing.origin, + key_manager: "VAULT_TRANSIT".to_string(), + tags: existing.tags, + }, + }) + } else { + Err(KmsError::key_already_exists(&key_id)) + }; + } + Err(KmsError::KeyNotFound { .. }) => {} + Err(error) => return Err(error), } - Err(KmsError::KeyNotFound { .. }) => {} - Err(error) => return Err(error), + + self.client.create_transit_key(&key_id).await?; + let metadata = TransitKeyMetadata::from_create_request(&request); + if self.client.create_key_metadata(&key_id, &metadata).await? { + return Ok(CreateKeyResponse { + key_id: key_id.clone(), + key_metadata: KeyMetadata { + key_id, + key_state: metadata.key_state, + key_usage: metadata.key_usage, + description: metadata.description, + creation_date: metadata.created_at, + deletion_date: metadata.deletion_date, + origin: metadata.origin, + key_manager: "VAULT_TRANSIT".to_string(), + tags: metadata.tags, + }, + }); + } + // A concurrent creator persisted metadata first; make sure the + // pre-check reads their record, not a stale cache entry. + self.client.metadata_cache.invalidate(&key_id).await; } - - self.client.create_transit_key(&key_id).await?; - let metadata = TransitKeyMetadata::from_create_request(&request); - self.client.store_key_metadata(&key_id, &metadata).await?; - - Ok(CreateKeyResponse { - key_id: key_id.clone(), - key_metadata: KeyMetadata { - key_id, - key_state: metadata.key_state, - key_usage: metadata.key_usage, - description: metadata.description, - creation_date: metadata.created_at, - deletion_date: metadata.deletion_date, - origin: metadata.origin, - key_manager: "VAULT_TRANSIT".to_string(), - tags: metadata.tags, - }, - }) + Err(KmsError::key_already_exists(&key_id)) } async fn encrypt(&self, request: EncryptRequest) -> Result { @@ -946,10 +1160,14 @@ impl KmsBackend for VaultTransitKmsBackend { self.client.delete_key_metadata(&key_id).await?; None } else { - let mut metadata = self.client.get_key_metadata(&key_id).await?; - metadata.key_state = KeyState::PendingDeletion; - metadata.deletion_date = Some(Zoned::now()); - self.client.store_key_metadata(&key_id, &metadata).await?; + let now = Zoned::now(); + self.client + .mutate_key_metadata(&key_id, |metadata| { + metadata.key_state = KeyState::PendingDeletion; + metadata.deletion_date = Some(now.clone()); + Ok(()) + }) + .await?; key_metadata = self.client.key_metadata_response(&key_id).await?; None } @@ -961,11 +1179,17 @@ impl KmsBackend for VaultTransitKmsBackend { return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30")); } - let mut metadata = self.client.get_key_metadata(&key_id).await?; let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400); - metadata.key_state = KeyState::PendingDeletion; - metadata.deletion_date = Some(scheduled.clone()); - self.client.store_key_metadata(&key_id, &metadata).await?; + self.client + .mutate_key_metadata(&key_id, |metadata| { + // Re-run the gate against every fresh snapshot: the check + // above used a possibly cached record. + ensure_key_state_permits(&key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?; + metadata.key_state = KeyState::PendingDeletion; + metadata.deletion_date = Some(scheduled.clone()); + Ok(()) + }) + .await?; key_metadata = self.client.key_metadata_response(&key_id).await?; Some(scheduled.to_string()) }; @@ -978,14 +1202,20 @@ impl KmsBackend for VaultTransitKmsBackend { } async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result { - let mut metadata = self.client.get_key_metadata(&request.key_id).await?; - if metadata.key_state != KeyState::PendingDeletion { - return Err(KmsError::invalid_key_state(format!("Key {} is not pending deletion", request.key_id))); - } - - metadata.key_state = KeyState::Enabled; - metadata.deletion_date = None; - self.client.store_key_metadata(&request.key_id, &metadata).await?; + let key_id = request.key_id.as_str(); + self.client + .mutate_key_metadata(key_id, |metadata| { + // Re-checked against every fresh snapshot: a concurrent sweep + // that tombstoned the key must fail this cancel, not be + // overwritten blind. + if metadata.key_state != KeyState::PendingDeletion { + return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion"))); + } + metadata.key_state = KeyState::Enabled; + metadata.deletion_date = None; + Ok(()) + }) + .await?; Ok(CancelKeyDeletionResponse { key_id: request.key_id.clone(), @@ -1033,27 +1263,51 @@ impl KmsBackend for VaultTransitKmsBackend { Err(error) => return Err(error), } - // A metadata read failure synthesizes an Enabled record (see - // TransitKeyMetadata::synthesized), which lands in StateChanged below: - // the worker never destroys material based on synthesized state. - let mut metadata = self.client.get_key_metadata(key_id).await?; - match metadata.key_state { - // Tombstone left by a crashed removal: complete it. - KeyState::Unavailable => {} - KeyState::PendingDeletion => { - match &metadata.deletion_date { - Some(deadline) if deadline <= now => {} - // Not yet due, or no persisted deadline — never auto-remove. - _ => return Ok(ExpiredKeyRemoval::NotExpired), - } - // Tombstone first: an Unavailable record is rejected by every - // state gate, and a crashed removal can simply be re-run. - metadata.key_state = KeyState::Unavailable; - self.client.store_key_metadata(key_id, &metadata).await?; - } - KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => { + // Tombstone under check-and-set: every attempt re-reads the record and + // re-validates state and due-ness, so a cancel_key_deletion racing the + // sweep either lands before the tombstone (the re-read sees Enabled + // and the sweep backs off) or after it (the cancel's own + // check-and-set write fails). + let mut tombstoned = false; + for _ in 0..METADATA_CAS_ATTEMPTS { + let Some((cas, mut metadata)) = self.client.read_metadata_from_kv_versioned(key_id).await? else { + // No persisted lifecycle record (pre-persistence key): the + // worker never destroys material whose scheduling state was + // never recorded. return Ok(ExpiredKeyRemoval::StateChanged); + }; + match metadata.key_state { + // Tombstone left by a crashed removal: complete it. + KeyState::Unavailable => { + tombstoned = true; + } + KeyState::PendingDeletion => { + match &metadata.deletion_date { + Some(deadline) if deadline <= now => {} + // Not yet due, or no persisted deadline — never auto-remove. + _ => return Ok(ExpiredKeyRemoval::NotExpired), + } + // Tombstone first: an Unavailable record is rejected by every + // state gate, and a crashed removal can simply be re-run. + metadata.key_state = KeyState::Unavailable; + if self.client.cas_write_metadata_to_kv(key_id, &metadata, cas).await? { + self.client.metadata_cache.insert(key_id.to_string(), metadata.clone()).await; + tombstoned = true; + } else { + // Lost the check-and-set race — most likely a + // concurrent cancel; re-read and re-decide. + self.client.metadata_cache.invalidate(key_id).await; + continue; + } + } + KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => { + return Ok(ExpiredKeyRemoval::StateChanged); + } } + break; + } + if !tombstoned { + return Err(VaultTransitKmsClient::metadata_cas_conflict(key_id)); } if !self.client.read_transit_key(key_id).await?.deletion_allowed { @@ -1432,10 +1686,14 @@ mod tests { let _ = client.schedule_key_deletion(&key_id, 7, None).await; } - /// Pins the known-risk fallback documented on `TransitKeyMetadata::synthesized`: - /// when KV metadata cannot be read, the synthesized record defaults to Enabled, - /// which weakens every state gate on that path. If this test turns red the - /// fallback semantics changed on purpose — update the comment there as well. + /// The persistence fallback for pre-metadata keys deliberately fabricates + /// an Enabled record (rustfs#4256 / rustfs#4262): those keys were usable + /// before metadata persistence existed and must stay usable once the + /// record is durably persisted. The old fail-open this test used to pin — + /// a failed metadata read or persist still yielded a usable Enabled key — + /// was flipped to fail closed for rustfs/backlog#1581; that side is + /// covered by `wired_encrypt_fails_closed_when_the_metadata_read_fails` + /// and `wired_synthesized_metadata_is_not_served_when_the_persist_fails`. #[test] fn synthesized_metadata_defaults_to_enabled() { let metadata = TransitKeyMetadata::synthesized(); @@ -1457,15 +1715,21 @@ mod tests { #[tokio::test] async fn wired_backend_lifecycle_overrides_reach_the_client() { let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; let vault = ScriptedVault::serve(vec![ - // disable: metadata cache miss reads KV, then persists Disabled. + // disable: versioned read (secret metadata + pinned version), then + // the check-and-set write persisting Disabled. + ScriptedResponse::ok(kv2_metadata_read_data(1)), ScriptedResponse::ok(metadata_read_data(&metadata)), ScriptedResponse::ok(kv2_write_ack()), - // enable: the state gate hits the metadata cache, so only the - // persisting write goes out. + // enable: another versioned read against the Disabled record, then + // the check-and-set write persisting Enabled. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&disabled)), ScriptedResponse::ok(kv2_write_ack()), - // rotate: the gate hits the cache again; the single rotate - // attempt fails and must not be retried. + // rotate: the state gate hits the metadata cache; the single + // rotate attempt fails and must not be retried. ScriptedResponse::error(503, "standby"), ]) .await; @@ -1493,7 +1757,392 @@ mod tests { assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); let requests = vault.requests(); - assert_eq!(requests.len(), 4, "gated reads, two writes and one rotate attempt: {requests:?}"); - assert_eq!(requests[3], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); + assert_eq!(requests.len(), 7, "two versioned read+write cycles plus one rotate attempt: {requests:?}"); + assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); + } + + /// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the + /// current secret version used as the check-and-set base. + fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value { + serde_json::json!({ + "cas_required": false, + "created_time": "2026-01-01T00:00:00Z", + "current_version": current_version, + "delete_version_after": "0s", + "max_versions": 0, + "oldest_version": 0, + "updated_time": "2026-01-01T00:00:00Z", + "custom_metadata": null, + "versions": {}, + }) + } + + const CAS_CONFLICT_MESSAGE: &str = "check-and-set parameter did not match the current version"; + + const METADATA_PATH: &str = "/v1/secret/data/rustfs/kms/transit-metadata/wired-key"; + const METADATA_VERSION_PATH: &str = "/v1/secret/metadata/rustfs/kms/transit-metadata/wired-key"; + + #[tokio::test] + async fn wired_disable_retries_past_a_cas_conflict_with_a_fresh_read() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // The conflict must trigger a fresh versioned read, then the + // write is check-and-set against the new snapshot. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::ok(kv2_write_ack()), + ]) + .await; + + client + .disable_key("wired-key", None) + .await + .expect("a single check-and-set conflict must be absorbed by a re-read"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 6, "two read+read+write cycles: {requests:?}"); + assert_eq!(requests[0], format!("GET {METADATA_VERSION_PATH}")); + assert_eq!(requests[1], format!("GET {METADATA_PATH}?version=1")); + assert_eq!(requests[2], format!("POST {METADATA_PATH}")); + assert_eq!(requests[3], format!("GET {METADATA_VERSION_PATH}"), "conflict must re-read: {requests:?}"); + assert_eq!(requests[4], format!("GET {METADATA_PATH}?version=2")); + assert_eq!(requests[5], format!("POST {METADATA_PATH}")); + } + + #[tokio::test] + async fn wired_disable_cas_conflict_budget_is_bounded() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut responses = Vec::new(); + for cycle in 0..3u64 { + responses.push(ScriptedResponse::ok(kv2_metadata_read_data(cycle + 1))); + responses.push(ScriptedResponse::ok(metadata_read_data(&enabled))); + responses.push(ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE)); + } + let (vault, client) = scripted_client(responses).await; + + let error = client + .disable_key("wired-key", None) + .await + .expect_err("exhausting the check-and-set budget must surface the conflict"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!( + error.to_string().contains("Concurrent modification"), + "the error must name the conflict: {error}" + ); + + let requests = vault.requests(); + assert_eq!(requests.len(), 9, "exactly three read+read+write cycles, no blind replays: {requests:?}"); + } + + #[tokio::test] + async fn wired_cas_conflict_reread_revalidates_the_state_gate() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + pending.key_state = KeyState::PendingDeletion; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // The concurrent writer scheduled the key for deletion; the + // re-read must re-run the state gate and reject the disable. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&pending)), + ]) + .await; + + let error = client + .disable_key("wired-key", None) + .await + .expect_err("the re-read state gate must reject a pending-deletion key"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("pending deletion"), "got {error}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 5, "the gate rejection must not issue another write: {requests:?}"); + assert!(requests[4].starts_with("GET "), "{requests:?}"); + } + + #[tokio::test] + async fn wired_encrypt_key_not_found_invalidates_the_cached_metadata() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; + let (vault, client) = scripted_client(vec![ + // First encrypt: gate reads Enabled and caches it, then the + // transit call reports the key gone server-side. + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::error(404, "encryption key not found"), + // Second encrypt: the state error must have dropped the cache + // entry, so the gate re-reads and sees the Disabled record. + ScriptedResponse::ok(metadata_read_data(&disabled)), + ]) + .await; + + let request = EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }; + let error = client + .encrypt(&request, None) + .await + .expect_err("the scripted 404 must fail the encrypt"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}"); + + let error = client + .encrypt(&request, None) + .await + .expect_err("the re-read Disabled record must reject the encrypt"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!( + requests.len(), + 3, + "the second gate must re-read instead of trusting the stale Enabled entry, \ + and must not reach the encrypt endpoint: {requests:?}" + ); + assert_eq!(requests[2], format!("GET {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn wired_metadata_cache_ttl_expiry_forces_a_fresh_read() { + let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; + let (vault, mut client) = scripted_client(vec![ + ScriptedResponse::ok(metadata_read_data(&enabled)), + ScriptedResponse::ok(serde_json::json!({ "ciphertext": "vault:v1:scripted" })), + // Post-expiry gate read observes the disable another node + // persisted in the meantime. + ScriptedResponse::ok(metadata_read_data(&disabled)), + ]) + .await; + // A 1ns TTL expires between any two awaits, standing in for the real + // 300s bound without a wall-clock sleep. + client.rebuild_metadata_cache_for_tests(METADATA_CACHE_CAPACITY, Duration::from_nanos(1)); + + let request = EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }; + client + .encrypt(&request, None) + .await + .expect("the first encrypt must pass the Enabled gate"); + + let error = client + .encrypt(&request, None) + .await + .expect_err("after TTL expiry the gate must see the remote disable"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "the expired entry must force a fresh KV read: {requests:?}"); + assert_eq!(requests[2], format!("GET {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn metadata_cache_capacity_is_bounded() { + let records: Vec<_> = (0..3) + .map(|_| { + ScriptedResponse::ok(metadata_read_data(&TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()))) + }) + .collect(); + let (_vault, mut client) = scripted_client(records).await; + client.rebuild_metadata_cache_for_tests(2, METADATA_CACHE_TTL); + + for key_id in ["key-a", "key-b", "key-c"] { + client + .get_key_metadata(key_id) + .await + .expect("each scripted metadata read must succeed"); + } + + client.metadata_cache.run_pending_tasks().await; + assert!( + client.metadata_cache.entry_count() <= 2, + "the cache must not hold more entries than its capacity, got {}", + client.metadata_cache.entry_count() + ); + } + + #[tokio::test] + async fn wired_encrypt_fails_closed_when_the_metadata_read_fails() { + let (vault, client) = scripted_client(vec![ScriptedResponse::error(403, "permission denied")]).await; + + let error = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("a failed metadata read must fail the encrypt, not synthesize Enabled"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 1, "the gate failure must never reach the encrypt endpoint: {requests:?}"); + } + + #[tokio::test] + async fn wired_synthesized_metadata_is_not_served_when_the_persist_fails() { + // Regression for the rustfs/backlog#1581 fail-open flip: a missing + // metadata record used to synthesize a usable Enabled record even when + // persisting it failed, letting encrypt proceed on state no other node + // could observe. The persist failure must now fail the read. + let (vault, client) = scripted_client(vec![ + ScriptedResponse::error(404, "no value found"), + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::error(500, "kv write failed"), + ]) + .await; + + let error = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("an unpersisted synthesized record must never gate an encrypt open"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 3, "read, existence check, failed persist — and no encrypt: {requests:?}"); + assert_eq!(requests[2], format!("POST {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn wired_synthesized_metadata_create_race_adopts_the_winning_record() { + let mut disabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + disabled.key_state = KeyState::Disabled; + let (vault, client) = scripted_client(vec![ + ScriptedResponse::error(404, "no value found"), + ScriptedResponse::ok(transit_key_read_data("wired-key")), + // Another node persisted a record first; the create-only + // check-and-set loses and the re-read adopts the winner. + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + ScriptedResponse::ok(metadata_read_data(&disabled)), + ]) + .await; + + let error = client + .encrypt( + &EncryptRequest { + key_id: "wired-key".to_string(), + plaintext: b"plaintext".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }, + None, + ) + .await + .expect_err("the winner's Disabled record must gate the encrypt, not the loser's Enabled one"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let requests = vault.requests(); + assert_eq!(requests.len(), 4, "the lost create race must re-read, never overwrite: {requests:?}"); + assert_eq!(requests[3], format!("GET {METADATA_PATH}"), "{requests:?}"); + } + + #[tokio::test] + async fn wired_backend_create_loses_the_metadata_create_race_and_read_confirms() { + let winner = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // Pre-check: the transit key does not exist yet. + ScriptedResponse::error(404, "not found"), + // Transit create succeeds, but a concurrent creator persists the + // metadata record first. + ScriptedResponse::ok(serde_json::json!({})), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // Second pass: the pre-check now read-confirms the winner. + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::ok(metadata_read_data(&winner)), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + let response = backend + .create_key(CreateKeyRequest { + key_name: Some("wired-key".to_string()), + ..Default::default() + }) + .await + .expect("losing the metadata create race to an identical record must recover the create"); + assert_eq!(response.key_metadata.key_state, KeyState::Enabled); + + let requests = vault.requests(); + assert_eq!(requests.len(), 5, "one lost create pass plus one read-confirm pass: {requests:?}"); + assert!( + requests[3].starts_with("GET ") && requests[4].starts_with("GET "), + "the recovery pass must be reads only: {requests:?}" + ); + } + + #[tokio::test] + async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() { + let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + pending.key_state = KeyState::PendingDeletion; + pending.deletion_date = Some(Zoned::now()); + let cancelled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // The transit key still exists. + ScriptedResponse::ok(transit_key_read_data("wired-key")), + // Versioned read finds a due pending-deletion record, but the + // tombstone write loses the check-and-set race to a cancel. + ScriptedResponse::ok(kv2_metadata_read_data(1)), + ScriptedResponse::ok(metadata_read_data(&pending)), + ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE), + // The re-read sees the cancelled (Enabled) record: back off. + ScriptedResponse::ok(kv2_metadata_read_data(2)), + ScriptedResponse::ok(metadata_read_data(&cancelled)), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + let now = Zoned::now() + Duration::from_secs(3600); + let outcome = backend + .remove_expired_key("wired-key", &now) + .await + .expect("losing the tombstone race to a cancel must back off cleanly"); + assert_eq!(outcome, ExpiredKeyRemoval::StateChanged); + + let requests = vault.requests(); + assert_eq!(requests.len(), 6, "no delete may follow a lost tombstone race: {requests:?}"); + assert!( + !requests + .iter() + .any(|line| line.contains("/transit/keys/wired-key/config") || line.starts_with("DELETE ")), + "the sweep must not touch the transit key after backing off: {requests:?}" + ); } } From b6d4689c750334fe569ccfbdd4fafa67bd23b23f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 10:05:40 +0800 Subject: [PATCH 09/12] feat(policy): parse and match KMS key resources with a kms:Decrypt action (#5515) Bring the dormant Resource::Kms variant to life: arn:aws:kms:::key/ patterns (empty-account form, wildcards allowed in the id, alias/ reserved as parse-only) now parse, validate, serialize back, and are matched by KMS statements against the requested key id carried in Args::object. Statements without KMS resources keep the legacy match-every-key behaviour, as do call sites that pass no key resource, so nothing changes until the admin/SSE authorization paths start passing key ids. Statement validation rejects KMS resources on non-KMS statements, and bucket policy validation rejects KMS actions and resources outright while stored policies keep deserializing; evaluation skips pure-KMS bucket policy statements with a warning. A kms:Decrypt action is added for the upcoming SSE-KMS read path. Refs rustfs/backlog#1582 (part of rustfs/backlog#1562) --- crates/policy/src/policy.rs | 6 + crates/policy/src/policy/action.rs | 3 + crates/policy/src/policy/policy.rs | 385 ++++++++++++++++++++ crates/policy/src/policy/resource.rs | 103 +++++- crates/policy/src/policy/statement.rs | 116 +++++- crates/policy/tests/policy_eval_proptest.rs | 153 ++++++++ 6 files changed, 751 insertions(+), 15 deletions(-) diff --git a/crates/policy/src/policy.rs b/crates/policy/src/policy.rs index 0c01b9729..651dae0ca 100644 --- a/crates/policy/src/policy.rs +++ b/crates/policy/src/policy.rs @@ -72,4 +72,10 @@ pub enum Error { #[error("invalid resource, type: '{0}', pattern: '{1}'")] InvalidResource(String, String), + + #[error("KMS resources require a statement whose actions are all KMS actions")] + KmsResourceWithNonKmsAction, + + #[error("bucket policies do not support KMS actions or resources")] + KmsUnsupportedInBucketPolicy, } diff --git a/crates/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs index 9d0790dae..fbefb87f1 100644 --- a/crates/policy/src/policy/action.rs +++ b/crates/policy/src/policy/action.rs @@ -728,6 +728,8 @@ pub enum KmsAction { ListKeysAction, #[strum(serialize = "kms:DescribeKey")] DescribeKeyAction, + #[strum(serialize = "kms:Decrypt")] + DecryptAction, } #[cfg(test)] @@ -764,6 +766,7 @@ mod tests { ("kms:RotateKey", KmsAction::RotateKeyAction), ("kms:ListKeys", KmsAction::ListKeysAction), ("kms:DescribeKey", KmsAction::DescribeKeyAction), + ("kms:Decrypt", KmsAction::DecryptAction), ] { let action = Action::try_from(raw).expect("Should parse KMS action"); assert_eq!(action, Action::KmsAction(expected)); diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index b2e766f5a..7188d3228 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -1760,6 +1760,391 @@ mod test { ); } + fn kms_args<'a>( + action: Action, + key_id: &'a str, + conditions: &'a HashMap>, + claims: &'a HashMap, + ) -> Args<'a> { + Args { + account: "testuser", + groups: &None, + action, + bucket: "", + conditions, + is_owner: false, + object: key_id, + claims, + deny_only: false, + } + } + + #[tokio::test] + async fn test_kms_statement_with_key_resource_scopes_by_key() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DisableKey"], + "Resource": ["arn:aws:kms:::key/key-a"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DisableKeyAction); + + assert!( + policy.is_allowed(&kms_args(action, "key-a", &conditions, &claims)).await, + "the granted key must be allowed" + ); + assert!( + !policy.is_allowed(&kms_args(action, "key-b", &conditions, &claims)).await, + "a key outside the granted resource must be denied" + ); + assert!( + !policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::EnableKeyAction), "key-a", &conditions, &claims)) + .await, + "an action outside the grant must stay denied even for the granted key" + ); + assert!( + policy.is_allowed(&kms_args(action, "", &conditions, &claims)).await, + "call sites that do not pass a key resource keep the legacy match-every-key behaviour" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_with_wildcard_key_resource() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:GenerateDataKey"], + "Resource": ["arn:aws:kms:::key/app-*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::GenerateDataKeyAction); + + assert!( + policy + .is_allowed(&kms_args(action, "app-primary", &conditions, &claims)) + .await + ); + assert!( + !policy + .is_allowed(&kms_args(action, "backup-primary", &conditions, &claims)) + .await + ); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_without_resource_matches_every_key() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + + for key_id in ["", "key-a", "any-other-key"] { + assert!( + policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::DisableKeyAction), key_id, &conditions, &claims)) + .await, + "resource-less KMS statement must keep matching every key (key_id: {key_id:?})" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_kms_deny_with_wildcard_resource_overrides_allow() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:*"], + "Resource": ["arn:aws:kms:::key/key-a"] + }, + { + "Effect": "Deny", + "Action": ["kms:DisableKey"], + "Resource": ["arn:aws:kms:::key/*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + + assert!( + !policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::DisableKeyAction), "key-a", &conditions, &claims)) + .await, + "a wildcard Deny must override the narrower Allow" + ); + assert!( + policy + .is_allowed(&kms_args(Action::KmsAction(KmsAction::RotateKeyAction), "key-a", &conditions, &claims)) + .await, + "actions outside the Deny keep the Allow" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_with_not_resource_excludes_keys() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DescribeKey"], + "NotResource": ["arn:aws:kms:::key/prod-*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DescribeKeyAction); + + assert!(policy.is_allowed(&kms_args(action, "dev-key", &conditions, &claims)).await); + assert!(!policy.is_allowed(&kms_args(action, "prod-key", &conditions, &claims)).await); + + Ok(()) + } + + #[tokio::test] + async fn test_kms_statement_with_s3_resource_is_treated_as_unscoped() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + // Statements combining KMS actions with S3 resources predate KMS resource + // support and were always evaluated as if unscoped. Pin that they still + // match every key (a warning is logged during evaluation). + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DisableKey"], + "Resource": ["arn:aws:s3:::somebucket/*"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DisableKeyAction); + + for key_id in ["", "key-a"] { + assert!( + policy.is_allowed(&kms_args(action, key_id, &conditions, &claims)).await, + "malformed KMS statement with S3 resources must keep matching every key (key_id: {key_id:?})" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_kms_alias_resource_parses_but_matches_no_key() -> Result<()> { + use crate::policy::action::{Action, KmsAction}; + + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:DescribeKey"], + "Resource": ["arn:aws:kms:::alias/app-alias"] + } + ] +}"#, + )?; + let conditions = HashMap::new(); + let claims = HashMap::new(); + let action = Action::KmsAction(KmsAction::DescribeKeyAction); + + assert!( + !policy.is_allowed(&kms_args(action, "app-alias", &conditions, &claims)).await, + "alias patterns are parse-only until alias resolution lands and must not match key requests" + ); + assert!( + policy.is_allowed(&kms_args(action, "", &conditions, &claims)).await, + "call sites without a key resource keep the legacy match-every-key behaviour" + ); + + Ok(()) + } + + #[test] + fn test_kms_resource_policy_round_trips() { + let policy = Policy::parse_config( + br#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:GenerateDataKey", "kms:Decrypt"], + "Resource": ["arn:aws:kms:::key/app-*", "arn:aws:kms:::alias/app-alias"] + } + ] +}"#, + ) + .expect("KMS resource policy should parse"); + + let json = serde_json::to_string(&policy).expect("policy should serialize"); + let round_trip = Policy::parse_config(json.as_bytes()).expect("serialized KMS resource policy should re-parse"); + assert_eq!(round_trip.statements[0].resources, policy.statements[0].resources); + assert_eq!(round_trip.statements[0].actions, policy.statements[0].actions); + + let value: serde_json::Value = serde_json::from_str(&json).expect("JSON valid"); + let resources: Vec<_> = value["Statement"][0]["Resource"] + .as_array() + .expect("Resource should serialize as an array") + .iter() + .map(|resource| resource.as_str().expect("Resource entries should be strings")) + .collect(); + assert_eq!(resources, vec!["arn:aws:kms:::key/app-*", "arn:aws:kms:::alias/app-alias"]); + } + + #[test] + fn test_kms_resource_with_non_kms_action_is_invalid() { + for action in ["s3:GetObject", "admin:ServerInfo", "sts:AssumeRole"] { + let data = format!( + r#"{{ + "Version": "2012-10-17", + "Statement": [ + {{ + "Effect": "Allow", + "Action": ["{action}"], + "Resource": ["arn:aws:kms:::key/key-a"] + }} + ] +}}"# + ); + + let result = Policy::parse_config(data.as_bytes()); + assert!( + matches!(result.as_ref().unwrap_err(), Error::PolicyError(IamError::KmsResourceWithNonKmsAction)), + "{action} with a KMS resource should fail with KmsResourceWithNonKmsAction, got: {result:?}" + ); + } + } + + #[test] + fn test_bucket_policy_with_kms_statement_is_invalid() { + for statement in [ + r#"{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["kms:GenerateDataKey"],"Resource":["arn:aws:s3:::bucket/*"]}"#, + r#"{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject"],"Resource":["arn:aws:kms:::key/key-a"]}"#, + r#"{"Effect":"Allow","Principal":{"AWS":"*"},"NotAction":["kms:*"],"Resource":["arn:aws:s3:::bucket/*"]}"#, + ] { + let data = format!(r#"{{"Version":"2012-10-17","Statement":[{statement}]}}"#); + let policy: BucketPolicy = + serde_json::from_str(&data).expect("bucket policy with KMS content should still deserialize"); + let result = policy.is_valid(); + assert!( + matches!(result.as_ref().unwrap_err(), Error::PolicyError(IamError::KmsUnsupportedInBucketPolicy)), + "bucket policy statement {statement} should fail with KmsUnsupportedInBucketPolicy, got: {result:?}" + ); + } + } + + #[tokio::test] + async fn test_stored_bucket_policy_with_kms_statement_loads_and_is_ignored() -> Result<()> { + // Policies stored before KMS statements were rejected at validation must keep + // deserializing, and their KMS statements must not affect bucket traffic. + let bucket_policy: BucketPolicy = serde_json::from_str( + r#"{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["kms:GenerateDataKey"], + "Resource": ["arn:aws:s3:::bucket/*"] + }, + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::bucket/*"] + } + ] +}"#, + )?; + + let conditions = HashMap::new(); + let args = BucketPolicyArgs { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::GetObjectAction), + bucket: "bucket", + conditions: &conditions, + is_owner: false, + object: "a.txt", + }; + assert!( + bucket_policy.is_allowed(&args).await, + "the S3 statement must keep working alongside an ignored KMS statement" + ); + + let put_args = BucketPolicyArgs { + account: "testuser", + groups: &None, + action: Action::S3Action(crate::policy::action::S3Action::PutObjectAction), + bucket: "bucket", + conditions: &conditions, + is_owner: false, + object: "a.txt", + }; + assert!( + !bucket_policy.is_allowed(&put_args).await, + "the ignored KMS statement must not grant anything" + ); + + Ok(()) + } + #[test] fn test_mixed_action_families_are_invalid_even_with_resource() { let data = r#" diff --git a/crates/policy/src/policy/resource.rs b/crates/policy/src/policy/resource.rs index 2b6c03e72..262b13f4e 100644 --- a/crates/policy/src/policy/resource.rs +++ b/crates/policy/src/policy/resource.rs @@ -40,7 +40,7 @@ impl Serialize for ResourceSet { for resource in &self.0 { let resource_str = match resource { Resource::S3(value) => format!("{}{}", Resource::S3_PREFIX, value), - Resource::Kms(value) => value.clone(), + Resource::Kms(value) => format!("{}{}", Resource::KMS_PREFIX, value), }; seq.serialize_element(&resource_str)?; } @@ -171,6 +171,20 @@ pub enum Resource { impl Resource { pub const S3_PREFIX: &'static str = "arn:aws:s3:::"; + /// KMS ARNs use the same empty-account form as [`Self::S3_PREFIX`]; the suffix is + /// `key/` (wildcards allowed in the id), `alias/`, or a bare `*`. + pub const KMS_PREFIX: &'static str = "arn:aws:kms:::"; + /// Resource-type segment for key ids; request-side KMS resource strings are + /// `key/` so they line up with these patterns. + pub const KMS_KEY_SEGMENT: &'static str = "key/"; + /// Resource-type segment reserved for key aliases. Alias patterns parse and + /// validate, but requests are always evaluated against `key/` strings, + /// so an alias pattern matches nothing until alias resolution lands. + pub const KMS_ALIAS_SEGMENT: &'static str = "alias/"; + + pub fn is_kms(&self) -> bool { + matches!(self, Resource::Kms(_)) + } pub async fn is_match(&self, resource: &str, conditions: &HashMap>) -> bool { self.is_match_with_resolver(resource, conditions, None).await @@ -228,10 +242,13 @@ impl Resource { impl TryFrom<&str> for Resource { type Error = Error; fn try_from(value: &str) -> std::result::Result { - let Some(value) = value.strip_prefix(Self::S3_PREFIX) else { + let resource = if let Some(suffix) = value.strip_prefix(Self::S3_PREFIX) { + Resource::S3(suffix.into()) + } else if let Some(suffix) = value.strip_prefix(Self::KMS_PREFIX) { + Resource::Kms(suffix.into()) + } else { return Err(IamError::InvalidResource("unknown".into(), value.into()).into()); }; - let resource = Resource::S3(value.into()); resource.is_valid()?; Ok(resource) @@ -248,13 +265,17 @@ impl Validator for Resource { } } Self::Kms(pattern) => { - if pattern.is_empty() + // A bare `*` matches every key resource; anything else must carry a + // resource-type segment. Key ids never contain separators (the KMS + // backends reject them), while alias names may nest ("alias/aws/s3"). + let well_formed = pattern == "*" || pattern - .char_indices() - .find(|&(_, c)| c == '/' || c == '\\' || c == '.') - .map(|(i, _)| i) - .is_some() - { + .strip_prefix(Self::KMS_KEY_SEGMENT) + .is_some_and(|id| !id.is_empty() && !id.contains('/') && !id.contains('\\')) + || pattern + .strip_prefix(Self::KMS_ALIAS_SEGMENT) + .is_some_and(|name| !name.is_empty() && !name.contains('\\')); + if !well_formed { return Err(IamError::InvalidResource("kms".into(), pattern.into()).into()); } } @@ -270,7 +291,7 @@ impl Serialize for Resource { { match self { Resource::S3(s) => serializer.serialize_str(&format!("{}{}", Self::S3_PREFIX, s)), - Resource::Kms(s) => serializer.serialize_str(s), + Resource::Kms(s) => serializer.serialize_str(&format!("{}{}", Self::KMS_PREFIX, s)), } } } @@ -308,9 +329,71 @@ mod tests { #[test_case("arn:aws:s3:::mybucket","mybucket/myobject" => false; "15")] #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/evil.txt" => false; "16")] #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/safe/../../victim-bucket/evil.txt" => false; "17")] + #[test_case("arn:aws:kms:::key/mykey","key/mykey" => true; "kms exact key")] + #[test_case("arn:aws:kms:::key/mykey","key/otherkey" => false; "kms wrong key")] + #[test_case("arn:aws:kms:::key/*","key/mykey" => true; "kms key wildcard")] + #[test_case("arn:aws:kms:::key/app-*","key/app-primary" => true; "kms key prefix wildcard")] + #[test_case("arn:aws:kms:::key/app-*","key/backup-primary" => false; "kms key prefix mismatch")] + #[test_case("arn:aws:kms:::key/mykey?","key/mykey1" => true; "kms key question mark")] + #[test_case("arn:aws:kms:::*","key/mykey" => true; "kms bare star")] + #[test_case("arn:aws:kms:::key/mykey","key/mykey/../otherkey" => false; "kms traversal cleaned")] + #[test_case("arn:aws:kms:::alias/myalias","key/myalias" => false; "kms alias never matches key")] + #[test_case("arn:aws:kms:::alias/*","key/mykey" => false; "kms alias wildcard never matches key")] + #[test_case("arn:aws:kms:::key/mykey","mykey" => false; "kms bare id lacks key segment")] fn test_resource_is_match(resource: &str, object: &str) -> bool { let resource: Resource = resource.try_into().unwrap(); pollster::block_on(resource.is_match(object, &HashMap::new())) } + + #[test_case("arn:aws:kms:::key/mykey" => true; "key id parses")] + #[test_case("arn:aws:kms:::key/*" => true; "key wildcard parses")] + #[test_case("arn:aws:kms:::key/app-key.v2" => true; "key id with dot parses")] + #[test_case("arn:aws:kms:::alias/myalias" => true; "alias parses")] + #[test_case("arn:aws:kms:::alias/aws/s3" => true; "nested alias parses")] + #[test_case("arn:aws:kms:::*" => true; "bare star parses")] + #[test_case("arn:aws:kms:::" => false; "empty suffix rejected")] + #[test_case("arn:aws:kms:::key/" => false; "empty key id rejected")] + #[test_case("arn:aws:kms:::alias/" => false; "empty alias name rejected")] + #[test_case("arn:aws:kms:::mykey" => false; "missing resource type segment rejected")] + #[test_case("arn:aws:kms:::key/a/b" => false; "separator in key id rejected")] + #[test_case("arn:aws:kms:::key/a\\b" => false; "backslash in key id rejected")] + #[test_case("arn:aws:kms:us-east-1:123456789012:key/mykey" => false; "region and account form rejected")] + fn test_kms_resource_parse(resource: &str) -> bool { + Resource::try_from(resource).is_ok() + } + + #[test] + fn test_kms_resource_serialization_round_trip() { + for raw in [ + "arn:aws:kms:::key/mykey", + "arn:aws:kms:::key/*", + "arn:aws:kms:::alias/myalias", + "arn:aws:kms:::*", + ] { + let resource = Resource::try_from(raw).expect("KMS resource should parse"); + assert!(resource.is_kms()); + + let json = serde_json::to_string(&resource).expect("KMS resource should serialize"); + assert_eq!(json, format!("\"{raw}\""), "serialization must write back the full ARN"); + + let round_trip: Resource = serde_json::from_str(&json).expect("serialized KMS resource should deserialize"); + assert_eq!(round_trip, resource); + } + } + + #[test] + fn test_kms_resource_set_serialization_round_trip() { + use crate::policy::resource::ResourceSet; + + let set: ResourceSet = + serde_json::from_str(r#"["arn:aws:kms:::key/app-*","arn:aws:kms:::alias/myalias"]"#).expect("set should parse"); + assert_eq!(set.len(), 2); + + let json = serde_json::to_string(&set).expect("set should serialize"); + assert_eq!(json, r#"["arn:aws:kms:::key/app-*","arn:aws:kms:::alias/myalias"]"#); + + let round_trip: ResourceSet = serde_json::from_str(&json).expect("serialized set should deserialize"); + assert_eq!(round_trip, set); + } } diff --git a/crates/policy/src/policy/statement.rs b/crates/policy/src/policy/statement.rs index 861c188e1..a45466713 100644 --- a/crates/policy/src/policy/statement.rs +++ b/crates/policy/src/policy/statement.rs @@ -16,6 +16,7 @@ use super::{ ActionSet, Args, BucketPolicyArgs, Effect, Error as IamError, Functions, ID, Principal, ResourceSet, Validator, action::{Action, S3Action}, function::key_name::{KeyName, S3KeyName}, + resource::Resource, variables::{VariableContext, VariableResolver}, }; use crate::error::{Error, Result}; @@ -173,13 +174,79 @@ impl Statement { Some(ActionFamily::Mixed) } + /// Resource scope check for KMS statements, which match `arn:aws:kms:::key/` + /// patterns instead of the bucket/object path grammar used by S3 statements. + /// + /// Call-site contract (wired up by the admin/SSE authorization paths): the requested + /// key identifier (pre-alias-resolution) travels in `args.object` with `args.bucket` + /// left empty. An empty `args.object` means the caller did not scope the request to a + /// key, which preserves the legacy match-every-key behaviour. + async fn kms_key_scope_matches(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool { + let kms_resources: Vec<&Resource> = self.resources.iter().filter(|resource| resource.is_kms()).collect(); + let kms_not_resources: Vec<&Resource> = self.not_resources.iter().filter(|resource| resource.is_kms()).collect(); + + if kms_resources.len() != self.resources.len() || kms_not_resources.len() != self.not_resources.len() { + // Statements combining KMS actions with S3 resources predate KMS resource + // support and were always evaluated as if unscoped; keep that behaviour + // but surface it, since the S3 patterns never constrain key access. + tracing::warn!( + sid = %self.sid.0, + "KMS statement carries non-KMS resources; they are ignored and the statement matches every key" + ); + } + + if kms_resources.is_empty() && kms_not_resources.is_empty() { + // No KMS resources: the statement scopes by action only (legacy form). + return true; + } + + if args.object.is_empty() { + // Call sites that do not pass a key resource keep the pre-resource-scoping + // behaviour where any key matches. + return true; + } + + let requested = format!("{}{}", Resource::KMS_KEY_SEGMENT, args.object); + + if !kms_resources.is_empty() { + let mut matched = false; + for resource in kms_resources { + if resource + .is_match_with_resolver(&requested, args.conditions, Some(resolver)) + .await + { + matched = true; + break; + } + } + if !matched { + return false; + } + } + + for resource in kms_not_resources { + if resource + .is_match_with_resolver(&requested, args.conditions, Some(resolver)) + .await + { + return false; + } + } + + true + } + /// Returns true when this statement would reach `conditions.evaluate_with_resolver` in - /// [`Statement::is_allowed`] (including the KMS shortcut path). Does not evaluate conditions. + /// [`Statement::is_allowed`] (including the KMS resource path). Does not evaluate conditions. pub(crate) async fn request_reaches_condition_eval(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool { if (!self.actions.is_match(&args.action) && !self.actions.is_empty()) || self.not_actions.is_match(&args.action) { return false; } + if self.is_kms() { + return self.kms_key_scope_matches(args, resolver).await; + } + let resource = build_resource( &args.action, args.bucket, @@ -187,10 +254,6 @@ impl Statement { self.conditions.references_key_name(&KeyName::S3(S3KeyName::S3Prefix)), ); - if self.is_kms() && (resource == "/" || self.resources.is_empty()) { - return true; - } - if self.resources.is_empty() && self.not_resources.is_empty() && !self.is_admin() && !self.is_sts() { return false; } @@ -273,6 +336,19 @@ impl Validator for Statement { return Err(IamError::BothResourceAndNotResource.into()); } + // KMS resources only make sense on pure-KMS statements. The reverse + // combination (KMS actions with S3 resources) predates KMS resources, + // may already be stored, and stays loadable; evaluation treats it as + // unscoped and warns. + let has_kms_resource = self + .resources + .iter() + .chain(self.not_resources.iter()) + .any(|resource| resource.is_kms()); + if has_kms_resource && !matches!(action_family, Some(ActionFamily::Kms)) { + return Err(IamError::KmsResourceWithNonKmsAction.into()); + } + self.actions.is_valid()?; self.not_actions.is_valid()?; self.resources.is_valid()?; @@ -323,6 +399,18 @@ pub struct BPStatement { impl BPStatement { /// Returns true when this statement would reach `conditions.evaluate` in [`BPStatement::is_allowed`]. pub(crate) async fn request_reaches_condition_eval(&self, args: &BucketPolicyArgs<'_>) -> bool { + if !self.actions.is_empty() && self.actions.iter().all(|action| matches!(action, Action::KmsAction(_))) { + // Bucket policies cannot grant or deny KMS access; such statements are + // rejected at validation but may exist in policies stored before that + // check. Skip them so they never influence bucket traffic. Statements + // mixing KMS with S3 actions keep evaluating their S3 actions as before. + tracing::warn!( + sid = %self.sid.0, + "ignoring bucket policy statement with KMS actions during evaluation" + ); + return false; + } + if !self.principal.is_match(args.account) { return false; } @@ -379,6 +467,24 @@ impl Validator for BPStatement { return Err(IamError::BothActionAndNotAction.into()); } + // Bucket policies govern S3 access; KMS grants belong in identity policies. + // Rejected here (PutBucketPolicy) only: deserialization stays permissive so + // stored policies from before this check keep loading, and evaluation skips + // pure-KMS statements with a warning. + let has_kms_action = self + .actions + .iter() + .chain(self.not_actions.iter()) + .any(|action| matches!(action, Action::KmsAction(_))); + let has_kms_resource = self + .resources + .iter() + .chain(self.not_resources.iter()) + .any(|resource| resource.is_kms()); + if has_kms_action || has_kms_resource { + return Err(IamError::KmsUnsupportedInBucketPolicy.into()); + } + if self.resources.is_empty() && self.not_resources.is_empty() { return Err(IamError::NonResource.into()); } diff --git a/crates/policy/tests/policy_eval_proptest.rs b/crates/policy/tests/policy_eval_proptest.rs index 6ca75e0fc..3de57a0a4 100644 --- a/crates/policy/tests/policy_eval_proptest.rs +++ b/crates/policy/tests/policy_eval_proptest.rs @@ -26,6 +26,12 @@ //! policy is also allowed by the widened policy; //! (c) an empty policy denies every non-owner request (default deny). //! +//! Properties (d)-(g) extend the same invariants to KMS key resources +//! (`arn:aws:kms:::key/`, backlog#1582): Deny-first over key scopes, +//! wildcard/resource-less supersets implying concrete key grants, exact key +//! scoping without cross-key leaks, and the legacy resource-less +//! match-every-key compatibility pin. +//! //! Pure evaluation: no IO, no global state, parallel-safe. Statements are built //! from JSON exactly like production policies arriving via PutPolicy. Generated //! bucket/key/action pools avoid wildcard metacharacters so resource patterns @@ -47,10 +53,23 @@ const OBJECT_ACTIONS: &[&str] = &[ "s3:PutObjectTagging", ]; +/// Key-scoped KMS actions safe to pair with an `arn:aws:kms:::key/` resource. +const KMS_KEY_ACTIONS: &[&str] = &[ + "kms:GenerateDataKey", + "kms:Decrypt", + "kms:DisableKey", + "kms:RotateKey", + "kms:DescribeKey", +]; + fn statement_json(effect: &str, action: &str, resource: &str) -> String { format!(r#"{{"Effect":"{effect}","Action":["{action}"],"Resource":["{resource}"]}}"#) } +fn resourceless_statement_json(effect: &str, action: &str) -> String { + format!(r#"{{"Effect":"{effect}","Action":["{action}"]}}"#) +} + fn policy_from_statements(statements: &[String]) -> Policy { let json = format!(r#"{{"Version":"2012-10-17","Statement":[{}]}}"#, statements.join(",")); serde_json::from_str(&json).expect("generated policy JSON should parse") @@ -88,6 +107,22 @@ fn action_strategy() -> impl Strategy { proptest::sample::select(OBJECT_ACTIONS) } +/// Strategy: a KMS key id without wildcard metacharacters or separators. +fn key_id_strategy() -> impl Strategy { + "[a-z][a-z0-9-]{2,11}" +} + +/// Strategy: one action name from the key-scoped KMS action pool. +fn kms_action_strategy() -> impl Strategy { + proptest::sample::select(KMS_KEY_ACTIONS) +} + +/// KMS evaluation contract: the requested key id travels in `args.object` with an +/// empty bucket (see `Statement::kms_key_scope_matches`). +fn is_allowed_for_key(policy: &Policy, action: &str, key_id: &str) -> bool { + is_allowed(policy, action, "", key_id) +} + proptest! { /// (a) Deny anywhere wins: a Deny statement matching the request denies it, /// no matter how many broad Allow statements surround it or at which index @@ -205,4 +240,122 @@ proptest! { "Policy::default() must deny {action} on {bucket}/{key}" ); } + + /// (d) KMS Deny anywhere wins: a Deny scoped to the exact key (or `key/*`) + /// denies the request no matter how many broad KMS Allow statements + /// (resource-less or `key/*`-scoped) surround it. + #[test] + fn kms_explicit_deny_anywhere_denies( + key_id in key_id_strategy(), + action in kms_action_strategy(), + allow_count in 0usize..4, + deny_pos_seed in 0usize..16, + broad in proptest::bool::ANY, + wildcard_deny in proptest::bool::ANY, + ) { + let mut statements: Vec = (0..allow_count) + .map(|_| { + if broad { + resourceless_statement_json("Allow", "kms:*") + } else { + statement_json("Allow", "kms:*", "arn:aws:kms:::key/*") + } + }) + .collect(); + + let deny_resource = if wildcard_deny { + "arn:aws:kms:::key/*".to_string() + } else { + format!("arn:aws:kms:::key/{key_id}") + }; + let deny = statement_json("Deny", action, &deny_resource); + let deny_pos = deny_pos_seed % (statements.len() + 1); + statements.insert(deny_pos, deny); + + let policy = policy_from_statements(&statements); + + if allow_count > 0 { + let mut allows_only = statements.clone(); + allows_only.remove(deny_pos); + let allow_policy = policy_from_statements(&allows_only); + prop_assert!( + is_allowed_for_key(&allow_policy, action, &key_id), + "sanity: the KMS Allow statements alone should permit {action} on key {key_id}" + ); + } + + prop_assert!( + !is_allowed_for_key(&policy, action, &key_id), + "explicit KMS Deny at index {deny_pos} of {} statements must deny {action} on key {key_id}", + statements.len() + ); + } + + /// (e) KMS wildcard superset implies the concrete key grant: whatever an + /// exact `key/` Allow permits is also permitted by `key/*`, by a bare + /// `arn:aws:kms:::*`, and by the legacy resource-less statement form. + #[test] + fn kms_wildcard_superset_implies_concrete_match( + key_id in key_id_strategy(), + action in kms_action_strategy(), + ) { + let narrow = policy_from_statements(&[statement_json( + "Allow", + action, + &format!("arn:aws:kms:::key/{key_id}"), + )]); + let widened = policy_from_statements(&[statement_json("Allow", "kms:*", "arn:aws:kms:::key/*")]); + let star = policy_from_statements(&[statement_json("Allow", "kms:*", "arn:aws:kms:::*")]); + let resourceless = policy_from_statements(&[resourceless_statement_json("Allow", "kms:*")]); + + prop_assert!(is_allowed_for_key(&narrow, action, &key_id), "narrow KMS policy must allow its own grant"); + prop_assert!(is_allowed_for_key(&widened, action, &key_id), "kms:* on key/* must imply the concrete grant"); + prop_assert!(is_allowed_for_key(&star, action, &key_id), "kms:* on arn:aws:kms:::* must imply the concrete grant"); + prop_assert!( + is_allowed_for_key(&resourceless, action, &key_id), + "the legacy resource-less KMS statement must imply the concrete grant" + ); + } + + /// (f) Key scoping is exact: an Allow on `key/` never leaks to a + /// different key id, while the compatibility contract keeps unscoped + /// requests (no key id passed) matching. + #[test] + fn kms_key_scope_does_not_leak_across_keys( + key_a in key_id_strategy(), + key_b in key_id_strategy(), + action in kms_action_strategy(), + ) { + prop_assume!(key_a != key_b); + + let policy = policy_from_statements(&[statement_json( + "Allow", + action, + &format!("arn:aws:kms:::key/{key_a}"), + )]); + + prop_assert!(is_allowed_for_key(&policy, action, &key_a), "the granted key must be allowed"); + prop_assert!( + !is_allowed_for_key(&policy, action, &key_b), + "an Allow scoped to key {key_a} must not leak to key {key_b}" + ); + prop_assert!( + is_allowed_for_key(&policy, action, ""), + "call sites that pass no key resource keep the legacy match-every-key behaviour" + ); + } + + /// (g) Legacy compatibility pin: resource-less KMS statements match every + /// generated key id, exactly as before KMS resources existed. + #[test] + fn kms_resourceless_statement_matches_every_key( + key_id in key_id_strategy(), + action in kms_action_strategy(), + ) { + let policy = policy_from_statements(&[resourceless_statement_json("Allow", action)]); + prop_assert!( + is_allowed_for_key(&policy, action, &key_id), + "resource-less KMS statement must keep matching {action} on key {key_id}" + ); + } } From 739efaaea1603e7ae3659c13ab88c3cf4b05214f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 10:05:47 +0800 Subject: [PATCH 10/12] feat(kms): restore local backend key material from sealed backup bundles (#5522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kms): add link_durably primitive and restore-marker startup guard The local backend gains the two pieces the bundle restore path builds on: a no-clobber hard-link publish primitive whose AlreadyExists case is idempotent only for byte-identical content, and a fail-closed startup guard that refuses to open a key directory holding a restore cutover marker. The durable commit protocol and the key-id containment check become pub(crate) so the restore module reuses them instead of copies. * feat(kms): record a master-key verifier and pre-seal decrypt probe in export Fill the manifest's master_key_verifier slot with an opaque one-way value (scheme-prefixed, bound to the backup id and the KDF salt) so a restore can detect a wrong operator-supplied master key before touching any target state, and probe-decrypt every artifact as stored under the backup KEK before the manifest may seal — digest equality alone only proves the ciphertext landed intact. The payload decryption tail is factored out and shared with the restore side so producer and consumer cannot drift on the framing. Also drops the stale KmsClient test import orphaned by the backend refactor (#5501); the test suite did not compile without this. * feat(kms): restore local backend key material from sealed backup bundles The consumer side of the Local bundle export, as a four-phase protocol: - Dry-run: full in-memory bundle decode (digest and AEAD verification of every artifact), KDF-drift detection against the compiled-in derivation, deployment and injected-generation checks (strictly lower is rejected, equal stays allowed for repeated drills), master-key verifier check, and target conflict enumeration - with zero writes. - Staging: artifacts are committed durably into the .restore-staging/ subdirectory (invisible to the backend's key scan and orphan-temp matcher) and every record is decryption-probed with the derived master key both in memory before staging and again from the staged bytes. - Commit marker + cutover: the durably published .restore-commit.json marker is the single commit point; cutover publishes staged files via link_durably (salt first, keys after), then durably removes the marker and drops staging. - Crash re-entry: before the marker the target top level is untouched and a re-run starts over; with the marker published, backend startup fails closed and a re-run with the same bundle rolls forward while abort_local_restore rolls back. Every interruption converges to the complete old or complete new state. Restore never goes through LocalKmsClient::new (which would mint a fresh salt); the only write mode is the explicit restore-into-empty-target policy, where an orphan salt or a foreign marker already counts as non-empty. The bundle source stays strictly read-only. Refs rustfs/backlog#1572 --- crates/kms/src/backends/local.rs | 139 +- crates/kms/src/backup/local_export.rs | 126 +- crates/kms/src/backup/local_restore.rs | 1970 ++++++++++++++++++++++++ crates/kms/src/backup/mod.rs | 12 +- 4 files changed, 2217 insertions(+), 30 deletions(-) create mode 100644 crates/kms/src/backup/local_restore.rs diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index 1a12b8593..490e5bdac 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -46,7 +46,10 @@ use zeroize::Zeroizing; /// `key_dir` is accepted, so identifiers already in use by existing deployments keep /// resolving. Only separators, traversal and the degenerate cases are refused, which is /// what stops `key_dir.join(...)` from escaping. -fn validate_key_id(key_id: &str) -> Result<()> { +/// +/// pub(crate) because the backup restore path applies the same containment +/// rule to key identifiers recovered from bundle artifacts. +pub(crate) fn validate_key_id(key_id: &str) -> Result<()> { if key_id.is_empty() { return Err(KmsError::invalid_key("key identifier must not be empty")); } @@ -68,7 +71,15 @@ fn validate_key_id(key_id: &str) -> Result<()> { } } -const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt"; +// The salt and restore-marker file names are pub(crate) so the backup/restore +// modules (`crate::backup`) address the exact on-disk names instead of copies +// that could drift. +pub(crate) const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt"; +/// Commit marker of an in-progress Local restore cutover (see +/// `crate::backup::local_restore`). Its presence means the key directory is +/// mid-cutover: startup must fail closed until the restore is rolled forward +/// or explicitly aborted. +pub(crate) const LOCAL_RESTORE_COMMIT_MARKER_FILE: &str = ".restore-commit.json"; // The KDF parameters are pub(crate) so the backup manifest contract // (`crate::backup`) records the exact compiled-in derivation instead of a // copy that could drift. @@ -87,7 +98,11 @@ pub(crate) const LOCAL_KMS_ARGON2_P_COST: u32 = 1; /// `foo.tmp-` is stored as `foo.tmp-.key` — so the `.key` guard /// plus the exact hyphenated-UUID check makes it impossible to match an /// authoritative file. -fn is_orphan_commit_temp_name(file_name: &str) -> bool { +/// +/// pub(crate) because the backup restore path applies the same classification +/// when it re-enters an interrupted run: a leftover commit temp is never +/// authoritative state, so it does not make a target non-empty. +pub(crate) fn is_orphan_commit_temp_name(file_name: &str) -> bool { if file_name.ends_with(".key") { return false; } @@ -110,12 +125,15 @@ fn is_orphan_commit_temp_name(file_name: &str) -> bool { /// /// This intentionally mirrors ecstore's fsync helpers without depending on the /// ecstore crate: the KMS backend stays decoupled from storage internals. -mod durable_file { +/// +/// pub(crate) because the backup restore path (`crate::backup::local_restore`) +/// commits staged files and its cutover marker through the same protocol. +pub(crate) mod durable_file { use std::io::{self, Write}; use std::path::{Path, PathBuf}; /// How the fully written temp file becomes visible under its final name. - pub(super) enum Publish { + pub(crate) enum Publish { /// Atomically replace whatever is at the destination via `rename`. Replace, /// Publish via `hard_link`, failing with [`CommitError::AlreadyExists`] @@ -124,7 +142,7 @@ mod durable_file { } #[derive(Debug)] - pub(super) enum CommitError { + pub(crate) enum CommitError { AlreadyExists, Io(io::Error), /// Test-only simulated crash: the protocol stops after the given step @@ -158,14 +176,14 @@ mod durable_file { /// to prove that every interrupted prefix recovers to either the complete /// old state or the complete new state. #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum CommitStep { + pub(crate) enum CommitStep { TempWritten, FileSynced, Published, DirSynced, } - pub(super) async fn commit( + pub(crate) async fn commit( temp_path: PathBuf, final_path: PathBuf, content: Vec, @@ -179,7 +197,7 @@ mod durable_file { /// Remove a published file durably: without the parent directory fsync a /// deleted key could resurface after power loss. - pub(super) async fn remove_durably(path: PathBuf) -> io::Result<()> { + pub(crate) async fn remove_durably(path: PathBuf) -> io::Result<()> { tokio::task::spawn_blocking(move || { std::fs::remove_file(&path)?; let parent = path @@ -191,6 +209,41 @@ mod durable_file { .map_err(io::Error::other)? } + /// Publish an already-durable file under a second name via `hard_link`, + /// then fsync the destination's parent directory. + /// + /// This is the restore cutover primitive: the source (a staged file that + /// went through [`commit`]) is already durable, so linking plus a parent + /// fsync is a complete publish. `AlreadyExists` is idempotent success only + /// when the destination content is byte-identical to the source — that is + /// exactly the re-entry case of a cutover interrupted after this link — + /// and a hard failure otherwise, so the primitive can never clobber or + /// silently accept foreign state. + pub(crate) async fn link_durably(source: PathBuf, dest: PathBuf) -> io::Result<()> { + tokio::task::spawn_blocking(move || { + match std::fs::hard_link(&source, &dest) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + let existing = std::fs::read(&dest)?; + let staged = std::fs::read(&source)?; + if existing != staged { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("destination {} already exists with different content", dest.display()), + )); + } + } + Err(error) => return Err(error), + } + let parent = dest + .parent() + .ok_or_else(|| io::Error::other("destination has no parent directory"))?; + fsync_dir(parent) + }) + .await + .map_err(io::Error::other)? + } + fn commit_blocking( temp_path: &Path, final_path: &Path, @@ -355,7 +408,7 @@ mod durable_file { /// Test-only failpoints simulating a crash after a given commit step. /// Armed per directory so parallel tests never affect each other. #[cfg(test)] - pub(super) mod failpoint { + pub(crate) mod failpoint { use super::CommitStep; use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -460,6 +513,11 @@ impl LocalKmsClient { debug!(path = ?config.key_dir, "KMS key directory created"); } + // The restore-marker guard must run before anything else touches the + // directory (in particular before salt load/creation): a directory + // mid-cutover holds an arbitrary mix of old and new state. + Self::ensure_no_restore_marker(&config).await?; + // Initialize master cipher if master key is provided let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key { let salt = Self::load_or_create_master_key_salt(&config).await?; @@ -491,6 +549,7 @@ impl LocalKmsClient { if !fs::try_exists(&config.key_dir).await? { return Err(KmsError::configuration_error("Local KMS key directory does not exist")); } + Self::ensure_no_restore_marker(&config).await?; let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key { let legacy_key = Self::derive_legacy_master_key(master_key)?; @@ -567,8 +626,36 @@ impl LocalKmsClient { Self::master_key_salt_path(&self.config) } + /// Operator-configured master key string, exposed for the backup export + /// module so it can record a one-way verifier in the bundle manifest. + /// Never log or persist this value. + pub(crate) fn configured_master_key(&self) -> Option<&str> { + self.config.master_key.as_deref() + } + + /// Fail closed while a restore cutover marker is present: the directory + /// then holds an arbitrary mix of pre-restore and restored state, and the + /// only valid next steps are re-running the restore with the same bundle + /// (roll forward) or explicitly aborting it. This mirrors the missing-salt + /// guard: startup must never paper over a half-applied restore. + async fn ensure_no_restore_marker(config: &LocalConfig) -> Result<()> { + let marker = config.key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + if fs::try_exists(&marker).await? { + return Err(KmsError::configuration_error(format!( + "Local KMS key directory has an unfinished restore (marker {} present); \ + re-run the restore with the same bundle to roll it forward or abort it explicitly", + marker.display() + ))); + } + Ok(()) + } + /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. - fn derive_master_key(master_key: &str, salt: &[u8]) -> Result> { + /// + /// pub(crate) because the backup restore path derives the same key from + /// the operator-supplied master key and the bundled salt for its verifier + /// check and staged decryption probe. + pub(crate) fn derive_master_key(master_key: &str, salt: &[u8]) -> Result> { let params = Params::new( LOCAL_KMS_ARGON2_M_COST_KIB, LOCAL_KMS_ARGON2_T_COST, @@ -585,7 +672,7 @@ impl LocalKmsClient { Ok(key) } - fn derive_legacy_master_key(master_key: &str) -> Result> { + pub(crate) fn derive_legacy_master_key(master_key: &str) -> Result> { let mut hasher = Sha256::new(); hasher.update(master_key.as_bytes()); hasher.update(b"rustfs-kms-local"); @@ -2453,6 +2540,34 @@ mod tests { assert!(!is_orphan_commit_temp_name("mykey.tmp-")); } + #[tokio::test] + async fn link_durably_publishes_no_clobber_and_is_content_idempotent() { + let temp_dir = TempDir::new().expect("temp dir"); + let source = temp_dir.path().join("staged"); + let dest = temp_dir.path().join("published"); + fs::write(&source, b"staged-content").await.expect("write source"); + + durable_file::link_durably(source.clone(), dest.clone()) + .await + .expect("first link must succeed"); + assert_eq!(fs::read(&dest).await.expect("read dest"), b"staged-content"); + + // Re-entry with identical content is idempotent success — exactly the + // resumed-cutover case. + durable_file::link_durably(source.clone(), dest.clone()) + .await + .expect("re-linking identical content must be idempotent"); + + // Existing content that differs is a hard failure, never a clobber. + let foreign = temp_dir.path().join("foreign"); + fs::write(&foreign, b"different-content").await.expect("write foreign"); + let error = durable_file::link_durably(foreign, dest.clone()) + .await + .expect_err("differing content must not be clobbered"); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert_eq!(fs::read(&dest).await.expect("dest unchanged"), b"staged-content"); + } + #[tokio::test] async fn durable_commit_fsyncs_every_write_path() { use durable_file::fsync_recorder; diff --git a/crates/kms/src/backup/local_export.rs b/crates/kms/src/backup/local_export.rs index 00bb905e8..c0f393057 100644 --- a/crates/kms/src/backup/local_export.rs +++ b/crates/kms/src/backup/local_export.rs @@ -14,9 +14,9 @@ //! Local backend backup export: sealed, KEK-protected bundle production. //! -//! This is the producer side only; restore lives in a follow-up change. The -//! admin API is not wired here either — callers construct the request and -//! supply the backup KEK explicitly. +//! This is the producer side; the consumer side lives in +//! [`crate::backup::local_restore`]. The admin API is not wired here — +//! callers construct the request and supply the backup KEK explicitly. //! //! # Bundle layout //! @@ -63,6 +63,7 @@ use aes_gcm::{ use jiff::Zoned; use rand::RngExt; use serde::Deserialize; +use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; use tokio::fs; use tokio::io::AsyncWriteExt; @@ -76,6 +77,12 @@ const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc"; const AEAD_NONCE_LEN: usize = 12; /// Domain-separation context for the artifact AAD binding. const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1"; +/// Domain-separation context for the master-key verifier. +const MASTER_KEY_VERIFIER_CONTEXT: &str = "rustfs-kms-local-master-key-verifier:v1"; +/// Verifier scheme prefix for the Argon2id (salted) derivation. +pub(crate) const MASTER_KEY_VERIFIER_ARGON2ID_PREFIX: &str = "argon2id-v1:"; +/// Verifier scheme prefix for the legacy pre-beta.9 SHA-256 derivation. +pub(crate) const MASTER_KEY_VERIFIER_LEGACY_PREFIX: &str = "legacy-sha256-v1:"; /// Caller-supplied backup KEK: a trust root deliberately separate from the /// business KMS hierarchy (it must not be a key that is itself part of the @@ -208,7 +215,15 @@ pub async fn export_local_backup( )); } - let manifest = build_and_write_bundle(kek, request, &snapshot).await?; + // The verifier lets a restore detect a wrong operator-supplied master key + // before touching any target state; dev-mode directories have no master + // key to verify. + let master_key_verifier = match client.configured_master_key() { + Some(master_key) => Some(compute_master_key_verifier(master_key, snapshot.salt.as_deref(), &request.backup_id)?), + None => None, + }; + + let manifest = build_and_write_bundle(kek, request, &snapshot, master_key_verifier).await?; Ok(manifest) } @@ -262,6 +277,21 @@ pub async fn decrypt_bundle_artifact( Err(error) => return Err(error.into()), }; + decrypt_artifact_payload(&manifest.backup_id, manifest.snapshot_generation, descriptor, kek, &payload) +} + +/// Verify and decrypt one artifact payload already read into memory. +/// +/// Fail-closed order: declared length, encrypted digest, nonce framing, then +/// AEAD authentication. Shared by [`decrypt_bundle_artifact`] and the export +/// pre-seal probe so producer and consumer can never drift on the framing. +fn decrypt_artifact_payload( + backup_id: &str, + snapshot_generation: u64, + descriptor: &ArtifactDescriptor, + kek: &BackupKek, + payload: &[u8], +) -> Result>> { if (payload.len() as u64) < descriptor.len { return Err(BackupError::truncated(format!( "artifact '{}' is {} bytes, manifest declares {}", @@ -280,7 +310,7 @@ pub async fn decrypt_bundle_artifact( )) .into()); } - if ContentDigest::sha256_of(&payload) != descriptor.encrypted_digest { + if ContentDigest::sha256_of(payload) != descriptor.encrypted_digest { return Err(BackupError::corrupted(format!("artifact '{}' does not match its manifest digest", descriptor.path)).into()); } if payload.len() < AEAD_NONCE_LEN { @@ -290,7 +320,7 @@ pub async fn decrypt_bundle_artifact( let (nonce_bytes, ciphertext) = payload.split_at(AEAD_NONCE_LEN); let mut nonce = [0u8; AEAD_NONCE_LEN]; nonce.copy_from_slice(nonce_bytes); - let aad = artifact_aad(&manifest.backup_id, manifest.snapshot_generation, &descriptor.path); + let aad = artifact_aad(backup_id, snapshot_generation, &descriptor.path); let plaintext = kek .cipher() .decrypt( @@ -360,6 +390,7 @@ async fn build_and_write_bundle( kek: &BackupKek, request: &LocalBackupExportRequest, snapshot: &CollectedSnapshot, + master_key_verifier: Option, ) -> Result { let mut artifacts = Vec::with_capacity(snapshot.records.len() + 1); for record in &snapshot.records { @@ -392,7 +423,7 @@ async fn build_and_write_bundle( snapshot_generation: request.snapshot_generation, backup_kek: kek.descriptor(), artifacts, - local_kdf: Some(local_kdf_descriptor(snapshot)), + local_kdf: Some(local_kdf_descriptor(snapshot, master_key_verifier)), key_versions: None, capability_discovery: None, completeness: CompletenessState::InProgress, @@ -448,13 +479,25 @@ async fn encrypt_and_write_artifact( ))); } - Ok(ArtifactDescriptor { + let descriptor = ArtifactDescriptor { kind, path: artifact_path.to_string(), len: payload.len() as u64, aead_algorithm: AeadAlgorithm::Aes256Gcm, encrypted_digest: digest, - }) + }; + + // Pre-seal decryption probe: digest equality only proves the ciphertext + // landed intact; this proves the stored artifact actually opens under the + // backup KEK and AAD binding before the manifest may reference it. + let reopened = decrypt_artifact_payload(&request.backup_id, request.snapshot_generation, &descriptor, kek, &written)?; + if reopened.as_slice() != plaintext { + return Err(KmsError::internal_error(format!( + "bundle artifact '{artifact_path}' failed the pre-seal decryption probe" + ))); + } + + Ok(descriptor) } /// AAD binding an artifact to its bundle identity and path. A JSON tuple @@ -464,6 +507,29 @@ fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) .expect("AAD tuple of strings and integers always serializes") } +/// Compute the opaque one-way master-key verifier recorded in the manifest: +/// `` + `hex(SHA-256(json(context, backup_id) || derived_key))`. +/// +/// The derivation follows the directory's KDF state: Argon2id over the +/// persistent salt when one exists, the legacy SHA-256 derivation otherwise. +/// The verifier is not an offline-guessing oracle: computing a candidate +/// requires the salt, which exists only inside the KEK-sealed bundle, and a +/// party holding the KEK already has the strictly stronger oracle of the +/// artifact AEAD tags — while every guess still pays the full Argon2id cost. +/// Binding `backup_id` prevents cross-bundle fingerprint correlation. +pub(crate) fn compute_master_key_verifier(master_key: &str, salt: Option<&[u8]>, backup_id: &str) -> Result { + let framing = serde_json::to_vec(&(MASTER_KEY_VERIFIER_CONTEXT, backup_id)) + .expect("verifier framing tuple of strings always serializes"); + let (prefix, derived) = match salt { + Some(salt) => (MASTER_KEY_VERIFIER_ARGON2ID_PREFIX, LocalKmsClient::derive_master_key(master_key, salt)?), + None => (MASTER_KEY_VERIFIER_LEGACY_PREFIX, LocalKmsClient::derive_legacy_master_key(master_key)?), + }; + let mut hasher = Sha256::new(); + hasher.update(&framing); + hasher.update(derived.as_slice()); + Ok(format!("{prefix}{}", hex::encode(hasher.finalize()))) +} + /// The bundle-level protection label is the weakest state observed across /// records: any plaintext-dev-only record marks the whole bundle, then any /// legacy-unspecified marker (unknown until read), and only a uniformly @@ -484,7 +550,7 @@ fn weakest_observed_protection(records: &[CollectedRecord]) -> AtRestProtection } } -fn local_kdf_descriptor(snapshot: &CollectedSnapshot) -> LocalKdfDescriptor { +fn local_kdf_descriptor(snapshot: &CollectedSnapshot, master_key_verifier: Option) -> LocalKdfDescriptor { let mut modes = Vec::new(); for (marker, mode) in [ (StoredKeyProtection::EncryptedMasterKey, AtRestProtection::EncryptedMasterKey), @@ -508,9 +574,7 @@ fn local_kdf_descriptor(snapshot: &CollectedSnapshot) -> LocalKdfDescriptor { LocalKdfDescriptor { derivation, protection_modes: modes, - // The verifier shape is left to the restore change; the schema keeps - // it optional so bundles without one stay valid. - master_key_verifier: None, + master_key_verifier, } } @@ -543,8 +607,9 @@ async fn write_new_file(path: &Path, bytes: &[u8]) -> Result<()> { /// Fsync a directory so freshly created bundle entries survive power loss. /// No-op on non-Unix platforms where directories cannot be opened for -/// syncing (mirrors the local backend's durable commit helper). -async fn fsync_dir(path: &Path) -> Result<()> { +/// syncing (mirrors the local backend's durable commit helper). Shared with +/// the restore module for the same durability points on the target side. +pub(crate) async fn fsync_dir(path: &Path) -> Result<()> { #[cfg(unix)] { let path = path.to_path_buf(); @@ -725,6 +790,37 @@ mod tests { assert_eq!(decrypted.as_slice(), source_record.as_slice()); } + #[tokio::test] + async fn export_records_a_master_key_verifier_matching_recomputation() { + let (client, _key_dir) = encrypted_client().await; + client.create_key("verified", "AES_256", None).await.expect("create key"); + + let bundle = TempDir::new().expect("bundle dir"); + let manifest = export_local_backup(&client, &test_kek(), &export_request(bundle.path().join("bundle"))) + .await + .expect("export should succeed"); + + let verifier = manifest + .local_kdf + .as_ref() + .expect("local kdf descriptor") + .master_key_verifier + .as_deref() + .expect("encrypted bundles must record a verifier"); + assert!(verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX), "got {verifier:?}"); + + // The verifier is a pure function of (master key, salt, backup id): + // the restore side recomputes it from the operator-supplied key and + // the bundled salt. + let salt = fs::read(client.master_key_salt_file()).await.expect("salt"); + let recomputed = compute_master_key_verifier("test-master-key", Some(&salt), "backup-0001").expect("recompute"); + assert_eq!(recomputed, verifier); + let wrong_key = compute_master_key_verifier("wrong-master-key", Some(&salt), "backup-0001").expect("recompute"); + assert_ne!(wrong_key, verifier, "a different master key must change the verifier"); + let other_bundle = compute_master_key_verifier("test-master-key", Some(&salt), "backup-0002").expect("recompute"); + assert_ne!(other_bundle, verifier, "verifiers must be bundle-bound"); + } + #[tokio::test] async fn export_fence_blocks_writers_until_released() { let (client, _key_dir) = encrypted_client().await; diff --git a/crates/kms/src/backup/local_restore.rs b/crates/kms/src/backup/local_restore.rs new file mode 100644 index 000000000..c496c16bc --- /dev/null +++ b/crates/kms/src/backup/local_restore.rs @@ -0,0 +1,1970 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Local backend restore: consume a sealed backup bundle into a key directory. +//! +//! The consumer side of [`crate::backup::local_export`]. The admin API is not +//! wired here — callers construct the request and supply the backup KEK and +//! the operator-re-supplied master key explicitly. The bundle source is +//! strictly read-only throughout: restore never writes to or deletes from it. +//! +//! # Four-phase protocol +//! +//! 1. **Dry-run (zero writes)** — [`dry_run_local_restore`] decodes the full +//! bundle chain in memory, checks the KDF descriptor against this build, +//! enumerates target conflicts, and verifies the operator-supplied master +//! key against the manifest's one-way verifier. It writes nothing. +//! 2. **Staging** — decrypted artifacts are committed durably into the +//! `.restore-staging/` subdirectory of the target. The leading dot and the +//! directory shape keep it invisible to the backend's key scan and its +//! orphan-temp matcher, so startup recovery can never delete staged state. +//! Every record is decryption-probed with the derived master key before +//! and after staging. +//! 3. **Commit marker + cutover** — the durably published +//! `.restore-commit.json` marker is the single commit point. Cutover +//! publishes each staged file into the target top level via the no-clobber +//! [`durable_file::link_durably`] primitive (salt first, keys after), then +//! durably removes the marker and drops the staging directory. +//! 4. **Crash re-entry** — before the marker, the target top level is +//! untouched and a re-run starts over; with the marker published, backend +//! startup fails closed and re-running the restore with the same bundle +//! rolls the cutover forward, while [`abort_local_restore`] rolls it back. +//! Every interruption converges to the complete old or complete new state. +//! +//! # Deliberately out of scope +//! +//! The `explicit-remap` conflict policy is not implemented: remapping stable +//! key ids requires proving that object envelopes migrate in lockstep, and a +//! bulk rekey is a non-goal of this change. Local has no rotation history +//! (backlog#1565), so per-key version and deletion-state regression cannot +//! become write operations under the empty-target policy; the snapshot +//! generation check freezes the injected observe-and-reject-strictly-lower +//! semantics for the admin layer. + +use crate::backends::local::{ + LOCAL_KMS_MASTER_KEY_SALT_FILE, LOCAL_KMS_MASTER_KEY_SALT_LEN, LOCAL_RESTORE_COMMIT_MARKER_FILE, LocalKmsClient, + StoredKeyProtection, durable_file, is_orphan_commit_temp_name, validate_key_id, +}; +use crate::backup::capability::AtRestProtection; +use crate::backup::dry_run::{ + ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport, +}; +use crate::backup::error::BackupError; +use crate::backup::local_export::{ + BackupKek, MASTER_KEY_VERIFIER_ARGON2ID_PREFIX, MASTER_KEY_VERIFIER_LEGACY_PREFIX, compute_master_key_verifier, + decrypt_bundle_artifact, fsync_dir, read_local_bundle_manifest, +}; +use crate::backup::manifest::{ArtifactKind, BackupManifest, ContentDigest, LocalKeyDerivation}; +use crate::error::{KmsError, Result}; +use aes_gcm::{ + Aes256Gcm, Nonce, + aead::{Aead, KeyInit}, +}; +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use tokio::fs; +use tracing::warn; +use zeroize::Zeroizing; + +/// Name of the staging subdirectory inside the restore target. +const RESTORE_STAGING_DIR: &str = ".restore-staging"; +/// Format version of the cutover commit marker. +const RESTORE_MARKER_FORMAT_VERSION: u32 = 1; +/// Sentinel key id for conflicts that concern the whole target rather than +/// one key (the snapshot generation). +const WHOLE_TARGET_CONFLICT_ID: &str = "*"; + +/// How restore resolves conflicts with existing target state. +/// +/// Silent overwrite or merge is never an option; `explicit-remap` is +/// deliberately not offered (see the module docs). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RestoreConflictPolicy { + /// Never write to the target. The default: any actual restore attempt is + /// refused, so mutating a key directory always requires the explicit + /// opt-in below. Dry-runs are unaffected. + #[default] + Fail, + /// Restore only into an empty target. A target counts as non-empty when + /// it holds any key file, a KDF salt (an orphan salt would poison every + /// key created later), a marker of a different restore, or any other + /// unexpected entry; only leftovers of an interrupted run of this same + /// restore are tolerated. + RestoreIntoEmptyTarget, +} + +/// Parameters of one restore (or dry-run) attempt. +/// +/// `observed_generation` is injected by the caller, mirroring the export +/// request's `snapshot_generation`: the admin layer owns where the target's +/// highest observed generation is tracked; this module freezes only the +/// strictly-lower-is-rejected semantics (equal generations stay allowed so +/// drills can be repeated). +pub struct LocalRestoreRequest { + /// Bundle directory produced by the Local export. Read-only. + pub bundle_dir: PathBuf, + /// Key directory to restore into. + pub target_key_dir: PathBuf, + /// Deployment identity of the restore target; must match the manifest. + pub target_deployment_identity: String, + /// Highest snapshot generation the target has already observed, when the + /// caller tracks one. + pub observed_generation: Option, + /// Operator-re-supplied master key string. The master key is outside the + /// backup domain, so it must arrive out of band; required whenever the + /// bundle carries encrypted-at-rest records. + pub master_key: Option, + /// Conflict policy; see [`RestoreConflictPolicy`]. + pub conflict_policy: RestoreConflictPolicy, + /// Unix permission bits for restored files, mirroring + /// `LocalConfig::file_permissions`. + pub file_permissions: Option, +} + +impl std::fmt::Debug for LocalRestoreRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalRestoreRequest") + .field("bundle_dir", &self.bundle_dir) + .field("target_key_dir", &self.target_key_dir) + .field("target_deployment_identity", &self.target_deployment_identity) + .field("observed_generation", &self.observed_generation) + .field("master_key", &self.master_key.as_ref().map(|_| "")) + .field("conflict_policy", &self.conflict_policy) + .field("file_permissions", &self.file_permissions) + .finish() + } +} + +impl LocalRestoreRequest { + fn validate(&self) -> Result<()> { + if self.target_deployment_identity.is_empty() { + return Err(KmsError::validation_error("restore target_deployment_identity must not be empty")); + } + Ok(()) + } +} + +/// Serializable outcome of a completed restore. Identifiers only; never key +/// material or secrets. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalRestoreReport { + /// Identifier of the restored bundle. + pub backup_id: String, + /// Snapshot generation the target now holds. + pub snapshot_generation: u64, + /// Stable key ids restored into the target. + pub restored_key_ids: Vec, + /// Whether the master-key KDF salt was restored. + pub salt_restored: bool, + /// Whether this run rolled forward a previously interrupted cutover. + pub resumed: bool, +} + +/// The cutover commit marker: its durable publication is the single commit +/// point of a restore. It names the exact bundle and the exact top-level +/// files the cutover publishes, so re-entry can verify it is completing the +/// same restore and an abort knows precisely what to take back. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RestoreCommitMarker { + format_version: u32, + backup_id: String, + manifest_digest: ContentDigest, + /// Final top-level file names in link order (salt first, then keys). + files: Vec, +} + +impl RestoreCommitMarker { + fn for_bundle(decoded: &DecodedBundle) -> Self { + let mut files = Vec::with_capacity(decoded.records.len() + 1); + if decoded.salt.is_some() { + files.push(LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string()); + } + files.extend(decoded.records.iter().map(|record| record.file_name.clone())); + Self { + format_version: RESTORE_MARKER_FORMAT_VERSION, + backup_id: decoded.manifest.backup_id.clone(), + manifest_digest: decoded.manifest.manifest_digest.clone(), + files, + } + } + + /// Strict decode: an unreadable or out-of-contract marker fails closed — + /// abort and roll-forward both refuse to guess what a foreign or damaged + /// marker was committing. + fn decode(bytes: &[u8]) -> Result { + let marker: Self = serde_json::from_slice(bytes) + .map_err(|error| BackupError::corrupted(format!("restore commit marker does not decode: {error}")))?; + if marker.format_version != RESTORE_MARKER_FORMAT_VERSION { + return Err(BackupError::corrupted(format!( + "restore commit marker has unknown format version {}", + marker.format_version + )) + .into()); + } + if marker.files.is_empty() { + return Err(BackupError::corrupted("restore commit marker lists no files").into()); + } + for name in &marker.files { + if name == LOCAL_KMS_MASTER_KEY_SALT_FILE { + continue; + } + match name.strip_suffix(".key") { + Some(stem) if validate_key_id(stem).is_ok() => {} + _ => { + return Err(BackupError::corrupted(format!( + "restore commit marker lists a file name outside the restore contract: {name:?}" + )) + .into()); + } + } + } + Ok(marker) + } + + fn matches_bundle(&self, manifest: &BackupManifest) -> bool { + self.backup_id == manifest.backup_id && self.manifest_digest == manifest.manifest_digest + } +} + +/// One key record decoded out of the bundle, fully re-verified. +struct DecodedRecord { + key_id: String, + /// Final on-disk file name (`.key`). + file_name: String, + protection: StoredKeyProtection, + /// Whether the record's material is AEAD-protected under the master key + /// (explicit marker, or a legacy record with a nonce). + effectively_encrypted: bool, + /// Base64-decoded `encrypted_key_material` field. + material: Zeroizing>, + nonce: Vec, + /// Raw record bytes exactly as exported; what staging writes. + plaintext: Zeroizing>, +} + +/// The fully decoded and cross-checked bundle content. +struct DecodedBundle { + manifest: BackupManifest, + records: Vec, + salt: Option>>, +} + +impl DecodedBundle { + fn any_record_encrypted(&self) -> bool { + self.records.iter().any(|record| record.effectively_encrypted) + } +} + +/// Minimal projection of a stored key record, mirroring the export-side +/// probe. Unknown fields are ignored on purpose: the record is restored +/// byte-identical, so restore must not constrain its schema. +#[derive(Deserialize)] +struct RestoredRecordProbe { + key_id: String, + #[serde(default)] + at_rest_protection: StoredKeyProtection, + encrypted_key_material: String, + #[serde(default)] + nonce: Vec, +} + +/// Zero-write restore preflight: evaluate the bundle against the target and +/// report every blocker, conflict, and external dependency mismatch. +/// +/// The bundle is decoded fully in memory (including AEAD verification of +/// every artifact); the target directory is only read. Environment failures +/// (unreadable directories) surface as errors, verdicts about the bundle or +/// the target surface inside the report. +pub async fn dry_run_local_restore(kek: &BackupKek, request: &LocalRestoreRequest) -> Result { + request.validate()?; + let mut report = RestoreDryRunReport { + backup_id: String::new(), + target_deployment_identity: request.target_deployment_identity.clone(), + blockers: Vec::new(), + conflicts: Vec::new(), + external_mismatches: Vec::new(), + }; + + let decoded = match decode_bundle(&request.bundle_dir, kek).await { + Ok(decoded) => decoded, + Err(error) => match blocker_for(&error) { + Some(blocker) => { + report.blockers.push(blocker); + return Ok(report); + } + None => return Err(error), + }, + }; + report.backup_id = decoded.manifest.backup_id.clone(); + + if let Some(detail) = kdf_drift_detail(&decoded.manifest) { + report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::UnsupportedBackend, + detail, + }); + } + if decoded.manifest.deployment_identity != request.target_deployment_identity { + report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::DeploymentMismatch, + detail: format!( + "bundle was produced by deployment '{}', restore target is '{}'", + decoded.manifest.deployment_identity, request.target_deployment_identity + ), + }); + } + if let Some(observed) = request.observed_generation + && decoded.manifest.snapshot_generation < observed + { + report.conflicts.push(RestoreConflict { + key_id: WHOLE_TARGET_CONFLICT_ID.to_string(), + kind: RestoreConflictKind::GenerationRegression, + detail: format!( + "target observed snapshot generation {observed}, bundle carries {}; equal generations stay allowed for repeated drills", + decoded.manifest.snapshot_generation + ), + }); + } + + match check_master_key(&decoded, request.master_key.as_deref())? { + MasterKeyCheck::Ok => {} + MasterKeyCheck::MissingPassphrase(detail) => report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::ExternalDependencyUnavailable, + detail, + }), + MasterKeyCheck::UnknownVerifierScheme(detail) => report.blockers.push(RestoreBlocker { + code: RestoreBlockerCode::BundleCorrupted, + detail, + }), + MasterKeyCheck::Mismatch { expected, observed } => report.external_mismatches.push(ExternalDependencyMismatch { + dependency: "local KMS master key (operator-supplied)".to_string(), + expected, + observed, + }), + } + + report + .conflicts + .extend(enumerate_target_conflicts(&request.target_key_dir, &decoded.manifest).await?); + Ok(report) +} + +/// Restore a sealed Local bundle into the target key directory. +/// +/// Fail-fast counterpart of the dry-run: the first violated precondition +/// aborts with its typed error before the target top level is touched. Only +/// [`RestoreConflictPolicy::RestoreIntoEmptyTarget`] ever writes. +pub async fn restore_local_backup(kek: &BackupKek, request: &LocalRestoreRequest) -> Result { + request.validate()?; + if request.conflict_policy == RestoreConflictPolicy::Fail { + return Err(KmsError::invalid_operation( + "restore conflict policy 'fail' never writes to the target; review the dry-run report and \ + re-run with the 'restore-into-empty-target' policy to proceed", + )); + } + + let decoded = decode_bundle(&request.bundle_dir, kek).await?; + if let Some(detail) = kdf_drift_detail(&decoded.manifest) { + return Err(KmsError::unsupported_capability("local-restore", detail)); + } + if decoded.manifest.deployment_identity != request.target_deployment_identity { + return Err(KmsError::invalid_operation(format!( + "bundle was produced by deployment '{}' but the restore target is '{}'", + decoded.manifest.deployment_identity, request.target_deployment_identity + ))); + } + if let Some(observed) = request.observed_generation + && decoded.manifest.snapshot_generation < observed + { + return Err(KmsError::invalid_operation(format!( + "restore would regress the snapshot generation: target observed {observed}, bundle carries {}", + decoded.manifest.snapshot_generation + ))); + } + match check_master_key(&decoded, request.master_key.as_deref())? { + MasterKeyCheck::Ok => {} + MasterKeyCheck::MissingPassphrase(detail) => return Err(KmsError::invalid_operation(detail)), + MasterKeyCheck::UnknownVerifierScheme(detail) => return Err(BackupError::corrupted(detail).into()), + MasterKeyCheck::Mismatch { .. } => { + return Err(KmsError::validation_error( + "operator-supplied master key does not match the bundle's master-key verifier", + )); + } + } + + // Decryption-probe every record in memory before anything is written: a + // wrong master key on a verifier-less legacy bundle must fail here, with + // the target untouched. + let ciphers = derive_ciphers(request.master_key.as_deref(), decoded.salt.as_deref().map(Vec::as_slice))?; + for record in &decoded.records { + probe_record_decryption(record, &ciphers)?; + } + + let mode = inspect_target_for_restore(&request.target_key_dir, &decoded.manifest).await?; + let expected_marker = RestoreCommitMarker::for_bundle(&decoded); + let resumed = match &mode { + TargetMode::Fresh { .. } => false, + TargetMode::RollForward(published) => { + if *published != expected_marker { + return Err(BackupError::corrupted( + "published restore marker names the same bundle but a different file set; \ + abort the interrupted restore before retrying", + ) + .into()); + } + true + } + }; + + // Phase B: staging. + let staging = request.target_key_dir.join(RESTORE_STAGING_DIR); + if let TargetMode::Fresh { stale_entries } = &mode { + // A fresh run owns no prior state: drop staging leftovers of an + // earlier interrupted attempt and any orphan commit temps, then start + // over from the bundle. + if fs::try_exists(&staging).await? { + remove_dir_all_durably(&request.target_key_dir, &staging).await?; + } + for stale in stale_entries { + durable_file::remove_durably(request.target_key_dir.join(stale)).await?; + } + } + fs::create_dir_all(&staging).await?; + if let Some(salt) = &decoded.salt { + stage_file(&staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, salt, request.file_permissions).await?; + } + for record in &decoded.records { + stage_file(&staging, &record.file_name, &record.plaintext, request.file_permissions).await?; + } + // Make the staging directory entry itself durable before the commit + // point: the marker must never survive a crash that its staged files did + // not. + fsync_dir(&request.target_key_dir).await?; + probe_staged_bundle(&staging, &decoded, &ciphers).await?; + + // Phase C: commit point, then cutover. + if !resumed { + publish_marker(&request.target_key_dir, &expected_marker, request.file_permissions).await?; + } + cutover_and_finish(&request.target_key_dir, &staging, &expected_marker).await?; + + Ok(LocalRestoreReport { + backup_id: decoded.manifest.backup_id.clone(), + snapshot_generation: decoded.manifest.snapshot_generation, + restored_key_ids: decoded.records.iter().map(|record| record.key_id.clone()).collect(), + salt_restored: decoded.salt.is_some(), + resumed, + }) +} + +/// Explicitly abort an interrupted restore, returning the target to its +/// pre-restore (empty) state. +/// +/// With a published marker this takes back exactly the files the marker +/// names, then the marker, then the staging directory. Without a marker only +/// staging leftovers exist and are removed. Fails closed on a marker it +/// cannot strictly decode. +pub async fn abort_local_restore(target_key_dir: &Path) -> Result<()> { + let marker_path = target_key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + let staging = target_key_dir.join(RESTORE_STAGING_DIR); + match fs::read(&marker_path).await { + Ok(bytes) => { + let marker = RestoreCommitMarker::decode(&bytes)?; + for file_name in &marker.files { + let path = target_key_dir.join(file_name); + if fs::try_exists(&path).await? { + durable_file::remove_durably(path).await?; + } + } + durable_file::remove_durably(marker_path).await?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if !fs::try_exists(&staging).await? { + return Err(KmsError::invalid_operation(format!( + "no restore is in progress under {}", + target_key_dir.display() + ))); + } + } + Err(error) => return Err(error.into()), + } + if fs::try_exists(&staging).await? { + remove_dir_all_durably(target_key_dir, &staging).await?; + } + Ok(()) +} + +/// Decode and cross-verify the complete bundle in memory. +/// +/// Every artifact is read, digest-verified, and AEAD-authenticated; every key +/// record is re-parsed, identity-checked against its artifact path, and +/// checked for protection consistency with the manifest's KDF descriptor. +async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result { + let manifest = read_local_bundle_manifest(bundle_dir).await?; + let descriptor = manifest + .local_kdf + .clone() + .ok_or_else(|| BackupError::corrupted("local bundle manifest carries no KDF descriptor"))?; + + let mut records = Vec::new(); + let mut salt: Option>> = None; + for artifact in &manifest.artifacts { + match artifact.kind { + ArtifactKind::KeyMaterial => { + let plaintext = decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?; + records.push(decode_key_record(artifact.path.as_str(), plaintext, &descriptor.protection_modes)?); + } + ArtifactKind::MasterKeySalt => { + if salt.is_some() { + return Err(BackupError::corrupted("bundle carries more than one master-key salt artifact").into()); + } + let plaintext = decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?; + if plaintext.len() != LOCAL_KMS_MASTER_KEY_SALT_LEN { + return Err(BackupError::corrupted(format!( + "bundled master-key salt is {} bytes, expected {}", + plaintext.len(), + LOCAL_KMS_MASTER_KEY_SALT_LEN + )) + .into()); + } + salt = Some(plaintext); + } + // No producer emits these for a Local bundle yet; restoring a + // bundle that carries state this implementation cannot apply + // would silently drop it, so fail closed instead. + other => { + return Err(KmsError::unsupported_capability( + "local-restore", + format!("bundle carries artifact kind {other:?}, which this restore implementation cannot apply"), + )); + } + } + } + + // The salt must be exactly as coherent with the KDF descriptor as the + // export invariants promise; anything else is a broken or tampered + // bundle. + match (&descriptor.derivation, salt.is_some()) { + (LocalKeyDerivation::Argon2id { .. }, false) => { + return Err( + BackupError::corrupted("KDF descriptor declares Argon2id but the bundle carries no salt artifact").into(), + ); + } + (LocalKeyDerivation::LegacySha256, true) => { + return Err( + BackupError::corrupted("KDF descriptor declares the legacy derivation but the bundle carries a salt").into(), + ); + } + _ => {} + } + + Ok(DecodedBundle { manifest, records, salt }) +} + +fn decode_key_record( + artifact_path: &str, + plaintext: Zeroizing>, + allowed_modes: &[AtRestProtection], +) -> Result { + let stem = Path::new(artifact_path) + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_suffix(".key.enc")) + .ok_or_else(|| { + BackupError::corrupted(format!( + "key material artifact path {artifact_path:?} does not follow the '.key.enc' layout" + )) + })? + .to_string(); + validate_key_id(&stem)?; + + let probe: RestoredRecordProbe = serde_json::from_slice(&plaintext) + .map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' does not deserialize: {error}")))?; + if probe.key_id != stem { + return Err(BackupError::corrupted(format!( + "bundled key record identity mismatch: artifact names {stem:?}, record names {:?}", + probe.key_id + )) + .into()); + } + if probe.encrypted_key_material.is_empty() { + return Err(BackupError::corrupted(format!("bundled key record '{stem}' carries no key material")).into()); + } + let material = + Zeroizing::new(BASE64.decode(&probe.encrypted_key_material).map_err(|error| { + BackupError::corrupted(format!("bundled key record '{stem}' material is not valid base64: {error}")) + })?); + if !allowed_modes.contains(&protection_mode(probe.at_rest_protection)) { + return Err(BackupError::corrupted(format!( + "bundled key record '{stem}' declares protection {:?}, absent from the manifest's KDF descriptor", + probe.at_rest_protection + )) + .into()); + } + + let effectively_encrypted = match probe.at_rest_protection { + StoredKeyProtection::EncryptedMasterKey => true, + StoredKeyProtection::PlaintextDevOnly => false, + StoredKeyProtection::LegacyUnspecified => !probe.nonce.is_empty(), + }; + if effectively_encrypted && probe.nonce.len() != 12 { + return Err(BackupError::corrupted(format!( + "bundled key record '{stem}' has an invalid nonce length ({} bytes, expected 12)", + probe.nonce.len() + )) + .into()); + } + + Ok(DecodedRecord { + file_name: format!("{stem}.key"), + key_id: stem, + protection: probe.at_rest_protection, + effectively_encrypted, + material, + nonce: probe.nonce, + plaintext, + }) +} + +fn protection_mode(protection: StoredKeyProtection) -> AtRestProtection { + match protection { + StoredKeyProtection::EncryptedMasterKey => AtRestProtection::EncryptedMasterKey, + StoredKeyProtection::PlaintextDevOnly => AtRestProtection::PlaintextDevOnly, + StoredKeyProtection::LegacyUnspecified => AtRestProtection::LegacyUnspecified, + } +} + +/// Detail message when the bundle's recorded KDF differs from what this build +/// compiles in; restoring such a bundle would produce keys this build cannot +/// decrypt. +fn kdf_drift_detail(manifest: &BackupManifest) -> Option { + match &manifest.local_kdf.as_ref()?.derivation { + derivation @ LocalKeyDerivation::Argon2id { .. } => { + let current = LocalKeyDerivation::current_argon2id(); + (*derivation != current).then(|| { + format!("bundle KDF descriptor {derivation:?} does not match this build's compiled-in derivation {current:?}") + }) + } + LocalKeyDerivation::LegacySha256 => None, + } +} + +enum MasterKeyCheck { + Ok, + MissingPassphrase(String), + UnknownVerifierScheme(String), + Mismatch { expected: String, observed: String }, +} + +/// Verify the operator-supplied master key against the manifest's one-way +/// verifier, and require one whenever the bundle carries encrypted records. +/// +/// Bundles without a verifier (produced before the verifier landed) fall +/// back to the staging-phase decryption probe; unknown verifier schemes fail +/// closed. +fn check_master_key(decoded: &DecodedBundle, master_key: Option<&str>) -> Result { + let Some(master_key) = master_key else { + if decoded.any_record_encrypted() { + return Ok(MasterKeyCheck::MissingPassphrase( + "bundle carries encrypted-at-rest key records; the operator must re-supply the master key before restore" + .to_string(), + )); + } + return Ok(MasterKeyCheck::Ok); + }; + + let verifier = decoded + .manifest + .local_kdf + .as_ref() + .and_then(|descriptor| descriptor.master_key_verifier.as_deref()); + let Some(verifier) = verifier else { + return Ok(MasterKeyCheck::Ok); + }; + + let observed = if verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX) { + let Some(salt) = decoded.salt.as_deref() else { + return Err( + BackupError::corrupted("manifest carries an Argon2id master-key verifier but the bundle has no salt").into(), + ); + }; + compute_master_key_verifier(master_key, Some(salt), &decoded.manifest.backup_id)? + } else if verifier.starts_with(MASTER_KEY_VERIFIER_LEGACY_PREFIX) { + compute_master_key_verifier(master_key, None, &decoded.manifest.backup_id)? + } else { + return Ok(MasterKeyCheck::UnknownVerifierScheme( + "manifest carries a master-key verifier with an unknown scheme prefix; failing closed".to_string(), + )); + }; + + if observed != verifier { + return Ok(MasterKeyCheck::Mismatch { + expected: verifier.to_string(), + observed, + }); + } + Ok(MasterKeyCheck::Ok) +} + +/// AEAD ciphers derived from the operator-supplied master key, mirroring the +/// backend's current-plus-legacy pair. +struct DerivedCiphers { + current: Option, + legacy: Option, +} + +fn derive_ciphers(master_key: Option<&str>, salt: Option<&[u8]>) -> Result { + let Some(master_key) = master_key else { + return Ok(DerivedCiphers { + current: None, + legacy: None, + }); + }; + let legacy = Aes256Gcm::new(&LocalKmsClient::derive_legacy_master_key(master_key)?); + let current = match salt { + Some(salt) => Aes256Gcm::new(&LocalKmsClient::derive_master_key(master_key, salt)?), + None => legacy.clone(), + }; + Ok(DerivedCiphers { + current: Some(current), + legacy: Some(legacy), + }) +} + +/// Decryption probe for one record, mirroring the backend's read path: the +/// current cipher first, with the legacy fallback only for legacy-unmarked +/// records. Local has no rotation history, so probing every record is the +/// full-coverage probe. +fn probe_record_decryption(record: &DecodedRecord, ciphers: &DerivedCiphers) -> Result<()> { + if !record.effectively_encrypted { + return Ok(()); + } + let Some(current) = ciphers.current.as_ref() else { + return Err(KmsError::invalid_operation(format!( + "bundled key record '{}' is encrypted at rest and requires the operator-supplied master key", + record.key_id + ))); + }; + let mut nonce = [0u8; 12]; + nonce.copy_from_slice(&record.nonce); + let nonce = Nonce::from(nonce); + match current.decrypt(&nonce, record.material.as_slice()) { + Ok(material) => { + drop(Zeroizing::new(material)); + Ok(()) + } + Err(_) if record.protection == StoredKeyProtection::LegacyUnspecified => { + let legacy = ciphers + .legacy + .as_ref() + .expect("legacy cipher exists whenever the current cipher does"); + let material = legacy + .decrypt(&nonce, record.material.as_slice()) + .map_err(|_| KmsError::material_authentication_failed(&record.key_id))?; + drop(Zeroizing::new(material)); + Ok(()) + } + Err(_) => Err(KmsError::material_authentication_failed(&record.key_id)), + } +} + +/// Re-run the decryption probe against the bytes that actually landed in +/// staging, so cutover only ever publishes state proven readable from disk. +async fn probe_staged_bundle(staging: &Path, decoded: &DecodedBundle, ciphers: &DerivedCiphers) -> Result<()> { + if let Some(salt) = &decoded.salt { + let staged_salt = fs::read(staging.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)).await?; + if staged_salt.as_slice() != salt.as_slice() { + return Err(BackupError::corrupted("staged master-key salt does not match the bundle").into()); + } + } + for record in &decoded.records { + let staged = Zeroizing::new(fs::read(staging.join(&record.file_name)).await?); + if staged.as_slice() != record.plaintext.as_slice() { + return Err( + BackupError::corrupted(format!("staged key record '{}' does not match the bundle", record.key_id)).into(), + ); + } + let staged_record = decode_key_record( + &format!("staged/{}.enc", record.file_name), + staged, + &decoded + .manifest + .local_kdf + .as_ref() + .expect("decoded bundle always carries a KDF descriptor") + .protection_modes, + )?; + probe_record_decryption(&staged_record, ciphers)?; + } + Ok(()) +} + +enum TargetMode { + /// The target is empty apart from removable leftovers of an interrupted + /// earlier attempt (staging, orphan commit temps). + Fresh { stale_entries: Vec }, + /// A published marker of this same bundle: complete its cutover. + RollForward(RestoreCommitMarker), +} + +/// Fail-fast target inspection for an actual restore. +async fn inspect_target_for_restore(target: &Path, manifest: &BackupManifest) -> Result { + if !fs::try_exists(target).await? { + return Ok(TargetMode::Fresh { + stale_entries: Vec::new(), + }); + } + + let marker = read_marker(target).await?; + if let Some(marker) = marker { + if !marker.matches_bundle(manifest) { + return Err(KmsError::invalid_operation(format!( + "target key directory holds a restore marker for a different bundle ('{}'); \ + roll that restore forward with its own bundle or abort it explicitly", + marker.backup_id + ))); + } + // Everything at the top level must belong to this restore: the + // marker's files (partially linked) and our own working entries. + let allowed: Vec<&str> = marker.files.iter().map(String::as_str).collect(); + for entry in list_target_entries(target).await? { + if !allowed.contains(&entry.as_str()) && !is_orphan_commit_temp_name(&entry) { + return Err(KmsError::invalid_operation(format!( + "target key directory holds an entry outside the interrupted restore: {entry:?}; \ + clean it up manually before resuming" + ))); + } + } + return Ok(TargetMode::RollForward(marker)); + } + + let mut stale_entries = Vec::new(); + for entry in list_target_entries(target).await? { + if is_orphan_commit_temp_name(&entry) { + // Unpublished commit temps are never authoritative; a re-run may + // remove them exactly as startup recovery would. + stale_entries.push(entry); + continue; + } + let detail = if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE { + "an orphan KDF salt would poison every key created after the restore; clean it up manually" + } else { + "restore only ever writes into an empty target" + }; + return Err(KmsError::invalid_operation(format!( + "target key directory is not empty (found {entry:?}); {detail}" + ))); + } + Ok(TargetMode::Fresh { stale_entries }) +} + +/// Top-level entries of the target, excluding the staging directory and the +/// commit marker (both handled separately). +async fn list_target_entries(target: &Path) -> Result> { + let mut names = Vec::new(); + let mut entries = fs::read_dir(target).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry + .file_name() + .to_str() + .ok_or_else(|| KmsError::invalid_operation("target key directory holds a non-UTF-8 entry; clean it up manually"))? + .to_string(); + if name == RESTORE_STAGING_DIR || name == LOCAL_RESTORE_COMMIT_MARKER_FILE { + continue; + } + names.push(name); + } + names.sort(); + Ok(names) +} + +async fn read_marker(target: &Path) -> Result> { + match fs::read(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)).await { + Ok(bytes) => Ok(Some(RestoreCommitMarker::decode(&bytes)?)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } +} + +/// Conflict enumeration for the dry-run: every target entry that would block +/// the restore, without failing on the first one. +async fn enumerate_target_conflicts(target: &Path, manifest: &BackupManifest) -> Result> { + if !fs::try_exists(target).await? { + return Ok(Vec::new()); + } + let mut conflicts = Vec::new(); + + let marker = match read_marker(target).await { + Ok(marker) => marker, + // A marker that does not strictly decode blocks exactly like a + // foreign one; the dry-run reports it instead of failing. + Err(error) => { + conflicts.push(RestoreConflict { + key_id: LOCAL_RESTORE_COMMIT_MARKER_FILE.to_string(), + kind: RestoreConflictKind::KeyAlreadyExists, + detail: format!("target holds an undecodable restore marker: {error}"), + }); + None + } + }; + let own_files: Vec = match &marker { + Some(marker) if marker.matches_bundle(manifest) => marker.files.clone(), + Some(marker) => { + conflicts.push(RestoreConflict { + key_id: LOCAL_RESTORE_COMMIT_MARKER_FILE.to_string(), + kind: RestoreConflictKind::KeyAlreadyExists, + detail: format!("target holds a restore marker for a different bundle ('{}')", marker.backup_id), + }); + Vec::new() + } + None => Vec::new(), + }; + + for entry in list_target_entries(target).await? { + if own_files.contains(&entry) || is_orphan_commit_temp_name(&entry) { + continue; + } + let (key_id, detail) = if let Some(stem) = entry.strip_suffix(".key") { + (stem.to_string(), format!("target already has key file {entry:?}")) + } else if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE { + ( + entry.clone(), + "target holds an orphan master-key KDF salt; it would poison every key created after the restore".to_string(), + ) + } else { + (entry.clone(), format!("target holds an unexpected entry {entry:?}")) + }; + conflicts.push(RestoreConflict { + key_id, + kind: RestoreConflictKind::KeyAlreadyExists, + detail, + }); + } + Ok(conflicts) +} + +/// Durably commit one staged file. Idempotent across roll-forward re-entry: +/// an already staged file is accepted only byte-identical. +async fn stage_file(staging: &Path, file_name: &str, content: &[u8], permissions: Option) -> Result<()> { + let final_path = staging.join(file_name); + match fs::read(&final_path).await { + Ok(existing) => { + let existing = Zeroizing::new(existing); + if existing.as_slice() == content { + return Ok(()); + } + return Err(BackupError::corrupted(format!( + "staging already holds '{file_name}' with different content; abort the interrupted restore before retrying" + )) + .into()); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let temp_path = staging.join(format!("{file_name}.tmp-{}", uuid::Uuid::new_v4())); + durable_file::commit(temp_path, final_path, content.to_vec(), permissions, durable_file::Publish::NoClobber) + .await + .map_err(KmsError::from) +} + +/// Durably publish the commit marker. This is the restore's single commit +/// point: before it, the target top level is untouched; after it, the only +/// ways out are roll-forward or explicit abort. +async fn publish_marker(target: &Path, marker: &RestoreCommitMarker, permissions: Option) -> Result<()> { + let content = serde_json::to_vec_pretty(marker) + .map_err(|error| KmsError::internal_error(format!("restore commit marker serialization failed: {error}")))?; + let final_path = target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + let temp_path = target.join(format!("{LOCAL_RESTORE_COMMIT_MARKER_FILE}.tmp-{}", uuid::Uuid::new_v4())); + durable_file::commit(temp_path, final_path, content, permissions, durable_file::Publish::NoClobber) + .await + .map_err(KmsError::from) +} + +/// The atomic cutover: publish every staged file under its final name (salt +/// first, keys after — the marker froze that order), durably remove the +/// marker, then drop the staging directory. +async fn cutover_and_finish(target: &Path, staging: &Path, marker: &RestoreCommitMarker) -> Result<()> { + for file_name in &marker.files { + durable_file::link_durably(staging.join(file_name), target.join(file_name)) + .await + .map_err(|error| KmsError::io_error(format!("restore cutover failed publishing '{file_name}': {error}")))?; + } + durable_file::remove_durably(target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE)) + .await + .map_err(|error| KmsError::io_error(format!("restore cutover failed removing the commit marker: {error}")))?; + // The restore is complete once the marker is gone; a leftover staging + // directory is inert (its entries are hard links of the published files), + // so its removal is cleanup, not protocol. + if let Err(error) = remove_dir_all_durably(target, staging).await { + warn!(%error, "restored Local KMS key directory kept an inert staging directory"); + } + Ok(()) +} + +async fn remove_dir_all_durably(parent: &Path, dir: &Path) -> Result<()> { + let dir = dir.to_path_buf(); + tokio::task::spawn_blocking(move || std::fs::remove_dir_all(&dir)) + .await + .map_err(|error| KmsError::io_error(error.to_string()))??; + fsync_dir(parent).await +} + +fn blocker_for(error: &KmsError) -> Option { + match error { + KmsError::Backup(inner) => Some(RestoreBlocker::from(inner)), + KmsError::UnsupportedCapability { .. } => Some(RestoreBlocker { + code: RestoreBlockerCode::UnsupportedBackend, + detail: error.to_string(), + }), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::backup::local_export::{LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, export_local_backup}; + use crate::config::LocalConfig; + use sha2::{Digest, Sha256}; + use std::time::SystemTime; + use tempfile::TempDir; + + const TEST_MASTER_KEY: &str = "test-master-key"; + const DEPLOYMENT: &str = "deployment-test"; + const BACKUP_ID: &str = "backup-0001"; + const SNAPSHOT_GENERATION: u64 = 7; + + fn local_config(dir: &Path, master_key: Option<&str>) -> LocalConfig { + LocalConfig { + key_dir: dir.to_path_buf(), + master_key: master_key.map(str::to_string), + file_permissions: Some(0o600), + } + } + + async fn source_with_keys(master_key: Option<&str>, keys: &[&str]) -> (LocalKmsClient, TempDir) { + let temp = TempDir::new().expect("temp dir"); + let client = LocalKmsClient::new(local_config(temp.path(), master_key)) + .await + .expect("client should initialize"); + for key in keys { + client.create_key(key, "AES_256", None).await.expect("create key"); + } + (client, temp) + } + + fn test_kek() -> BackupKek { + BackupKek::new("backup-kek-test", 1, [0x42; 32]).expect("kek") + } + + async fn export_bundle(client: &LocalKmsClient, dir: &Path) -> (PathBuf, BackupManifest) { + let destination = dir.join("bundle"); + let manifest = export_local_backup( + client, + &test_kek(), + &LocalBackupExportRequest { + backup_id: BACKUP_ID.to_string(), + deployment_identity: DEPLOYMENT.to_string(), + rustfs_version: "1.0.0-test".to_string(), + snapshot_generation: SNAPSHOT_GENERATION, + destination: destination.clone(), + }, + ) + .await + .expect("export should succeed"); + (destination, manifest) + } + + fn restore_request(bundle: &Path, target: &Path) -> LocalRestoreRequest { + LocalRestoreRequest { + bundle_dir: bundle.to_path_buf(), + target_key_dir: target.to_path_buf(), + target_deployment_identity: DEPLOYMENT.to_string(), + observed_generation: None, + master_key: Some(TEST_MASTER_KEY.to_string()), + conflict_policy: RestoreConflictPolicy::RestoreIntoEmptyTarget, + file_permissions: Some(0o600), + } + } + + /// Recursive (path, content SHA-256, mtime) snapshot; equality across an + /// operation proves it performed no persistent write at all. + fn snapshot_tree(dir: &Path) -> Vec<(String, Vec, SystemTime)> { + fn walk(root: &Path, dir: &Path, out: &mut Vec<(String, Vec, SystemTime)>) { + for entry in std::fs::read_dir(dir).expect("read dir") { + let entry = entry.expect("dir entry"); + let path = entry.path(); + if path.is_dir() { + walk(root, &path, out); + continue; + } + let relative = path.strip_prefix(root).expect("under root").to_string_lossy().into_owned(); + let content = std::fs::read(&path).expect("read file"); + let mtime = entry.metadata().expect("metadata").modified().expect("mtime"); + out.push((relative, Sha256::digest(&content).to_vec(), mtime)); + } + } + let mut out = Vec::new(); + walk(dir, dir, &mut out); + out.sort(); + out + } + + async fn top_level_names(dir: &Path) -> Vec { + let mut names = Vec::new(); + let mut entries = fs::read_dir(dir).await.expect("read dir"); + while let Some(entry) = entries.next_entry().await.expect("dir entry") { + names.push(entry.file_name().to_string_lossy().into_owned()); + } + names.sort(); + names + } + + /// Re-seal the bundle manifest after mutating it, keeping the digest + /// contract intact so only the mutated aspect is under test. + async fn reseal_manifest(bundle_dir: &Path, mutate: impl FnOnce(&mut BackupManifest)) { + let path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE); + let mut manifest = BackupManifest::decode(&fs::read(&path).await.expect("manifest bytes")).expect("manifest decodes"); + mutate(&mut manifest); + let sealed = manifest.seal().expect("re-seal"); + fs::write(&path, sealed.encode().expect("encode")) + .await + .expect("write manifest"); + } + + /// The full acceptance check for a restored directory: raw files + /// byte-identical to the source, and every record decodable through the + /// read-only export constructor with byte-identical material. + async fn assert_restored_matches_source(source: &LocalKmsClient, target: &Path, keys: &[&str]) { + for key in keys { + let source_bytes = fs::read(source.key_directory().join(format!("{key}.key"))) + .await + .expect("source record"); + let restored_bytes = fs::read(target.join(format!("{key}.key"))).await.expect("restored record"); + assert_eq!(source_bytes, restored_bytes, "record {key} must restore byte-for-byte"); + } + let source_salt = fs::read(source.master_key_salt_file()).await.expect("source salt"); + let restored_salt = fs::read(target.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)) + .await + .expect("restored salt"); + assert_eq!(source_salt, restored_salt, "salt must restore byte-for-byte"); + + let restored = LocalKmsClient::new_for_key_export(local_config(target, Some(TEST_MASTER_KEY))) + .await + .expect("restored directory must open read-only"); + for key in keys { + let source_material = source.decrypt_key_material_for_export(key).await.expect("source material"); + let restored_material = restored + .decrypt_key_material_for_export(key) + .await + .expect("restored material"); + assert_eq!( + source_material.as_ref(), + restored_material.as_ref(), + "key material for {key} must survive the restore byte-for-byte" + ); + } + } + + #[tokio::test] + async fn dry_run_is_zero_write_and_permits_clean_restore() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + fs::create_dir_all(&target).await.expect("create target"); + + let bundle_before = snapshot_tree(&bundle); + let target_before = snapshot_tree(&target); + let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("dry run should succeed"); + + assert!(report.restore_permitted(), "clean bundle and empty target must permit: {report:?}"); + assert_eq!(report.backup_id, BACKUP_ID); + assert_eq!(report.target_deployment_identity, DEPLOYMENT); + assert_eq!(snapshot_tree(&bundle), bundle_before, "dry run must not touch the bundle"); + assert_eq!(snapshot_tree(&target), target_before, "dry run must not touch the target"); + + // A missing target directory stays missing. + let missing = work.path().join("missing-target"); + let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, &missing)) + .await + .expect("dry run should succeed"); + assert!(report.restore_permitted()); + assert!(!missing.exists(), "dry run must not create the target directory"); + } + + #[tokio::test] + async fn dry_run_flags_deployment_mismatch_and_generation_regression() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.target_deployment_identity = "deployment-other".to_string(); + request.observed_generation = Some(SNAPSHOT_GENERATION + 1); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::DeploymentMismatch), + "expected a deployment mismatch blocker: {report:?}" + ); + assert!( + report + .conflicts + .iter() + .any(|conflict| conflict.kind == RestoreConflictKind::GenerationRegression), + "expected a generation regression conflict: {report:?}" + ); + + // Equal generations are allowed so drills can repeat. + let mut request = restore_request(&bundle, &target); + request.observed_generation = Some(SNAPSHOT_GENERATION); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!(report.restore_permitted(), "equal generation must not conflict: {report:?}"); + } + + #[tokio::test] + async fn dry_run_checks_the_operator_master_key() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let verifier = manifest + .local_kdf + .as_ref() + .and_then(|kdf| kdf.master_key_verifier.as_deref()) + .expect("encrypted bundle must carry a verifier"); + assert!(verifier.starts_with(MASTER_KEY_VERIFIER_ARGON2ID_PREFIX), "got {verifier:?}"); + + // No passphrase: encrypted records make the master key a required + // external dependency. + let mut request = restore_request(&bundle, &target); + request.master_key = None; + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::ExternalDependencyUnavailable), + "expected a missing-master-key blocker: {report:?}" + ); + + // Wrong passphrase: reported as an external dependency mismatch. + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert_eq!(report.external_mismatches.len(), 1, "{report:?}"); + assert_eq!(report.external_mismatches[0].expected, verifier); + assert!(!report.restore_permitted()); + } + + #[tokio::test] + async fn dry_run_enumerates_existing_target_state() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + + // A populated target: a key file, the salt, and a stray file. + let (_occupant, target_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["occupied"]).await; + fs::write(target_dir.path().join("operator-notes.txt"), b"leftover") + .await + .expect("stray file"); + + let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, target_dir.path())) + .await + .expect("dry run"); + let conflicting_ids: Vec<&str> = report.conflicts.iter().map(|conflict| conflict.key_id.as_str()).collect(); + assert!(conflicting_ids.contains(&"occupied"), "{report:?}"); + assert!(conflicting_ids.contains(&LOCAL_KMS_MASTER_KEY_SALT_FILE), "{report:?}"); + assert!(conflicting_ids.contains(&"operator-notes.txt"), "{report:?}"); + assert!(!report.restore_permitted()); + } + + #[tokio::test] + async fn restore_into_empty_target_round_trips_byte_for_byte() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let bundle_before = snapshot_tree(&bundle); + let report = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("restore should succeed"); + + assert_eq!( + report, + LocalRestoreReport { + backup_id: BACKUP_ID.to_string(), + snapshot_generation: SNAPSHOT_GENERATION, + restored_key_ids: vec!["alpha".to_string(), "beta".to_string()], + salt_restored: true, + resumed: false, + } + ); + assert_eq!(snapshot_tree(&bundle), bundle_before, "the bundle source must stay byte-identical"); + assert_eq!( + top_level_names(&target).await, + vec![ + LOCAL_KMS_MASTER_KEY_SALT_FILE.to_string(), + "alpha.key".to_string(), + "beta.key".to_string() + ], + "cutover must leave no marker, staging, or temp files behind" + ); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + + // The restored directory boots as a normal backend directory. + let restored = LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))) + .await + .expect("restored directory must pass startup validation"); + restored.describe_key("alpha", None).await.expect("restored key describes"); + } + + #[tokio::test] + async fn default_fail_policy_never_writes() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.conflict_policy = RestoreConflictPolicy::Fail; + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("the default policy must refuse to write"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(!target.exists(), "the fail policy must not create the target"); + } + + #[tokio::test] + async fn non_empty_targets_are_rejected() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + + // Existing key material. + let (_occupant, occupied) = source_with_keys(Some(TEST_MASTER_KEY), &["occupied"]).await; + let before = snapshot_tree(occupied.path()); + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, occupied.path())) + .await + .expect_err("occupied target must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert_eq!(snapshot_tree(occupied.path()), before, "the rejected target must stay untouched"); + + // An orphan salt alone makes the target non-empty: a foreign salt + // would poison every key created after the restore. + let orphan = TempDir::new().expect("orphan dir"); + fs::write(orphan.path().join(LOCAL_KMS_MASTER_KEY_SALT_FILE), [7u8; 16]) + .await + .expect("seed orphan salt"); + let before = snapshot_tree(orphan.path()); + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, orphan.path())) + .await + .expect_err("orphan salt must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("salt"), "got {error}"); + assert_eq!(snapshot_tree(orphan.path()), before); + + // A marker of a different restore. + let foreign = TempDir::new().expect("foreign dir"); + let mut digest = manifest.manifest_digest.clone(); + digest.hex = digest.hex.chars().rev().collect(); + let foreign_marker = RestoreCommitMarker { + format_version: RESTORE_MARKER_FORMAT_VERSION, + backup_id: "backup-foreign".to_string(), + manifest_digest: digest, + files: vec!["foreign.key".to_string()], + }; + fs::write( + foreign.path().join(LOCAL_RESTORE_COMMIT_MARKER_FILE), + serde_json::to_vec_pretty(&foreign_marker).expect("marker json"), + ) + .await + .expect("seed foreign marker"); + let before = snapshot_tree(foreign.path()); + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, foreign.path())) + .await + .expect_err("foreign marker must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(error.to_string().contains("different bundle"), "got {error}"); + assert_eq!(snapshot_tree(foreign.path()), before); + } + + #[tokio::test] + async fn wrong_passphrase_fails_via_verifier_before_any_target_write() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("wrong master key must be rejected by the verifier"); + assert!(matches!(error, KmsError::ValidationError { .. }), "got {error:?}"); + assert!(!target.exists(), "the verifier must reject before any target write"); + } + + #[tokio::test] + async fn wrong_passphrase_fails_via_probe_when_bundle_has_no_verifier() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + // Simulate a bundle produced before the verifier landed. + reseal_manifest(&bundle, |manifest| { + manifest + .local_kdf + .as_mut() + .expect("local bundle has a KDF descriptor") + .master_key_verifier = None; + }) + .await; + + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("wrong master key must be caught by the decryption probe"); + assert!(matches!(error, KmsError::MaterialAuthenticationFailed { .. }), "got {error:?}"); + assert!(!target.exists(), "the in-memory probe must reject before any target write"); + + // The right passphrase still restores the verifier-less bundle. + restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("verifier-less bundle must restore with the right master key"); + assert_restored_matches_source(&client, &target, &["alpha"]).await; + } + + #[tokio::test] + async fn bundle_defects_fail_closed_on_restore() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["victim"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + let request = restore_request(&bundle, &target); + + let artifact_file = bundle.join(&manifest.artifacts[0].path); + let original_artifact = std::fs::read(&artifact_file).expect("artifact bytes"); + + // Tampered artifact byte. + let mut tampered = original_artifact.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0x01; + std::fs::write(&artifact_file, &tampered).expect("write tampered"); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("tampered artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted), + "{report:?}" + ); + + // Truncated artifact. + std::fs::write(&artifact_file, &original_artifact[..original_artifact.len() - 4]).expect("truncate"); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("truncated artifact must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Truncated { .. })), "got {error:?}"); + std::fs::write(&artifact_file, &original_artifact).expect("restore artifact"); + + // Wrong KEK identity and wrong KEK material. + let wrong_id = BackupKek::new("other-kek", 1, [0x42; 32]).expect("kek"); + let error = restore_local_backup(&wrong_id, &request) + .await + .expect_err("wrong KEK id must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), "got {error:?}"); + let wrong_material = BackupKek::new("backup-kek-test", 1, [0x24; 32]).expect("kek"); + let error = restore_local_backup(&wrong_material, &request) + .await + .expect_err("wrong KEK material must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + + // Missing manifest: the bundle never sealed. + std::fs::remove_file(bundle.join(LOCAL_BUNDLE_MANIFEST_FILE)).expect("remove manifest"); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("unsealed bundle must be rejected"); + assert!(matches!(error, KmsError::Backup(BackupError::IncompleteBundle { .. })), "got {error:?}"); + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::IncompleteBundle), + "{report:?}" + ); + + assert!(!target.exists(), "no rejected bundle may touch the target"); + } + + #[tokio::test] + async fn kdf_parameter_drift_is_rejected() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + reseal_manifest(&bundle, |manifest| { + let descriptor = manifest.local_kdf.as_mut().expect("local bundle has a KDF descriptor"); + match &mut descriptor.derivation { + LocalKeyDerivation::Argon2id { memory_kib, .. } => *memory_kib *= 2, + other => panic!("expected an Argon2id descriptor, got {other:?}"), + } + }) + .await; + + let request = restore_request(&bundle, &target); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("a KDF parameter drift must be rejected"); + assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "got {error:?}"); + assert!(!target.exists()); + + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::UnsupportedBackend), + "{report:?}" + ); + } + + #[tokio::test] + async fn unknown_verifier_scheme_fails_closed() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + reseal_manifest(&bundle, |manifest| { + manifest + .local_kdf + .as_mut() + .expect("local bundle has a KDF descriptor") + .master_key_verifier = Some("post-quantum-v2:0123abcd".to_string()); + }) + .await; + + let request = restore_request(&bundle, &target); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("an unknown verifier scheme must fail closed"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + assert!(!target.exists()); + + let report = dry_run_local_restore(&test_kek(), &request).await.expect("dry run"); + assert!( + report + .blockers + .iter() + .any(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted), + "{report:?}" + ); + } + + #[tokio::test] + async fn generation_regression_is_rejected_and_equal_is_allowed() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let mut request = restore_request(&bundle, &target); + request.observed_generation = Some(SNAPSHOT_GENERATION + 1); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("a strictly lower bundle generation must be rejected"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + assert!(!target.exists()); + + request.observed_generation = Some(SNAPSHOT_GENERATION); + restore_local_backup(&test_kek(), &request) + .await + .expect("an equal generation must stay restorable for repeated drills"); + assert_restored_matches_source(&client, &target, &["alpha"]).await; + } + + #[tokio::test] + async fn dev_mode_bundle_round_trips_without_passphrase() { + let (client, _source_dir) = source_with_keys(None, &["dev-key"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.master_key_verifier, None, "dev-mode bundles have no master key to verify"); + + let mut request = restore_request(&bundle, &target); + request.master_key = None; + let report = restore_local_backup(&test_kek(), &request) + .await + .expect("dev-mode restore must succeed without a passphrase"); + assert!(!report.salt_restored); + assert_eq!(report.restored_key_ids, vec!["dev-key".to_string()]); + + let source_bytes = fs::read(client.key_directory().join("dev-key.key")).await.expect("source"); + let restored_bytes = fs::read(target.join("dev-key.key")).await.expect("restored"); + assert_eq!(source_bytes, restored_bytes); + + let restored = LocalKmsClient::new(local_config(&target, None)) + .await + .expect("restored dev directory must boot"); + restored.describe_key("dev-key", None).await.expect("restored key describes"); + } + + #[tokio::test] + async fn legacy_sha256_bundle_round_trips() { + // A pre-beta.9 directory: legacy SHA-256 encrypted record, no salt, + // no protection marker (same fixture as the backend's beta.5 test; + // the material decrypts to 32 x 0x42 under this master key). + let source_dir = TempDir::new().expect("legacy dir"); + let master_key = "beta5-test-master-key"; + let record = serde_json::json!({ + "key_id": "beta5-key", + "version": 1u32, + "algorithm": "AES_256", + "usage": "EncryptDecrypt", + "status": "Active", + "description": serde_json::Value::Null, + "metadata": std::collections::HashMap::::new(), + "created_at": "2024-01-01T00:00:00+00:00", + "rotated_at": serde_json::Value::Null, + "created_by": "beta5-fixture", + "encrypted_key_material": "xjwGa4Lj4qzKg6XQl8s2btyFkPHPChMAkjqs268TFGyvFUv8WjDD5HQCUDLViZmt", + "nonce": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + }); + fs::write( + source_dir.path().join("beta5-key.key"), + serde_json::to_vec_pretty(&record).expect("record json"), + ) + .await + .expect("write legacy record"); + + // Export through the read-only constructor so no Argon2 salt gets + // created: the bundle must record the legacy derivation. + let client = LocalKmsClient::new_for_key_export(local_config(source_dir.path(), Some(master_key))) + .await + .expect("open legacy dir read-only"); + let work = TempDir::new().expect("work dir"); + let (bundle, manifest) = export_bundle(&client, work.path()).await; + let kdf = manifest.local_kdf.as_ref().expect("local kdf descriptor"); + assert_eq!(kdf.derivation, LocalKeyDerivation::LegacySha256); + let verifier = kdf.master_key_verifier.as_deref().expect("legacy verifier"); + assert!(verifier.starts_with(MASTER_KEY_VERIFIER_LEGACY_PREFIX), "got {verifier:?}"); + + let target = work.path().join("target"); + let mut request = restore_request(&bundle, &target); + request.master_key = Some("wrong-master-key".to_string()); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("the legacy verifier must reject a wrong master key"); + assert!(matches!(error, KmsError::ValidationError { .. }), "got {error:?}"); + assert!(!target.exists()); + + request.master_key = Some(master_key.to_string()); + let report = restore_local_backup(&test_kek(), &request).await.expect("legacy restore"); + assert!(!report.salt_restored, "a legacy bundle carries no salt"); + + // The restored directory boots (creating a fresh Argon2 salt is + // legitimate for legacy records) and the material still decrypts via + // the legacy fallback. + let restored = LocalKmsClient::new(local_config(&target, Some(master_key))) + .await + .expect("restored legacy directory must boot"); + let material = restored + .decrypt_key_material_for_export("beta5-key") + .await + .expect("legacy material must decrypt after restore"); + assert_eq!(material.as_ref(), &[0x42; 32]); + } + + #[tokio::test] + async fn crash_during_staging_keeps_top_level_untouched_and_rerun_converges() { + use durable_file::{CommitStep, failpoint}; + + for step in [ + CommitStep::TempWritten, + CommitStep::FileSynced, + CommitStep::Published, + CommitStep::DirSynced, + ] { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + let request = restore_request(&bundle, &target); + + failpoint::arm(&target, step); + let error = restore_local_backup(&test_kek(), &request) + .await + .expect_err("armed staging commit must simulate a crash"); + failpoint::disarm(&target); + assert!(error.to_string().contains("injected crash"), "step {step:?}: {error}"); + + // Top level: only the staging directory, never a marker or a key. + assert_eq!( + top_level_names(&target).await, + vec![RESTORE_STAGING_DIR.to_string()], + "step {step:?}: a staging crash must leave the top level untouched" + ); + + restore_local_backup(&test_kek(), &request) + .await + .unwrap_or_else(|error| panic!("step {step:?}: re-run must converge: {error}")); + assert_restored_matches_source(&client, &target, &["alpha"]).await; + } + } + + /// Drive the phases manually up to the marker commit, crash the commit at + /// every step, and prove both re-entry outcomes: an unpublished marker + /// temp is removable leftover state, a published marker rolls forward. + #[tokio::test] + async fn crash_around_marker_publish_recovers() { + use durable_file::{CommitStep, failpoint}; + + for step in [ + CommitStep::TempWritten, + CommitStep::FileSynced, + CommitStep::Published, + CommitStep::DirSynced, + ] { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + let request = restore_request(&bundle, &target); + + // Stage everything through the real phase functions. + let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); + let staging = target.join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("create staging"); + let salt = decoded.salt.as_ref().expect("bundle salt"); + stage_file(&staging, LOCAL_KMS_MASTER_KEY_SALT_FILE, salt, Some(0o600)) + .await + .expect("stage salt"); + for record in &decoded.records { + stage_file(&staging, &record.file_name, &record.plaintext, Some(0o600)) + .await + .expect("stage record"); + } + let marker = RestoreCommitMarker::for_bundle(&decoded); + + failpoint::arm(&target, step); + let error = publish_marker(&target, &marker, Some(0o600)) + .await + .expect_err("armed marker commit must simulate a crash"); + failpoint::disarm(&target); + assert!(error.to_string().contains("injected crash"), "step {step:?}: {error}"); + + let marker_path = target.join(LOCAL_RESTORE_COMMIT_MARKER_FILE); + let published = matches!(step, CommitStep::Published | CommitStep::DirSynced); + assert_eq!(marker_path.exists(), published, "step {step:?}"); + if !published { + // The unpublished marker temp is exactly the shape startup + // recovery and a restore re-run both classify as removable. + let temp = top_level_names(&target) + .await + .into_iter() + .find(|name| is_orphan_commit_temp_name(name)); + assert!(temp.is_some(), "step {step:?}: expected a leftover marker temp"); + } else { + // A published marker makes backend startup fail closed. + let error = match LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))).await { + Ok(_) => panic!("step {step:?}: startup must fail closed on a published marker"), + Err(error) => error, + }; + assert!(matches!(error, KmsError::ConfigurationError { .. }), "step {step:?}: {error:?}"); + assert!(error.to_string().contains("unfinished restore"), "step {step:?}: {error}"); + } + + let report = restore_local_backup(&test_kek(), &request) + .await + .unwrap_or_else(|error| panic!("step {step:?}: re-run must converge: {error}")); + assert_eq!(report.resumed, published, "step {step:?}"); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + } + } + + #[tokio::test] + async fn crash_mid_cutover_fails_startup_then_rolls_forward_or_aborts() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + + // Reconstruct the exact state of a cutover killed after k of n links. + let build_partial_cutover = |target: PathBuf| { + let bundle = bundle.clone(); + async move { + let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); + let staging = target.join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("create staging"); + stage_file( + &staging, + LOCAL_KMS_MASTER_KEY_SALT_FILE, + decoded.salt.as_ref().expect("bundle salt"), + Some(0o600), + ) + .await + .expect("stage salt"); + for record in &decoded.records { + stage_file(&staging, &record.file_name, &record.plaintext, Some(0o600)) + .await + .expect("stage record"); + } + let marker = RestoreCommitMarker::for_bundle(&decoded); + publish_marker(&target, &marker, Some(0o600)).await.expect("publish marker"); + // First link only (the salt), then "power loss". + durable_file::link_durably( + staging.join(LOCAL_KMS_MASTER_KEY_SALT_FILE), + target.join(LOCAL_KMS_MASTER_KEY_SALT_FILE), + ) + .await + .expect("link salt"); + } + }; + + // Outcome 1: roll forward with the same bundle. + let target = work.path().join("target-forward"); + build_partial_cutover(target.clone()).await; + let error = match LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))).await { + Ok(_) => panic!("startup must fail closed mid-cutover"), + Err(error) => error, + }; + assert!(error.to_string().contains("unfinished restore"), "got {error}"); + assert!(matches!( + LocalKmsClient::new_for_key_export(local_config(&target, Some(TEST_MASTER_KEY))).await, + Err(KmsError::ConfigurationError { .. }) + )); + let report = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("roll-forward must converge"); + assert!(report.resumed); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + + // Outcome 2: explicit abort returns the complete old (empty) state. + let target = work.path().join("target-abort"); + build_partial_cutover(target.clone()).await; + abort_local_restore(&target).await.expect("abort must succeed"); + assert_eq!( + top_level_names(&target).await, + Vec::::new(), + "abort must take back every published file, the marker, and staging" + ); + // The aborted target is a healthy empty directory again: both a fresh + // backend start and a fresh restore work. + restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("a fresh restore after abort must succeed"); + assert_restored_matches_source(&client, &target, &["alpha", "beta"]).await; + } + + #[tokio::test] + async fn roll_forward_requires_identical_file_set() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha", "beta"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + + let decoded = decode_bundle(&bundle, &test_kek()).await.expect("decode bundle"); + fs::create_dir_all(target.join(RESTORE_STAGING_DIR)) + .await + .expect("create staging"); + let mut marker = RestoreCommitMarker::for_bundle(&decoded); + marker.files.pop(); + publish_marker(&target, &marker, Some(0o600)).await.expect("publish marker"); + + let error = restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect_err("a marker with a diverging file set must fail closed"); + assert!(matches!(error, KmsError::Backup(BackupError::Corrupted { .. })), "got {error:?}"); + assert!(error.to_string().contains("different file set"), "got {error}"); + } + + #[tokio::test] + async fn completed_restore_tolerates_inert_staging_leftover() { + let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + let work = TempDir::new().expect("work dir"); + let (bundle, _manifest) = export_bundle(&client, work.path()).await; + let target = work.path().join("target"); + restore_local_backup(&test_kek(), &restore_request(&bundle, &target)) + .await + .expect("restore"); + + // A crash between marker removal and staging cleanup leaves an inert + // staging directory; startup must ignore it entirely. + let staging = target.join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("recreate staging"); + fs::write(staging.join("leftover"), b"inert").await.expect("seed leftover"); + + let restored = LocalKmsClient::new(local_config(&target, Some(TEST_MASTER_KEY))) + .await + .expect("startup must ignore a leftover staging directory"); + restored.describe_key("alpha", None).await.expect("key describes"); + assert!(staging.exists(), "startup must not touch the staging leftover"); + } + + #[tokio::test] + async fn startup_removes_unpublished_marker_temp() { + let (client, source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await; + drop(client); + let temp_name = format!("{LOCAL_RESTORE_COMMIT_MARKER_FILE}.tmp-{}", uuid::Uuid::new_v4()); + let temp_path = source_dir.path().join(&temp_name); + fs::write(&temp_path, b"unpublished marker").await.expect("seed marker temp"); + + let client = LocalKmsClient::new(local_config(source_dir.path(), Some(TEST_MASTER_KEY))) + .await + .expect("an unpublished marker temp must not block startup"); + assert!(!temp_path.exists(), "startup recovery must remove the unpublished marker temp"); + client.describe_key("alpha", None).await.expect("key describes"); + } + + #[tokio::test] + async fn abort_without_restore_in_progress_errors_and_staging_only_is_cleaned() { + let empty = TempDir::new().expect("empty dir"); + let error = abort_local_restore(empty.path()) + .await + .expect_err("nothing to abort must be an error"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}"); + + let staging_only = TempDir::new().expect("staging dir"); + let staging = staging_only.path().join(RESTORE_STAGING_DIR); + fs::create_dir_all(&staging).await.expect("create staging"); + fs::write(staging.join("partial.key"), b"partial") + .await + .expect("seed partial"); + abort_local_restore(staging_only.path()) + .await + .expect("a pre-marker abort only removes staging"); + assert_eq!(top_level_names(staging_only.path()).await, Vec::::new()); + } +} diff --git a/crates/kms/src/backup/mod.rs b/crates/kms/src/backup/mod.rs index d3f9134e9..53b0fa741 100644 --- a/crates/kms/src/backup/mod.rs +++ b/crates/kms/src/backup/mod.rs @@ -16,9 +16,10 @@ //! //! The contract side defines the versioned backup manifest, the per-backend //! responsibility matrix, typed failure modes, and the restore dry-run -//! report. [`local_export`] implements the producer side for the Local -//! backend as a crate-internal API; restore orchestration and the admin API -//! build on these pieces in follow-up changes. +//! report. [`local_export`] implements the producer side and +//! [`local_restore`] the consumer side for the Local backend as +//! crate-internal APIs; the admin API builds on these pieces in follow-up +//! changes. //! //! # Bundle model //! @@ -51,6 +52,7 @@ mod capability; mod dry_run; mod error; pub mod local_export; +pub mod local_restore; mod manifest; pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility}; @@ -62,6 +64,10 @@ pub use local_export::{ BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup, read_local_bundle_manifest, }; +pub use local_restore::{ + LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore, + restore_local_backup, +}; pub use manifest::{ AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest, DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, From 9080ea8ea0c1751fc77f0d9c313cc7e87abe98fc Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 10:14:32 +0800 Subject: [PATCH 11/12] test(object-lock): cover unretained version cleanup (#5504) Co-authored-by: cxymds --- .../src/object_lock/object_lock_test.rs | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/e2e_test/src/object_lock/object_lock_test.rs b/crates/e2e_test/src/object_lock/object_lock_test.rs index 7cb04cf4d..0748ad499 100644 --- a/crates/e2e_test/src/object_lock/object_lock_test.rs +++ b/crates/e2e_test/src/object_lock/object_lock_test.rs @@ -26,6 +26,7 @@ use super::common::*; use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::{ByteStream, DateTimeFormat}; use aws_sdk_s3::types::{ CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus, @@ -2120,6 +2121,127 @@ async fn test_multipart_default_retention_fixed_at_create() { // Versioning Auto-Enable Tests // ============================================================================ +#[tokio::test] +#[serial] +async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() { + init_logging(); + info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)"); + + let mut env = ObjectLockTestEnvironment::new() + .await + .expect("failed to create Object Lock test environment"); + env.start_rustfs().await.expect("failed to start RustFS"); + + let bucket = "test-object-lock-delete-cleanup"; + let key = "unretained-object"; + + env.create_object_lock_bucket(bucket) + .await + .expect("failed to create Object Lock bucket"); + let client = env.s3_client(); + + let put_response = client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"unretained data")) + .send() + .await + .expect("failed to upload unretained object"); + let object_version_id = put_response + .version_id() + .expect("Object Lock buckets must create versioned objects") + .to_string(); + + let delete_response = client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("failed to create delete marker"); + assert_eq!(delete_response.delete_marker(), Some(true)); + let delete_marker_version_id = delete_response + .version_id() + .expect("Deleting without a version ID must create a delete marker") + .to_string(); + + let get_error = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect_err("GET must not return an object hidden by a delete marker"); + assert_eq!(get_error.raw_response().map(|response| response.status().as_u16()), Some(404)); + assert_eq!(get_error.as_service_error().and_then(|error| error.code()), Some("NoSuchKey")); + + let listed_objects = client + .list_objects_v2() + .bucket(bucket) + .send() + .await + .expect("failed to list current objects"); + assert!( + listed_objects.contents().iter().all(|object| object.key() != Some(key)), + "ListObjectsV2 must hide objects whose latest version is a delete marker" + ); + + let listed_versions = client + .list_object_versions() + .bucket(bucket) + .send() + .await + .expect("failed to list object versions"); + assert!( + listed_versions + .versions() + .iter() + .any(|version| version.key() == Some(key) && version.version_id() == Some(object_version_id.as_str())), + "The data version must remain until it is explicitly deleted" + ); + assert!( + listed_versions + .delete_markers() + .iter() + .any(|marker| marker.key() == Some(key) && marker.version_id() == Some(delete_marker_version_id.as_str())), + "ListObjectVersions must expose the delete marker" + ); + + client + .delete_object() + .bucket(bucket) + .key(key) + .version_id(object_version_id) + .send() + .await + .expect("failed to delete the data version"); + client + .delete_object() + .bucket(bucket) + .key(key) + .version_id(delete_marker_version_id) + .send() + .await + .expect("failed to delete the delete marker"); + + let remaining_versions = client + .list_object_versions() + .bucket(bucket) + .send() + .await + .expect("failed to list versions after cleanup"); + assert!(remaining_versions.versions().is_empty()); + assert!(remaining_versions.delete_markers().is_empty()); + + client + .delete_bucket() + .bucket(bucket) + .send() + .await + .expect("Deleting every version must remove xl.meta so the bucket can be deleted normally"); +} + #[tokio::test] #[serial] async fn test_versioning_auto_enabled_with_object_lock() { From 4b6b6f14bde8bca9823e026c9a37551f159a099e Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 1 Aug 2026 10:43:52 +0800 Subject: [PATCH 12/12] feat(hotpath): use mimalloc in inner counting allocator (#5523) * feat(hotpath): use mimalloc in counting allocator * fix(hotpath): adapt mimalloc for allocation counting --------- Co-authored-by: overtrue --- rustfs/src/main.rs | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 74d652be3..75f9c81c4 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -12,9 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +use std::alloc::{GlobalAlloc, Layout}; + +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +#[derive(Default)] +struct DefaultMiMalloc; + +#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] +// SAFETY: allocation and deallocation are forwarded unchanged to MiMalloc, so +// MiMalloc's GlobalAlloc guarantees apply to every returned pointer and layout. +#[allow(unsafe_code)] +unsafe impl GlobalAlloc for DefaultMiMalloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: the caller upholds GlobalAlloc's contract for layout. + unsafe { mimalloc::MiMalloc.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: ptr and layout came from this allocator and are forwarded unchanged. + unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) } + } +} + #[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))] #[global_allocator] -static GLOBAL: hotpath::CountingAllocator = hotpath::CountingAllocator::new(); +static GLOBAL: hotpath::CountingAllocator = hotpath::CountingAllocator::new(); #[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))] #[global_allocator] @@ -25,3 +48,15 @@ fn main() { rustfs::startup_entrypoint::run_process(); } + +#[cfg(all(test, feature = "hotpath", feature = "hotpath-alloc"))] +mod tests { + #[test] + #[allow(unsafe_code)] + fn hotpath_allocator_uses_mimalloc() { + let allocation = Box::new([0_u8; 64]); + + // SAFETY: the live Box pointer is valid to inspect for heap ownership. + assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) }); + } +}