# Copyright 2024 RustFS Team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= # RustFS KMS backend — Prometheus alerting rules # ============================================================================= # # Metric source: the KMS operation-policy choke point in # crates/kms/src/policy.rs, except KmsKeyRotationOverdue, which reads the # label-less key-lifecycle gauge published by the deletion worker's sweep # (crates/kms/src/deletion_worker.rs). All label values are bounded static # strings (operation, op_class, outcome, error_class, backend, scope); key # identifiers, key material, and tokens never appear in labels. # # Response procedures: docs/operations/kms-observability-runbook.md # # IMPORTANT — threshold status: every numeric threshold below is a # conservative default chosen without a production baseline. Calibrate against # a staging baseline before relying on these alerts for paging, and prefer # loosening over tightening until the baseline exists. Formal SLO targets are # deliberately not encoded here (see rustfs/backlog#1584). # # NOTE: prometheus.yml loads /etc/prometheus/rules/*.yml — keep the .yml # extension or the file is silently ignored by the docker-compose stack. # # Validate: promtool check rules rustfs-kms-alerts.yml # ============================================================================= groups: # ========================================================================== # Critical alerts — immediate action required # ========================================================================== - name: rustfs-kms-critical interval: 30s rules: # ------------------------------------------------------------------ # 1. KmsBackendFatalErrors # Any attempt failure classified as fatal (non-retryable): auth # or permission errors, malformed requests, missing keys. The # policy never retries these, so even a low rate means real # operations are failing right now. # ------------------------------------------------------------------ - alert: KmsBackendFatalErrors expr: | sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m])) > 0 for: 5m labels: severity: critical component: kms annotations: summary: "KMS backend fatal errors on operation {{ $labels.operation }}" description: >- Attempt failures classified as fatal are occurring at {{ $value | printf "%.3f" }}/s on operation {{ $labels.operation }}. Fatal failures are not retried: each one is a KMS backend call that failed permanently (authentication, permissions, malformed request, or a missing key/version). runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendfatalerrors" # ------------------------------------------------------------------ # 2. KmsBackendHighErrorRate # Sustained share of operations terminating without success # (fatal, budget/deadline exhaustion, admission backpressure, # or an open circuit). The cancelled outcome is excluded because # shutdowns legitimately produce it. # The traffic guard keeps a single failure on a near-idle # cluster from firing the alert. # Threshold: 5% for 10m — conservative default, calibrate # against a staging baseline. # ------------------------------------------------------------------ - alert: KmsBackendHighErrorRate expr: | ( sum(rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total[5m])), 1e-9) ) > 0.05 and sum(rate(rustfs_kms_backend_operations_total[5m])) > 0.02 for: 10m labels: severity: critical component: kms annotations: summary: "KMS backend non-success ratio above 5% for 10m" description: >- {{ $value | humanizePercentage }} of KMS backend operations are terminating in fatal, budget_exhausted, deadline_exceeded, backpressure_timeout, backpressure_rejected, or circuit_open. Object encryption and decryption paths depending on the KMS are degraded or failing. runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate" # ========================================================================== # Warning alerts — investigation needed # ========================================================================== - name: rustfs-kms-warning interval: 30s rules: # ------------------------------------------------------------------ # 3. KmsBackendP99LatencyHigh # p99 wall-clock duration of whole operations (attempts plus # backoff) is sustained above 2 seconds. Because the histogram # includes retries, a high p99 usually means the retry policy # is absorbing backend failures, not that every call is slow. # Threshold: 2s for 10m — conservative default, calibrate # against a staging baseline. # ------------------------------------------------------------------ - alert: KmsBackendP99LatencyHigh expr: | histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])) ) > 2 for: 10m labels: severity: warning component: kms annotations: summary: "KMS backend operation p99 latency above 2s for 10m" description: >- The 99th-percentile KMS backend operation duration is {{ $value | humanizeDuration }}, including retries and backoff. Encryption and decryption latency is leaking into S3 request latency. runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendp99latencyhigh" # ------------------------------------------------------------------ # 4. KmsBackendAttemptFailureSpike # Aggregate attempt-failure rate (all error classes) sustained # above an absolute floor. An absolute threshold is used instead # of an offset-1d baseline ratio because fresh deployments have # no baseline and an empty offset vector would keep a ratio # alert from ever firing; switch to a baseline-relative form # (see rustfs-get-optimization-alerts.yaml for the pattern) # once a stable staging baseline exists. # Threshold: 0.5/s for 10m — conservative default, calibrate # against a staging baseline. # ------------------------------------------------------------------ - alert: KmsBackendAttemptFailureSpike expr: | sum(rate(rustfs_kms_backend_attempt_failures_total[5m])) > 0.5 for: 10m labels: severity: warning component: kms annotations: summary: "KMS backend attempt failures above 0.5/s for 10m" description: >- KMS backend attempts are failing at {{ $value | printf "%.2f" }}/s across all error classes. The retry policy may still be masking these from callers — check the error-class breakdown before it stops absorbing them. runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendattemptfailurespike" # ------------------------------------------------------------------ # 5. KmsBackendRetryBudgetExhausted # Operations are running out of retry budget (budget_exhausted) # or operation deadline (deadline_exceeded). These surface to # callers as failed KMS operations even though every individual # failure was retryable — the backend is unhealthy for longer # than the policy can bridge. # Threshold: 0.05/s for 10m — conservative default, calibrate # against a staging baseline. # ------------------------------------------------------------------ - alert: KmsBackendRetryBudgetExhausted expr: | sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m])) > 0.05 for: 10m labels: severity: warning component: kms annotations: summary: "KMS backend operations exhausting retry budget ({{ $labels.outcome }})" description: >- KMS backend operations are terminating as {{ $labels.outcome }} at {{ $value | printf "%.3f" }}/s. Retryable failures are outlasting the retry budget, so callers are seeing hard failures. runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted" # ------------------------------------------------------------------ # 6. KmsBackendCircuitOpen # Direct circuit-state signal, independent of operation traffic. # A transient open can recover on its first half-open probe; alert # only when the circuit remains open or half-open for one minute. # ------------------------------------------------------------------ - alert: KmsBackendCircuitOpen expr: | rustfs_kms_backend_circuit_open > 0 for: 1m labels: severity: warning component: kms annotations: summary: "KMS backend circuit open ({{ $labels.backend }}/{{ $labels.scope }})" description: >- The KMS backend circuit for {{ $labels.backend }} scope {{ $labels.scope }} has remained open or half-open for one minute. Operations in this scope can terminate as circuit_open until the half-open probe succeeds or returns a non-retryable failure. runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen" # ------------------------------------------------------------------ # 7. KmsKeyRotationOverdue # The least recently rotated usable key has gone more than 400 # days without a rotation (measured from creation for keys with # no recorded rotation). Direct gauge state published by the # deletion worker's sweep, so no traffic guard applies; the # one-hour hold only bridges scrape gaps. The worker runs only # on backends with the schedule_deletion capability, so on the # Static backend the series never exists and this alert cannot # fire — that backend cannot rotate either; see the rotation # driver matrix in docs/operations/kms-backend-security.md. # Threshold: 400 days — conservative default sitting above a # one-year rotation policy. Align it with the rotation period # your compliance policy requires, and with # RUSTFS_KMS_ROTATION_MAX_AGE_SECS so the per-key rotation_due # verdict and this aggregate alert agree. # ------------------------------------------------------------------ - alert: KmsKeyRotationOverdue expr: | rustfs_kms_oldest_key_rotation_age_seconds > (400 * 86400) for: 1h labels: severity: warning component: kms annotations: summary: "Oldest KMS key unrotated for more than 400 days" description: >- The least recently rotated usable KMS key was last rotated {{ $value | humanizeDuration }} ago (measured from creation for keys with no recorded rotation). List keys through the admin API and read rotation_due / rotation_due_reason for the per-key verdict; an "unsupported" reason means the backend cannot rotate at all. runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmskeyrotationoverdue"