# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# =============================================================================
# RustFS KMS backend — Prometheus alerting rules
# =============================================================================
#
# Metric source: the KMS operation-policy choke point in
# crates/kms/src/policy.rs, except KmsKeyRotationOverdue, which reads the
# label-less key-lifecycle gauge published by the deletion worker's sweep
# (crates/kms/src/deletion_worker.rs). All label values are bounded static
# strings (operation, op_class, outcome, error_class, backend, scope); key
# identifiers, key material, and tokens never appear in labels.
#
# Response procedures: docs/operations/kms-observability-runbook.md
#
# IMPORTANT — threshold status: every numeric threshold below is a
# conservative default chosen without a production baseline. Calibrate against
# a staging baseline before relying on these alerts for paging, and prefer
# loosening over tightening until the baseline exists. Formal SLO targets are
# deliberately not encoded here (see rustfs/backlog#1584).
#
# NOTE: prometheus.yml loads /etc/prometheus/rules/*.yml — keep the .yml
# extension or the file is silently ignored by the docker-compose stack.
#
# Validate: promtool check rules rustfs-kms-alerts.yml
# =============================================================================

groups:
  # ==========================================================================
  # Critical alerts — immediate action required
  # ==========================================================================
  - name: rustfs-kms-critical
    interval: 30s
    rules:
      # ------------------------------------------------------------------
      # 1. KmsBackendFatalErrors
      #    Any attempt failure classified as fatal (non-retryable): auth
      #    or permission errors, malformed requests, missing keys. The
      #    policy never retries these, so even a low rate means real
      #    operations are failing right now.
      # ------------------------------------------------------------------
      - alert: KmsBackendFatalErrors
        expr: |
          sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m])) > 0
        for: 5m
        labels:
          severity: critical
          component: kms
        annotations:
          summary: "KMS backend fatal errors on operation {{ $labels.operation }}"
          description: >-
            Attempt failures classified as fatal are occurring at
            {{ $value | printf "%.3f" }}/s on operation
            {{ $labels.operation }}. Fatal failures are not retried:
            each one is a KMS backend call that failed permanently
            (authentication, permissions, malformed request, or a
            missing key/version).
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendfatalerrors"

      # ------------------------------------------------------------------
      # 2. KmsBackendHighErrorRate
      #    Sustained share of operations terminating without success
      #    (fatal, budget/deadline exhaustion, admission backpressure,
      #    or an open circuit). The cancelled outcome is excluded because
      #    shutdowns legitimately produce it.
      #    The traffic guard keeps a single failure on a near-idle
      #    cluster from firing the alert.
      #    Threshold: 5% for 10m — conservative default, calibrate
      #    against a staging baseline.
      # ------------------------------------------------------------------
      - alert: KmsBackendHighErrorRate
        expr: |
          (
            sum(rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))
            /
            clamp_min(sum(rate(rustfs_kms_backend_operations_total[5m])), 1e-9)
          ) > 0.05
          and
          sum(rate(rustfs_kms_backend_operations_total[5m])) > 0.02
        for: 10m
        labels:
          severity: critical
          component: kms
        annotations:
          summary: "KMS backend non-success ratio above 5% for 10m"
          description: >-
            {{ $value | humanizePercentage }} of KMS backend operations
            are terminating in fatal, budget_exhausted,
            deadline_exceeded, backpressure_timeout,
            backpressure_rejected, or circuit_open. Object encryption
            and decryption paths depending on the KMS are degraded or
            failing.
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate"

  # ==========================================================================
  # Warning alerts — investigation needed
  # ==========================================================================
  - name: rustfs-kms-warning
    interval: 30s
    rules:
      # ------------------------------------------------------------------
      # 3. KmsBackendP99LatencyHigh
      #    p99 wall-clock duration of whole operations (attempts plus
      #    backoff) is sustained above 2 seconds. Because the histogram
      #    includes retries, a high p99 usually means the retry policy
      #    is absorbing backend failures, not that every call is slow.
      #    Threshold: 2s for 10m — conservative default, calibrate
      #    against a staging baseline.
      # ------------------------------------------------------------------
      - alert: KmsBackendP99LatencyHigh
        expr: |
          histogram_quantile(0.99,
            sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m]))
          ) > 2
        for: 10m
        labels:
          severity: warning
          component: kms
        annotations:
          summary: "KMS backend operation p99 latency above 2s for 10m"
          description: >-
            The 99th-percentile KMS backend operation duration is
            {{ $value | humanizeDuration }}, including retries and
            backoff. Encryption and decryption latency is leaking into
            S3 request latency.
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendp99latencyhigh"

      # ------------------------------------------------------------------
      # 4. KmsBackendAttemptFailureSpike
      #    Aggregate attempt-failure rate (all error classes) sustained
      #    above an absolute floor. An absolute threshold is used instead
      #    of an offset-1d baseline ratio because fresh deployments have
      #    no baseline and an empty offset vector would keep a ratio
      #    alert from ever firing; switch to a baseline-relative form
      #    (see rustfs-get-optimization-alerts.yaml for the pattern)
      #    once a stable staging baseline exists.
      #    Threshold: 0.5/s for 10m — conservative default, calibrate
      #    against a staging baseline.
      # ------------------------------------------------------------------
      - alert: KmsBackendAttemptFailureSpike
        expr: |
          sum(rate(rustfs_kms_backend_attempt_failures_total[5m])) > 0.5
        for: 10m
        labels:
          severity: warning
          component: kms
        annotations:
          summary: "KMS backend attempt failures above 0.5/s for 10m"
          description: >-
            KMS backend attempts are failing at
            {{ $value | printf "%.2f" }}/s across all error classes.
            The retry policy may still be masking these from callers —
            check the error-class breakdown before it stops absorbing
            them.
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendattemptfailurespike"

      # ------------------------------------------------------------------
      # 5. KmsBackendRetryBudgetExhausted
      #    Operations are running out of retry budget (budget_exhausted)
      #    or operation deadline (deadline_exceeded). These surface to
      #    callers as failed KMS operations even though every individual
      #    failure was retryable — the backend is unhealthy for longer
      #    than the policy can bridge.
      #    Threshold: 0.05/s for 10m — conservative default, calibrate
      #    against a staging baseline.
      # ------------------------------------------------------------------
      - alert: KmsBackendRetryBudgetExhausted
        expr: |
          sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m])) > 0.05
        for: 10m
        labels:
          severity: warning
          component: kms
        annotations:
          summary: "KMS backend operations exhausting retry budget ({{ $labels.outcome }})"
          description: >-
            KMS backend operations are terminating as
            {{ $labels.outcome }} at {{ $value | printf "%.3f" }}/s.
            Retryable failures are outlasting the retry budget, so
            callers are seeing hard failures.
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted"

      # ------------------------------------------------------------------
      # 6. KmsBackendCircuitOpen
      #    Direct circuit-state signal, independent of operation traffic.
      #    A transient open can recover on its first half-open probe; alert
      #    only when the circuit remains open or half-open for one minute.
      # ------------------------------------------------------------------
      - alert: KmsBackendCircuitOpen
        expr: |
          rustfs_kms_backend_circuit_open > 0
        for: 1m
        labels:
          severity: warning
          component: kms
        annotations:
          summary: "KMS backend circuit open ({{ $labels.backend }}/{{ $labels.scope }})"
          description: >-
            The KMS backend circuit for {{ $labels.backend }} scope
            {{ $labels.scope }} has remained open or half-open for one
            minute. Operations in this scope can terminate as
            circuit_open until the half-open probe succeeds or returns
            a non-retryable failure.
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"

      # ------------------------------------------------------------------
      # 7. KmsKeyRotationOverdue
      #    The least recently rotated usable key has gone more than 400
      #    days without a rotation (measured from creation for keys with
      #    no recorded rotation). Direct gauge state published by the
      #    deletion worker's sweep, so no traffic guard applies; the
      #    one-hour hold only bridges scrape gaps. The worker runs only
      #    on backends with the schedule_deletion capability, so on the
      #    Static backend the series never exists and this alert cannot
      #    fire — that backend cannot rotate either; see the rotation
      #    driver matrix in docs/operations/kms-backend-security.md.
      #    Threshold: 400 days — conservative default sitting above a
      #    one-year rotation policy. Align it with the rotation period
      #    your compliance policy requires, and with
      #    RUSTFS_KMS_ROTATION_MAX_AGE_SECS so the per-key rotation_due
      #    verdict and this aggregate alert agree.
      # ------------------------------------------------------------------
      - alert: KmsKeyRotationOverdue
        expr: |
          rustfs_kms_oldest_key_rotation_age_seconds > (400 * 86400)
        for: 1h
        labels:
          severity: warning
          component: kms
        annotations:
          summary: "Oldest KMS key unrotated for more than 400 days"
          description: >-
            The least recently rotated usable KMS key was last rotated
            {{ $value | humanizeDuration }} ago (measured from creation
            for keys with no recorded rotation). List keys through the
            admin API and read rotation_due / rotation_due_reason for
            the per-key verdict; an "unsupported" reason means the
            backend cannot rotate at all.
          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmskeyrotationoverdue"
