feat(kms): bound backend concurrency and failures (#5651)

.docker/observability/prometheus-rules/rustfs-kms-alerts.yml

@@ -17,9 +17,9 @@
 # =============================================================================
 #
 # Metric source: the KMS operation-policy choke point in
-# crates/kms/src/policy.rs. All label values are static enum strings
-# (operation, op_class, outcome, error_class); key identifiers, key material,
-# and tokens never appear in labels.
+# crates/kms/src/policy.rs. All label values are bounded static strings
+# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
+# key material, and tokens never appear in labels.
 #
 # Response procedures: docs/operations/kms-observability-runbook.md
 #
@@ -70,8 +70,9 @@ groups:
       # ------------------------------------------------------------------
       # 2. KmsBackendHighErrorRate
       #    Sustained share of operations terminating without success
-      #    (fatal, budget_exhausted, deadline_exceeded). The cancelled
-      #    outcome is excluded because shutdowns legitimately produce it.
+      #    (fatal, budget/deadline exhaustion, admission backpressure,
+      #    or an open circuit). The cancelled outcome is excluded because
+      #    shutdowns legitimately produce it.
       #    The traffic guard keeps a single failure on a near-idle
       #    cluster from firing the alert.
       #    Threshold: 5% for 10m — conservative default, calibrate
@@ -94,9 +95,11 @@ groups:
           summary: "KMS backend non-success ratio above 5% for 10m"
           description: >-
             {{ $value | humanizePercentage }} of KMS backend operations
-            are terminating in fatal, budget_exhausted, or
-            deadline_exceeded. Object encryption and decryption paths
-            depending on the KMS are degraded or failing.
+            are terminating in fatal, budget_exhausted,
+            deadline_exceeded, backpressure_timeout,
+            backpressure_rejected, or circuit_open. Object encryption
+            and decryption paths depending on the KMS are degraded or
+            failing.
           runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate"
 
   # ==========================================================================
@@ -186,3 +189,26 @@ groups:
             Retryable failures are outlasting the retry budget, so
             callers are seeing hard failures.
           runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted"
+
+      # ------------------------------------------------------------------
+      # 6. KmsBackendCircuitOpen
+      #    Direct circuit-state signal, independent of operation traffic.
+      #    A transient open can recover on its first half-open probe; alert
+      #    only when the circuit remains open or half-open for one minute.
+      # ------------------------------------------------------------------
+      - alert: KmsBackendCircuitOpen
+        expr: |
+          rustfs_kms_backend_circuit_open > 0
+        for: 1m
+        labels:
+          severity: warning
+          component: kms
+        annotations:
+          summary: "KMS backend circuit open ({{ $labels.backend }}/{{ $labels.scope }})"
+          description: >-
+            The KMS backend circuit for {{ $labels.backend }} scope
+            {{ $labels.scope }} has remained open or half-open for one
+            minute. Operations in this scope can terminate as
+            circuit_open until the half-open probe succeeds or returns
+            a non-retryable failure.
+          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"

crates/kms/src/backends/aws.rs

@@ -309,11 +309,11 @@ impl AwsKmsBackend {
             loader = loader.region(aws_sdk_kms::config::Region::new(region.clone()));
         }
         let sdk_config = loader.load().await;
-        if sdk_config.region().is_none() {
+        let Some(region) = sdk_config.region() else {
             return Err(KmsError::configuration_error(
                 "AWS KMS backend could not resolve a region; set the backend region or AWS_REGION",
             ));
-        }
+        };
 
         let mut builder = aws_sdk_kms::config::Builder::from(&sdk_config)
             // `crate::policy` owns retries and timeouts; leaving the SDK's own
@@ -324,13 +324,18 @@ impl AwsKmsBackend {
             builder = builder.endpoint_url(endpoint_url);
         }
 
-        Ok(Self::with_client(aws_sdk_kms::Client::from_conf(builder.build()), &config))
+        Ok(Self::with_client(
+            aws_sdk_kms::Client::from_conf(builder.build()),
+            &config,
+            aws_backend_config.endpoint_url.as_deref().unwrap_or_default(),
+            region.as_ref(),
+        ))
     }
 
-    fn with_client(client: aws_sdk_kms::Client, config: &KmsConfig) -> Self {
+    fn with_client(client: aws_sdk_kms::Client, config: &KmsConfig, endpoint: &str, region: &str) -> Self {
         Self {
             client,
-            retry: RetryPolicy::from_config(config),
+            retry: RetryPolicy::for_backend(config, "aws", endpoint, Some(region), "operations"),
             cancel: CancellationToken::new(),
         }
     }
@@ -826,6 +831,7 @@ mod tests {
     use aws_smithy_types::body::SdkBody;
     use base64::Engine as _;
     use base64::engine::general_purpose::STANDARD as BASE64;
+    use std::sync::atomic::{AtomicU64, Ordering};
 
     /// AWS KMS speaks awsJson1_1; every request goes to `/` on the regional
     /// endpoint, so the replayed request side carries no useful assertion.
@@ -857,6 +863,15 @@ mod tests {
         )
     }
 
+    fn scripted_endpoint() -> String {
+        static NEXT_SCRIPTED_BACKEND_ID: AtomicU64 = AtomicU64::new(0);
+
+        format!(
+            "https://scripted-{}.example.invalid",
+            NEXT_SCRIPTED_BACKEND_ID.fetch_add(1, Ordering::Relaxed)
+        )
+    }
+
     fn scripted_backend(events: Vec<ReplayEvent>) -> (StaticReplayClient, AwsKmsBackend) {
         let http_client = StaticReplayClient::new(events);
         let sdk_config = aws_sdk_kms::Config::builder()
@@ -867,10 +882,20 @@ mod tests {
             .retry_config(aws_sdk_kms::config::retry::RetryConfig::disabled())
             .build();
         let kms_config = KmsConfig::aws(Some("us-east-1".to_string()));
-        let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config);
+        let endpoint = scripted_endpoint();
+        let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config, &endpoint, "us-east-1");
         (http_client, backend)
     }
 
+    fn aws_config(endpoint: &str, region: &str) -> KmsConfig {
+        let mut config = KmsConfig::aws(Some(region.to_owned()));
+        let BackendConfig::Aws(aws) = &mut config.backend_config else {
+            panic!("AWS constructor must create an AWS backend configuration");
+        };
+        aws.endpoint_url = Some(endpoint.to_owned());
+        config
+    }
+
     fn key_metadata_json(key_id: &str, state: &str) -> serde_json::Value {
         serde_json::json!({
             "KeyMetadata": {
@@ -900,6 +925,27 @@ mod tests {
             .expect("capabilities should deserialize into a flat bool map")
     }
 
+    #[tokio::test]
+    async fn aws_backend_new_identity_includes_endpoint_and_region() {
+        let endpoint = scripted_endpoint();
+        let first = AwsKmsBackend::new(aws_config(&endpoint, "us-east-1"))
+            .await
+            .expect("first AWS backend");
+        let matching = AwsKmsBackend::new(aws_config(&endpoint, "us-east-1"))
+            .await
+            .expect("matching AWS backend");
+        let other_region = AwsKmsBackend::new(aws_config(&endpoint, "us-west-2"))
+            .await
+            .expect("other-region AWS backend");
+        let other_endpoint = AwsKmsBackend::new(aws_config(&scripted_endpoint(), "us-east-1"))
+            .await
+            .expect("other-endpoint AWS backend");
+
+        assert!(first.retry.shares_active_capacity_with(&matching.retry));
+        assert!(!first.retry.shares_active_capacity_with(&other_region.retry));
+        assert!(!first.retry.shares_active_capacity_with(&other_endpoint.retry));
+    }
+
     #[tokio::test]
     async fn aws_backend_capabilities_golden() {
         let (_http, backend) = scripted_backend(Vec::new());
@@ -1023,7 +1069,7 @@ mod tests {
         let mut kms_config = KmsConfig::aws(Some("us-east-1".to_string()));
         kms_config.timeout = std::time::Duration::from_millis(5_000);
         kms_config.retry_attempts = 1;
-        let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config);
+        let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config, "", "us-east-1");
 
         let error = backend
             .describe_key(DescribeKeyRequest {

crates/kms/src/backends/vault.rs

@@ -227,7 +227,13 @@ impl VaultKmsClient {
             attempt_timeout: kms_config.effective_timeout(),
         };
         let source = token_source_for(&config.auth_method, &settings)?;
-        let policy = VaultCredentialPolicy::from_kms_config(kms_config, &config.auth_method);
+        let policy = VaultCredentialPolicy::from_kms_config(
+            kms_config,
+            &config.auth_method,
+            "vault-kv2",
+            &config.address,
+            config.namespace.as_deref(),
+        );
         let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?);
 
         info!(address = %config.address, "Vault KMS backend connected");
@@ -237,7 +243,7 @@ impl VaultKmsClient {
             kv_mount: config.kv_mount.clone(),
             key_path_prefix: config.key_path_prefix.clone(),
             dek_crypto: AesDekCrypto::new(),
-            retry: RetryPolicy::from_config(kms_config),
+            retry: RetryPolicy::for_backend(kms_config, "vault-kv2", &config.address, config.namespace.as_deref(), "operations"),
             cancel: CancellationToken::new(),
         })
     }

crates/kms/src/backends/vault_credentials.rs

@@ -520,8 +520,10 @@ impl VaultConnectionSettings {
 /// Refresh and fail-closed tuning for a [`VaultCredentialProvider`].
 #[derive(Debug, Clone)]
 pub(crate) struct VaultCredentialPolicy {
-    /// Retry budget for one login/renewal cycle.
+    /// Retry budget for one login cycle.
     pub(crate) retry: RetryPolicy,
+    /// Retry budget for one token-renewal cycle.
+    pub(crate) renew_retry: RetryPolicy,
     /// Fail-closed margin: once the current token is within this window of
     /// expiry without a successful refresh, [`VaultCredentialProvider::current`]
     /// refuses to hand it out.
@@ -536,8 +538,14 @@ impl VaultCredentialPolicy {
     /// The default safety window equals the per-attempt timeout: a request
     /// issued now can stay in flight for up to one attempt timeout, so the
     /// token must outlive at least that.
-    pub(crate) fn from_kms_config(config: &KmsConfig, auth_method: &VaultAuthMethod) -> Self {
-        let retry = RetryPolicy::from_config(config);
+    pub(crate) fn from_kms_config(
+        config: &KmsConfig,
+        auth_method: &VaultAuthMethod,
+        backend: &'static str,
+        endpoint: &str,
+        namespace: Option<&str>,
+    ) -> Self {
+        let retry = RetryPolicy::for_credentials(config, backend, endpoint, namespace, "credentials-login");
         let safety_window = match auth_method {
             VaultAuthMethod::AppRole {
                 refresh_safety_window_secs: Some(secs),
@@ -551,6 +559,7 @@ impl VaultCredentialPolicy {
         };
         Self {
             retry,
+            renew_retry: RetryPolicy::for_credentials(config, backend, endpoint, namespace, "credentials-renew"),
             safety_window,
             retry_interval: DEFAULT_REFRESH_RETRY_INTERVAL,
         }
@@ -717,7 +726,7 @@ impl VaultCredentialProvider {
 
         let renewable = current.lease.map(|lease| lease.renewable).unwrap_or(false);
         let renewed = if renewable {
-            match policy::execute("vault_token_renew", OpClass::Auth, &self.policy.retry, cancel, || {
+            match policy::execute("vault_token_renew", OpClass::Auth, &self.policy.renew_retry, cancel, || {
                 self.source.renew(&current.client)
             })
             .await
@@ -868,19 +877,56 @@ mod tests {
     /// Tight retry budget so paused-clock tests stay deterministic: one
     /// attempt per cycle, failed cycles spaced by `retry_interval`.
     fn test_policy(safety_window: Duration, retry_interval: Duration) -> VaultCredentialPolicy {
+        let retry = RetryPolicy::for_test(
+            Duration::from_secs(1),
+            Duration::from_secs(1),
+            1,
+            Duration::from_millis(10),
+            Duration::from_millis(10),
+        );
+        let renew_retry = RetryPolicy::for_test(
+            Duration::from_secs(1),
+            Duration::from_secs(1),
+            1,
+            Duration::from_millis(10),
+            Duration::from_millis(10),
+        );
         VaultCredentialPolicy {
-            retry: RetryPolicy {
-                attempt_timeout: Duration::from_secs(1),
-                op_deadline: Duration::from_secs(1),
-                max_attempts: 1,
-                base_backoff: Duration::from_millis(10),
-                max_backoff: Duration::from_millis(10),
-            },
+            retry,
+            renew_retry,
             safety_window,
             retry_interval,
         }
     }
 
+    #[test]
+    fn configured_login_and_renewal_use_reserved_credential_capacity() {
+        let config = KmsConfig::default();
+        let auth_method = VaultAuthMethod::Token {
+            token: TEST_TOKEN.to_string(),
+        };
+        let credentials = VaultCredentialPolicy::from_kms_config(
+            &config,
+            &auth_method,
+            "vault-kv2",
+            "https://credential-policy.example.invalid",
+            Some("team-namespace"),
+        );
+        let operations = RetryPolicy::for_backend(
+            &config,
+            "vault-kv2",
+            "https://credential-policy.example.invalid",
+            Some("team-namespace"),
+            "operations",
+        );
+
+        assert!(credentials.retry.uses_credential_reserve());
+        assert!(credentials.renew_retry.uses_credential_reserve());
+        assert!(!operations.uses_credential_reserve());
+        assert!(credentials.retry.shares_active_capacity_with(&credentials.renew_retry));
+        assert!(credentials.retry.shares_active_capacity_with(&operations));
+    }
+
     /// Shared observable state of a [`ScriptedSource`].
     #[derive(Debug, Default)]
     struct ScriptedState {
@@ -1084,11 +1130,12 @@ mod tests {
             "expected CredentialsUnavailable, got {error:?}"
         );
 
-        // Recovery: the next retry cycle succeeds, installs a fresh
-        // generation, and the provider serves requests again.
+        // Recovery: after the failed-refresh circuit's cool-down, its single
+        // half-open probe succeeds, installs a fresh generation, and the
+        // provider serves requests again.
         state.fail_renew.store(false, Ordering::SeqCst);
         state.fail_login.store(false, Ordering::SeqCst);
-        tokio::time::sleep(Duration::from_secs(6)).await;
+        tokio::time::sleep(Duration::from_secs(31)).await;
         let handle = provider.current().expect("provider must recover after a successful refresh");
         assert!(handle.generation >= 1);
 

crates/kms/src/backends/vault_transit.rs

@@ -230,8 +230,16 @@ impl VaultTransitKmsClient {
             attempt_timeout: kms_config.effective_timeout(),
         };
         let source = token_source_for(&config.auth_method, &settings)?;
-        let policy = VaultCredentialPolicy::from_kms_config(kms_config, &config.auth_method);
+        let policy = VaultCredentialPolicy::from_kms_config(
+            kms_config,
+            &config.auth_method,
+            "vault-transit",
+            &config.address,
+            config.namespace.as_deref(),
+        );
         let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?);
+        let retry =
+            RetryPolicy::for_backend(kms_config, "vault-transit", &config.address, config.namespace.as_deref(), "operations");
 
         Ok(Self {
             credentials,
@@ -242,7 +250,7 @@ impl VaultTransitKmsClient {
                 .max_capacity(METADATA_CACHE_CAPACITY)
                 .time_to_live(METADATA_CACHE_TTL)
                 .build(),
-            retry: RetryPolicy::from_config(kms_config),
+            retry,
             cancel: CancellationToken::new(),
         })
     }

crates/kms/src/backup/vault_restore.rs

@@ -452,12 +452,24 @@ impl VaultRestoreClient {
             attempt_timeout: kms_config.effective_timeout(),
         };
         let source = token_source_for(&target.auth_method, &settings)?;
-        let policy = VaultCredentialPolicy::from_kms_config(kms_config, &target.auth_method);
+        let policy = VaultCredentialPolicy::from_kms_config(
+            kms_config,
+            &target.auth_method,
+            "vault-restore",
+            &target.address,
+            target.namespace.as_deref(),
+        );
         let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?);
         Ok(Self {
             credentials,
             kv_mount: target.kv_mount.clone(),
-            retry: RetryPolicy::from_config(kms_config),
+            retry: RetryPolicy::for_backend(
+                kms_config,
+                "vault-restore",
+                &target.address,
+                target.namespace.as_deref(),
+                "operations",
+            ),
             cancel: CancellationToken::new(),
         })
     }

deploy/observability/grafana/rustfs-kms-observability.json

@@ -21,7 +21,7 @@
       }
     ]
   },
-  "description": "KMS backend operation metrics emitted at the operation-policy choke point (crates/kms/src/policy.rs). All label values are static enum strings; key identifiers, key material, and tokens never appear in labels. These metrics do not carry the RustFS `server` label — use your scrape topology (job/instance or promoted OTel resource attributes) to split by node. Alert response procedures: docs/operations/kms-observability-runbook.md.",
+  "description": "KMS backend operation metrics emitted at the operation-policy choke point (crates/kms/src/policy.rs). All label values are bounded enums or fixed call-site tokens; key identifiers, key material, and tokens never appear in labels. These metrics do not carry the RustFS `server` label — use your scrape topology (job/instance or promoted OTel resource attributes) to split by node. Alert response procedures: docs/operations/kms-observability-runbook.md.",
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
@@ -34,7 +34,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
-      "description": "Terminal outcomes of KMS backend operations. `fatal` means a non-retryable failure ended the operation on first observation; `budget_exhausted` and `deadline_exceeded` mean retries ran out; `cancelled` is normal during shutdown.",
+      "description": "Terminal outcomes of KMS backend operations. `fatal` is non-retryable; `budget_exhausted` and `deadline_exceeded` mean retry limits ran out; `backpressure_timeout` and `backpressure_rejected` are admission failures; `circuit_open` means the breaker opened or rejected the operation; `cancelled` is normal during shutdown.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -218,7 +218,7 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
-      "description": "Share of operations that terminated in fatal, budget_exhausted, or deadline_exceeded. The cancelled outcome is plotted separately because shutdown windows legitimately spike it. The ratio is meaningless at near-zero traffic — read it together with the operation rate panels.",
+      "description": "Share of operations that terminated in fatal, budget_exhausted, deadline_exceeded, backpressure_timeout, backpressure_rejected, or circuit_open. The cancelled outcome is plotted separately because shutdown windows legitimately spike it. The ratio is meaningless at near-zero traffic — read it together with the operation rate panels.",
       "fieldConfig": {
         "defaults": {
           "color": {

docs/architecture/global-state-inventory.md

@@ -78,6 +78,7 @@ of reaching across module boundaries.
 | `DRIVE_TIMEOUT_PROFILE_CACHE`, `DRIVE_TIMEOUT_HEALTH_POLICY_CACHE` | `crates/ecstore/src/disk/disk_store.rs` | Cache or constant / owner-local cache | Drive timeout environment caches stay local to the disk-store owner. |
 | `TIER_FREE_VERSION_RECOVERY_STARTED`, `TIER_DELETE_JOURNAL_RECOVERY_STARTED` | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | Cache or constant / owner-local static guard | Lifecycle recovery single-run guards stay local to lifecycle operations. |
 | `REMOTE_DELETE_INFLIGHT`, `REMOTE_DELETE_LIMITER`, `REMOTE_DELETE_BREAKER`, `REMOTE_TIER_DELETE_TEST_HOOK` | `crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs` | Cache or constant / owner-local static guard | Remote tier delete concurrency, breaker, and test hook state stay local to the tier sweeper owner. |
+| `ACTIVE_REGISTRY`, `BackendCapacity` | `crates/kms/src/policy.rs` | Process-global owner-local admission capacity registry | KMS policy generations share only active semaphore capacity by backend identity; each generation owns fresh bounded queues and circuit breakers. Callers access this state only through `RetryPolicy`. |
 
 ## RustFS Owner-Local Static Inventory
 

docs/operations/kms-observability-runbook.md

@@ -4,11 +4,11 @@ This runbook covers the KMS metrics, the Grafana dashboard that visualizes them,
 
 ## Metric reference
 
-Across every family below, label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression.
+Across every family below, label values come from bounded enums or fixed call-site tokens — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression.
 
 ### Backend operation metrics
 
-All four are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through.
+All six are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through.
 
 | Metric | Type | Labels | Meaning |
 | --- | --- | --- | --- |
@@ -16,15 +16,19 @@ All four are emitted at the single operation-policy choke point (`crates/kms/src
 | `rustfs_kms_backend_attempt_failures_total` | counter | `operation`, `error_class` | Individual failed attempts, including attempts the retry policy later absorbed |
 | `rustfs_kms_backend_operation_duration_seconds` | histogram | `operation`, `outcome` | Wall-clock duration of a whole operation, including retries and backoff sleeps |
 | `rustfs_kms_backend_operation_attempts` | histogram | `operation`, `outcome` | Number of attempts one operation used before completing |
+| `rustfs_kms_backend_in_flight` | gauge | `backend`, `scope` | External backend attempts currently in flight after admission |
+| `rustfs_kms_backend_circuit_open` | gauge | `backend`, `scope` | Open or half-open circuits; `0` means closed |
 
 Label values:
 
-- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `cancelled` (shutdown or caller cancellation).
+- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `backpressure_timeout` (the deadline elapsed before capacity admission completed), `backpressure_rejected` (active capacity and the bounded queue were full or unavailable), `circuit_open` (a retryable failure opened the breaker or an open breaker rejected the operation), `cancelled` (shutdown or caller cancellation).
 - `op_class`: `read_idempotent` (safe to retry), `mutating_non_idempotent` (never replayed — a retryable failure terminates after a single attempt because the server may have processed the request), `auth` (login and token renewal).
 - `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version).
 - `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`.
 
-Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of these four series on a cluster using those backends is expected, not an outage. The families below sit above the backend layer and are emitted regardless.
+Admission sharing follows two different boundaries. Total active backend capacity is shared by backend identity and capped at 64; ordinary operations are limited to 63 so login and renewal always retain one reserved slot without exceeding the total cap. Each backend configuration generation owns fresh bounded queues and circuit breakers for its policy scopes, so a failed reconfiguration candidate cannot inherit or mutate the running generation's admission state.
+
+Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of these six series on a cluster using those backends is expected, not an outage. The families below sit above the backend layer and are emitted regardless.
 
 ### Key metadata cache metrics
 
@@ -116,14 +120,16 @@ Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operatio
 
 ### KmsBackendHighErrorRate
 
-Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, or `deadline_exceeded`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page.
+Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, `deadline_exceeded`, `backpressure_timeout`, `backpressure_rejected`, or `circuit_open`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page.
 
 Investigation:
 
 1. Break the failures down by outcome: `sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))`.
 2. If `fatal` dominates, follow [KmsBackendFatalErrors](#kmsbackendfatalerrors).
 3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted) — the backend is unavailable or too slow for longer than the retry policy can bridge.
-4. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured.
+4. If `backpressure_timeout` or `backpressure_rejected` dominates, compare `rustfs_kms_backend_in_flight` by `backend` and `scope`; total active capacity is shared by backend identity, one slot is reserved for credential refresh, and each configuration generation has fresh scope-local bounded queues.
+5. If `circuit_open` dominates, follow [KmsBackendCircuitOpen](#kmsbackendcircuitopen).
+6. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured.
 
 Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts in this runbook.
 
@@ -170,9 +176,23 @@ Investigation:
 
 Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings in RustFS logs; Vault availability monitoring.
 
+### KmsBackendCircuitOpen
+
+Meaning: `rustfs_kms_backend_circuit_open` has remained above `0` for a `backend` and `scope` for one minute. This direct gauge alert does not depend on operation traffic: it remains visible when the circuit is open and rejecting calls, and while the single half-open recovery probe is running.
+
+Investigation:
+
+1. Identify the affected scope with `rustfs_kms_backend_circuit_open > 0`.
+2. Break recent rejections down by operation: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="circuit_open"}[5m]))`.
+3. Check `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total{error_class=~"retryable_conn|retryable_status|attempt_timeout"}[5m]))` to distinguish transport failures, retryable backend responses such as a sealed Vault, and attempt timeouts. An attempt timeout counts toward the breaker as a retryable connection failure.
+4. After the open interval, the next eligible operation is the only half-open probe. A success or non-retryable failure closes the circuit; a retryable failure reopens it. A non-retryable probe still fails as `fatal`, so follow [KmsBackendFatalErrors](#kmsbackendfatalerrors) even after the circuit gauge clears. Do not restart RustFS just to clear the state.
+5. Remember the sharing boundary: each configuration generation has fresh scope-local breaker and queue state, while total active capacity is shared by backend identity with one slot reserved for credential refresh. Check other scopes for capacity pressure even when their circuits remain closed.
+
+Related signals: `circuit_open`, `backpressure_timeout`, and `backpressure_rejected` on the "Backend Operation Rate by Outcome" panel; `rustfs_kms_backend_in_flight`; Vault availability and seal status.
+
 ## Threshold calibration
 
-Every numeric threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).
+Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).
 
 ## Coverage gaps