test(kms): exercise real Vault Raft failover (#5653)

2026-08-05 12:57:42 +00:00 · 2026-08-03 05:25:23 +08:00
parent fbb6cebeb4
commit 9dd0461f3e
2 changed files with 680 additions and 0 deletions
@@ -0,0 +1,390 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Ignored live test for a real three-node Vault Raft leader failure.
+//!
+//! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
+//! kills the active node while this test continuously decrypts through a
+//! surviving standby. KV2 and Transit requests must remain successful, use a
+//! bounded number of attempts, and leave the circuit and in-flight gauges at
+//! zero after a new leader is elected.
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::time::Duration;
+
+use metrics_util::MetricKind;
+use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
+use rustfs_kms::backends::KmsBackend as KmsBackendTrait;
+use rustfs_kms::backends::vault::VaultKmsBackend;
+use rustfs_kms::backends::vault_transit::VaultTransitKmsBackend;
+use rustfs_kms::{
+    BackendConfig, CreateKeyRequest, DecryptRequest, GenerateDataKeyRequest, KeySpec, KeyUsage, KmsBackend, KmsConfig,
+    VaultAuthMethod, VaultConfig, VaultTransitConfig,
+};
+use tokio_util::sync::CancellationToken;
+
+const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total";
+const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total";
+const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
+const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
+const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
+const MAX_ATTEMPTS: u32 = 10;
+
+type MetricEntry = (
+    metrics_util::CompositeKey,
+    Option<metrics::Unit>,
+    Option<metrics::SharedString>,
+    DebugValue,
+);
+
+fn required_env(name: &str) -> String {
+    std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set by scripts/test/vault_ha_kms_live.sh"))
+}
+
+fn auth_method() -> VaultAuthMethod {
+    VaultAuthMethod::approle(required_env("RUSTFS_TEST_VAULT_ROLE_ID"), required_env("RUSTFS_TEST_VAULT_SECRET_ID"))
+}
+
+fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
+    KmsConfig {
+        backend,
+        backend_config,
+        allow_insecure_dev_defaults: true,
+        timeout: Duration::from_secs(2),
+        retry_attempts: MAX_ATTEMPTS,
+        enable_cache: false,
+        ..KmsConfig::default()
+    }
+}
+
+fn kv2_config(address: &str) -> KmsConfig {
+    config(
+        KmsBackend::VaultKv2,
+        BackendConfig::VaultKv2(Box::new(VaultConfig {
+            address: address.to_string(),
+            auth_method: auth_method(),
+            namespace: None,
+            mount_path: "transit".to_string(),
+            kv_mount: "secret".to_string(),
+            key_path_prefix: "rustfs/kms/ha-kv2".to_string(),
+            tls: None,
+        })),
+    )
+}
+
+fn transit_config(address: &str) -> KmsConfig {
+    config(
+        KmsBackend::VaultTransit,
+        BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
+            address: address.to_string(),
+            auth_method: auth_method(),
+            namespace: None,
+            mount_path: "transit".to_string(),
+            metadata_kv_mount: "secret".to_string(),
+            metadata_key_prefix: "rustfs/kms/ha-transit-metadata".to_string(),
+            tls: None,
+        })),
+    )
+}
+
+fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
+    labels.iter().all(|(label, expected)| {
+        key.labels()
+            .any(|candidate| candidate.key() == *label && candidate.value() == *expected)
+    })
+}
+
+fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
+    snapshot
+        .iter()
+        .filter_map(|(composite, _, _, value)| {
+            let matches = composite.kind() == MetricKind::Counter
+                && composite.key().name() == name
+                && labels_match(composite.key(), labels);
+            match (matches, value) {
+                (true, DebugValue::Counter(count)) => Some(*count),
+                _ => None,
+            }
+        })
+        .sum()
+}
+
+fn gauge_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Option<f64> {
+    snapshot.iter().find_map(|(composite, _, _, value)| {
+        let matches =
+            composite.kind() == MetricKind::Gauge && composite.key().name() == name && labels_match(composite.key(), labels);
+        match (matches, value) {
+            (true, DebugValue::Gauge(value)) => Some(value.into_inner()),
+            _ => None,
+        }
+    })
+}
+
+fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
+    snapshot
+        .iter()
+        .filter_map(|(composite, _, _, value)| {
+            let matches = composite.kind() == MetricKind::Histogram
+                && composite.key().name() == name
+                && labels_match(composite.key(), labels);
+            match (matches, value) {
+                (true, DebugValue::Histogram(values)) => Some(values),
+                _ => None,
+            }
+        })
+        .flatten()
+        .map(|value| value.into_inner())
+        .collect()
+}
+
+fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
+    ["retryable_conn", "retryable_status", "attempt_timeout"]
+        .into_iter()
+        .map(|error_class| {
+            counter_value(
+                snapshot,
+                ATTEMPT_FAILURES_TOTAL,
+                &[("operation", operation), ("error_class", error_class)],
+            )
+        })
+        .sum()
+}
+
+async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
+    tokio::time::timeout(Duration::from_secs(20), async {
+        while counter.load(Ordering::SeqCst) < minimum {
+            tokio::time::sleep(Duration::from_millis(25)).await;
+        }
+    })
+    .await
+    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+}
+
+async fn wait_for_file(path: &Path, description: &str) {
+    tokio::time::timeout(Duration::from_secs(70), async {
+        while !path.exists() {
+            tokio::time::sleep(Duration::from_millis(25)).await;
+        }
+    })
+    .await
+    .unwrap_or_else(|_| panic!("timed out waiting for {description}"));
+}
+
+async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
+    backend: Arc<B>,
+    request: DecryptRequest,
+    expected: Vec<u8>,
+    completed: Arc<AtomicU64>,
+    failed: Arc<AtomicBool>,
+    stop: CancellationToken,
+) {
+    while !stop.is_cancelled() {
+        match backend.decrypt(request.clone()).await {
+            Ok(response) if response.plaintext == expected => {
+                completed.fetch_add(1, Ordering::SeqCst);
+            }
+            Ok(_) | Err(_) => {
+                failed.store(true, Ordering::SeqCst);
+                return;
+            }
+        }
+    }
+}
+
+async fn exercise_failover(snapshotter: &Snapshotter) {
+    let address = required_env("RUSTFS_TEST_VAULT_ADDRESS");
+    let marker = PathBuf::from(required_env("RUSTFS_TEST_VAULT_FAILOVER_MARKER"));
+    let elected = marker.with_extension("elected");
+    let old_leader = required_env("RUSTFS_TEST_VAULT_OLD_LEADER");
+
+    let kv2 = Arc::new(VaultKmsBackend::new(kv2_config(&address)).await.expect("build KV2 backend"));
+    let transit = Arc::new(
+        VaultTransitKmsBackend::new(transit_config(&address))
+            .await
+            .expect("build Transit backend"),
+    );
+    let context = HashMap::from([("live".to_string(), "vault-ha-failover".to_string())]);
+
+    let kv2_key = format!("rustfs-ha-kv2-{}", uuid::Uuid::new_v4());
+    kv2.create_key(CreateKeyRequest {
+        key_name: Some(kv2_key.clone()),
+        key_usage: KeyUsage::EncryptDecrypt,
+        ..Default::default()
+    })
+    .await
+    .expect("create KV2 key");
+    let kv2_data_key = kv2
+        .generate_data_key(GenerateDataKeyRequest {
+            key_id: kv2_key,
+            key_spec: KeySpec::Aes256,
+            encryption_context: context.clone(),
+        })
+        .await
+        .expect("generate KV2 data key");
+
+    let transit_key = format!("rustfs-ha-transit-{}", uuid::Uuid::new_v4());
+    transit
+        .create_key(CreateKeyRequest {
+            key_name: Some(transit_key.clone()),
+            key_usage: KeyUsage::EncryptDecrypt,
+            ..Default::default()
+        })
+        .await
+        .expect("create Transit key");
+    let transit_data_key = transit
+        .generate_data_key(GenerateDataKeyRequest {
+            key_id: transit_key,
+            key_spec: KeySpec::Aes256,
+            encryption_context: context.clone(),
+        })
+        .await
+        .expect("generate Transit data key");
+
+    let kv2_request = DecryptRequest {
+        ciphertext: kv2_data_key.ciphertext_blob,
+        encryption_context: context.clone(),
+        grant_tokens: Vec::new(),
+    };
+    let transit_request = DecryptRequest {
+        ciphertext: transit_data_key.ciphertext_blob,
+        encryption_context: context,
+        grant_tokens: Vec::new(),
+    };
+
+    for _ in 0..2 {
+        let kv2_response = kv2
+            .decrypt(kv2_request.clone())
+            .await
+            .expect("healthy KV2 decrypt before failover");
+        assert!(
+            kv2_response.plaintext == kv2_data_key.plaintext_key,
+            "healthy KV2 decrypt returned unexpected plaintext"
+        );
+        let transit_response = transit
+            .decrypt(transit_request.clone())
+            .await
+            .expect("healthy Transit decrypt before failover");
+        assert!(
+            transit_response.plaintext == transit_data_key.plaintext_key,
+            "healthy Transit decrypt returned unexpected plaintext"
+        );
+    }
+    let baseline = snapshotter.snapshot().into_vec();
+    assert_eq!(
+        retryable_failures(&baseline, "vault_kv2_read_key"),
+        0,
+        "healthy KV2 baseline must not retry"
+    );
+    assert_eq!(
+        retryable_failures(&baseline, "vault_transit_decrypt"),
+        0,
+        "healthy Transit baseline must not retry"
+    );
+
+    let stop = CancellationToken::new();
+    let failed = Arc::new(AtomicBool::new(false));
+    let kv2_completed = Arc::new(AtomicU64::new(0));
+    let transit_completed = Arc::new(AtomicU64::new(0));
+    let kv2_worker = tokio::spawn(decrypt_loop(
+        Arc::clone(&kv2),
+        kv2_request,
+        kv2_data_key.plaintext_key,
+        Arc::clone(&kv2_completed),
+        Arc::clone(&failed),
+        stop.clone(),
+    ));
+    let transit_worker = tokio::spawn(decrypt_loop(
+        Arc::clone(&transit),
+        transit_request,
+        transit_data_key.plaintext_key,
+        Arc::clone(&transit_completed),
+        Arc::clone(&failed),
+        stop.clone(),
+    ));
+
+    wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
+    wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
+    std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
+
+    wait_for_file(&elected, "the replacement Vault leader").await;
+    let new_leader = std::fs::read_to_string(&elected).expect("read replacement Vault leader marker");
+    assert_ne!(new_leader.trim(), old_leader, "the killed active node cannot remain leader");
+
+    let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
+    let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
+    wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
+    wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
+
+    stop.cancel();
+    kv2_worker.await.expect("KV2 decrypt worker must join");
+    transit_worker.await.expect("Transit decrypt worker must join");
+    assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
+}
+
+#[test]
+#[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
+fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    metrics::with_local_recorder(&recorder, || {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("current-thread runtime must build")
+            .block_on(exercise_failover(&snapshotter));
+    });
+    let snapshot = snapshotter.snapshot().into_vec();
+
+    assert_eq!(
+        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
+        0,
+        "a bounded leader election must not open the circuit"
+    );
+    assert_eq!(
+        counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
+        0,
+        "leader failover must recover within the retry budget"
+    );
+
+    for (backend, operation) in [
+        ("vault-kv2", "vault_kv2_read_key"),
+        ("vault-transit", "vault_transit_decrypt"),
+    ] {
+        assert!(
+            retryable_failures(&snapshot, operation) > 0,
+            "{operation} must observe the killed leader as a retryable attempt failure"
+        );
+        let attempts = histogram_values(&snapshot, OPERATION_ATTEMPTS, &[("operation", operation), ("outcome", "success")]);
+        assert!(!attempts.is_empty(), "{operation} must record successful attempts");
+        assert!(
+            attempts
+                .iter()
+                .all(|attempts| (1.0..=f64::from(MAX_ATTEMPTS)).contains(attempts)),
+            "{operation} attempts must stay within the configured budget: {attempts:?}"
+        );
+        assert_eq!(
+            gauge_value(&snapshot, IN_FLIGHT, &[("backend", backend), ("scope", "operations")]),
+            Some(0.0),
+            "{backend} must release every in-flight permit"
+        );
+        assert_eq!(
+            gauge_value(&snapshot, CIRCUIT_OPEN, &[("backend", backend), ("scope", "operations")]),
+            Some(0.0),
+            "{backend} circuit must remain closed after recovery"
+        );
+    }
+}
@@ -0,0 +1,290 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run the ignored RustFS KMS failover test against an ephemeral, official
+# three-node Vault cluster using integrated Raft storage. The active container
+# is killed only after KV2 and Transit decrypt loops report ready.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+VAULT_IMAGE="${RUSTFS_TEST_VAULT_IMAGE:-hashicorp/vault:1.17.6}"
+
+for command in docker jq; do
+  if ! command -v "$command" >/dev/null 2>&1; then
+    echo "$command is required for the live Vault HA test" >&2
+    exit 1
+  fi
+done
+if ! docker info >/dev/null 2>&1; then
+  echo "Docker is not available" >&2
+  exit 1
+fi
+if ! docker image inspect "$VAULT_IMAGE" >/dev/null 2>&1; then
+  docker pull "$VAULT_IMAGE" >/dev/null
+fi
+
+TMP_DIR="$(mktemp -d -t rustfs-vault-ha-live.XXXXXX)"
+RUN_ID="rustfs-kms-ha-$$-${RANDOM}"
+NETWORK="${RUN_ID}"
+MARKER="${TMP_DIR}/failover-ready"
+ROOT_TOKEN=""
+UNSEAL_KEY=""
+TEST_PID=""
+declare -a NODES=("${RUN_ID}-1" "${RUN_ID}-2" "${RUN_ID}-3")
+
+cleanup() {
+  if [[ -n "$TEST_PID" ]]; then
+    kill "$TEST_PID" 2>/dev/null || true
+    wait "$TEST_PID" 2>/dev/null || true
+  fi
+  for node in "${NODES[@]}"; do
+    docker rm -f "$node" >/dev/null 2>&1 || true
+  done
+  docker network rm "$NETWORK" >/dev/null 2>&1 || true
+  rm -rf "$TMP_DIR"
+}
+trap cleanup EXIT
+
+docker network create "$NETWORK" >/dev/null
+
+start_node() {
+  local index="$1"
+  local node="${NODES[$((index - 1))]}"
+  local config
+  config="$(jq -cn \
+    --arg api "http://${node}:8200" \
+    --arg cluster "http://${node}:8201" \
+    --arg node_id "node${index}" \
+    '{
+      ui: false,
+      disable_mlock: true,
+      api_addr: $api,
+      cluster_addr: $cluster,
+      listener: [{tcp: {
+        address: "0.0.0.0:8200",
+        cluster_address: "0.0.0.0:8201",
+        tls_disable: true
+      }}],
+      storage: {raft: {path: "/vault/file", node_id: $node_id, performance_multiplier: 1}}
+    }')"
+  docker run -d \
+    --name "$node" \
+    --network "$NETWORK" \
+    --cap-add IPC_LOCK \
+    -p 127.0.0.1::8200 \
+    -e VAULT_LOCAL_CONFIG="$config" \
+    "$VAULT_IMAGE" server >/dev/null
+}
+
+vault_exec() {
+  local node="$1"
+  shift
+  docker exec \
+    -e VAULT_ADDR=http://127.0.0.1:8200 \
+    -e VAULT_TOKEN="$ROOT_TOKEN" \
+    "$node" vault "$@"
+}
+
+wait_started() {
+  local node="$1"
+  local status
+  for _ in $(seq 1 120); do
+    status="$(docker exec -e VAULT_ADDR=http://127.0.0.1:8200 "$node" vault status -format=json 2>/dev/null || true)"
+    if jq -e '.initialized == false or .sealed == true or .sealed == false' >/dev/null 2>&1 <<<"$status"; then
+      return 0
+    fi
+    sleep 0.25
+  done
+  echo "$node did not start" >&2
+  docker logs "$node" >&2 || true
+  return 1
+}
+
+active_node() {
+  local node status address
+  for node in "${NODES[@]}"; do
+    if ! docker inspect "$node" >/dev/null 2>&1; then
+      continue
+    fi
+    status="$(docker exec -e VAULT_ADDR=http://127.0.0.1:8200 "$node" vault status -format=json 2>/dev/null || true)"
+    address="http://${node}:8200"
+    if jq -e --arg address "$address" \
+      '.ha_enabled == true and .sealed == false and (.is_self == true or (.leader_address == $address and .active_time != "0001-01-01T00:00:00Z"))' \
+      >/dev/null 2>&1 <<<"$status"; then
+      printf '%s\n' "$node"
+      return 0
+    fi
+  done
+  return 1
+}
+
+host_address() {
+  local node="$1"
+  local port
+  port="$(docker port "$node" 8200/tcp | awk -F: 'NR == 1 {print $NF}')"
+  if [[ ! "$port" =~ ^[0-9]+$ ]]; then
+    echo "failed to resolve host port for $node" >&2
+    return 1
+  fi
+  printf 'http://127.0.0.1:%s\n' "$port"
+}
+
+for index in 1 2 3; do
+  start_node "$index"
+  wait_started "${NODES[$((index - 1))]}"
+done
+
+INIT_JSON="$(docker exec -e VAULT_ADDR=http://127.0.0.1:8200 "${NODES[0]}" vault operator init -format=json -key-shares=1 -key-threshold=1)"
+ROOT_TOKEN="$(jq -r '.root_token' <<<"$INIT_JSON")"
+UNSEAL_KEY="$(jq -r '.unseal_keys_b64[0]' <<<"$INIT_JSON")"
+if [[ -z "$ROOT_TOKEN" || -z "$UNSEAL_KEY" || "$ROOT_TOKEN" == null || "$UNSEAL_KEY" == null ]]; then
+  echo "Vault initialization did not return the expected credentials" >&2
+  exit 1
+fi
+
+vault_exec "${NODES[0]}" operator unseal "$UNSEAL_KEY" >/dev/null
+for node in "${NODES[@]:1}"; do
+  vault_exec "$node" operator raft join "http://${NODES[0]}:8200" >/dev/null
+  vault_exec "$node" operator unseal "$UNSEAL_KEY" >/dev/null
+done
+
+for _ in $(seq 1 240); do
+  if vault_exec "${NODES[0]}" operator raft list-peers -format=json 2>/dev/null \
+    | jq -e '.data.config.servers | length == 3 and all(.[]; .voter == true)' >/dev/null; then
+    break
+  fi
+  sleep 0.25
+done
+if ! vault_exec "${NODES[0]}" operator raft list-peers -format=json \
+  | jq -e '.data.config.servers | length == 3 and all(.[]; .voter == true)' >/dev/null; then
+  echo "Vault Raft cluster did not stabilize with three voters" >&2
+  vault_exec "${NODES[0]}" operator raft list-peers -format=json >&2 || true
+  exit 1
+fi
+
+OLD_LEADER="$(active_node)"
+if [[ -z "$OLD_LEADER" ]]; then
+  echo "Vault did not elect an active node" >&2
+  exit 1
+fi
+STANDBY=""
+for node in "${NODES[@]}"; do
+  if [[ "$node" != "$OLD_LEADER" ]]; then
+    STANDBY="$node"
+    break
+  fi
+done
+VAULT_ADDR="$(host_address "$STANDBY")"
+
+vault_exec "$OLD_LEADER" audit enable file file_path=/tmp/vault-audit.log >/dev/null
+vault_exec "$OLD_LEADER" secrets enable -path=secret kv-v2 >/dev/null
+vault_exec "$OLD_LEADER" secrets enable -path=transit transit >/dev/null
+vault_exec "$OLD_LEADER" auth enable approle >/dev/null
+
+POLICY_NAME="rustfs-kms-ha"
+ROLE_NAME="rustfs-kms-ha"
+POLICY_FILE="${TMP_DIR}/kms-policy.hcl"
+cat >"$POLICY_FILE" <<'EOF'
+path "secret/data/rustfs/kms/ha-kv2/*" {
+  capabilities = ["create", "read", "update"]
+}
+path "secret/metadata/rustfs/kms/ha-kv2/*" {
+  capabilities = ["list", "read", "delete"]
+}
+path "secret/metadata/rustfs/kms/ha-kv2" {
+  capabilities = ["list"]
+}
+path "secret/data/rustfs/kms/ha-transit-metadata/*" {
+  capabilities = ["create", "read", "update"]
+}
+path "secret/metadata/rustfs/kms/ha-transit-metadata/*" {
+  capabilities = ["list", "read", "delete"]
+}
+path "secret/metadata/rustfs/kms/ha-transit-metadata" {
+  capabilities = ["list"]
+}
+path "transit/keys" {
+  capabilities = ["list"]
+}
+path "transit/keys/*" {
+  capabilities = ["create", "read", "update"]
+}
+path "transit/encrypt/*" {
+  capabilities = ["update"]
+}
+path "transit/decrypt/*" {
+  capabilities = ["update"]
+}
+EOF
+docker cp "$POLICY_FILE" "${OLD_LEADER}:/tmp/kms-policy.hcl"
+vault_exec "$OLD_LEADER" policy write "$POLICY_NAME" /tmp/kms-policy.hcl >/dev/null
+vault_exec "$OLD_LEADER" write "auth/approle/role/${ROLE_NAME}" \
+  token_policies="$POLICY_NAME" \
+  token_ttl=10m \
+  token_max_ttl=20m \
+  secret_id_ttl=30m \
+  secret_id_num_uses=0 \
+  >/dev/null
+ROLE_ID="$(vault_exec "$OLD_LEADER" read -field=role_id "auth/approle/role/${ROLE_NAME}/role-id")"
+SECRET_ID="$(vault_exec "$OLD_LEADER" write -f -field=secret_id "auth/approle/role/${ROLE_NAME}/secret-id")"
+
+cd "$PROJECT_ROOT"
+cargo test -p rustfs-kms --test vault_ha_failover_live --no-run
+env \
+  -u RUSTFS_KMS_VAULT_TOKEN \
+  -u RUSTFS_KMS_VAULT_TOKEN_FILE \
+  HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \
+  NO_PROXY=127.0.0.1,localhost no_proxy=127.0.0.1,localhost \
+  RUSTFS_TEST_VAULT_ADDRESS="$VAULT_ADDR" \
+  RUSTFS_TEST_VAULT_ROLE_ID="$ROLE_ID" \
+  RUSTFS_TEST_VAULT_SECRET_ID="$SECRET_ID" \
+  RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
+  RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
+  cargo test -p rustfs-kms --test vault_ha_failover_live \
+    vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
+    --ignored --nocapture --test-threads=1 &
+TEST_PID=$!
+
+for _ in $(seq 1 240); do
+  if [[ -f "$MARKER" ]]; then
+    break
+  fi
+  if ! kill -0 "$TEST_PID" 2>/dev/null; then
+    wait "$TEST_PID"
+  fi
+  sleep 0.25
+done
+if [[ ! -f "$MARKER" ]]; then
+  echo "KMS live test did not reach failover readiness" >&2
+  exit 1
+fi
+
+docker kill "$OLD_LEADER" >/dev/null
+NEW_LEADER=""
+for _ in $(seq 1 240); do
+  NEW_LEADER="$(active_node || true)"
+  if [[ -n "$NEW_LEADER" && "$NEW_LEADER" != "$OLD_LEADER" ]]; then
+    break
+  fi
+  sleep 0.25
+done
+if [[ -z "$NEW_LEADER" || "$NEW_LEADER" == "$OLD_LEADER" ]]; then
+  echo "Vault did not elect a replacement leader after killing $OLD_LEADER" >&2
+  exit 1
+fi
+printf '%s' "$NEW_LEADER" >"${MARKER%/*}/failover-ready.elected"
+
+wait "$TEST_PID"
+TEST_PID=""
+
+for index in 1 2 3; do
+  docker cp "${NODES[$((index - 1))]}:/tmp/vault-audit.log" "${TMP_DIR}/vault-audit-${index}.log" >/dev/null 2>&1 || true
+done
+APPROLE_LOGINS="$(jq -s '[.[] | select(.type == "request" and .request.path == "auth/approle/login" and .request.operation == "update")] | length' \
+  "${TMP_DIR}"/vault-audit-*.log 2>/dev/null || true)"
+if [[ "$APPROLE_LOGINS" != 2 ]]; then
+  echo "expected exactly two AppRole logins (one per backend), got ${APPROLE_LOGINS:-unavailable}" >&2
+  exit 1
+fi
+
+echo "Vault HA failover passed: 3 Raft voters, ${OLD_LEADER} killed, ${NEW_LEADER} elected, 2 AppRole logins"