From 9dd0461f3ea7e05825321e031ce9e61944a64cb3 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 3 Aug 2026 05:25:23 +0800 Subject: [PATCH] test(kms): exercise real Vault Raft failover (#5653) --- crates/kms/tests/vault_ha_failover_live.rs | 390 +++++++++++++++++++++ scripts/test/vault_ha_kms_live.sh | 290 +++++++++++++++ 2 files changed, 680 insertions(+) create mode 100644 crates/kms/tests/vault_ha_failover_live.rs create mode 100755 scripts/test/vault_ha_kms_live.sh diff --git a/crates/kms/tests/vault_ha_failover_live.rs b/crates/kms/tests/vault_ha_failover_live.rs new file mode 100644 index 000000000..197198d5d --- /dev/null +++ b/crates/kms/tests/vault_ha_failover_live.rs @@ -0,0 +1,390 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Ignored live test for a real three-node Vault Raft leader failure. +//! +//! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and +//! kills the active node while this test continuously decrypts through a +//! surviving standby. KV2 and Transit requests must remain successful, use a +//! bounded number of attempts, and leave the circuit and in-flight gauges at +//! zero after a new leader is elected. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; + +use metrics_util::MetricKind; +use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; +use rustfs_kms::backends::KmsBackend as KmsBackendTrait; +use rustfs_kms::backends::vault::VaultKmsBackend; +use rustfs_kms::backends::vault_transit::VaultTransitKmsBackend; +use rustfs_kms::{ + BackendConfig, CreateKeyRequest, DecryptRequest, GenerateDataKeyRequest, KeySpec, KeyUsage, KmsBackend, KmsConfig, + VaultAuthMethod, VaultConfig, VaultTransitConfig, +}; +use tokio_util::sync::CancellationToken; + +const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; +const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; +const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts"; +const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight"; +const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open"; +const MAX_ATTEMPTS: u32 = 10; + +type MetricEntry = ( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, +); + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set by scripts/test/vault_ha_kms_live.sh")) +} + +fn auth_method() -> VaultAuthMethod { + VaultAuthMethod::approle(required_env("RUSTFS_TEST_VAULT_ROLE_ID"), required_env("RUSTFS_TEST_VAULT_SECRET_ID")) +} + +fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig { + KmsConfig { + backend, + backend_config, + allow_insecure_dev_defaults: true, + timeout: Duration::from_secs(2), + retry_attempts: MAX_ATTEMPTS, + enable_cache: false, + ..KmsConfig::default() + } +} + +fn kv2_config(address: &str) -> KmsConfig { + config( + KmsBackend::VaultKv2, + BackendConfig::VaultKv2(Box::new(VaultConfig { + address: address.to_string(), + auth_method: auth_method(), + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/ha-kv2".to_string(), + tls: None, + })), + ) +} + +fn transit_config(address: &str) -> KmsConfig { + config( + KmsBackend::VaultTransit, + BackendConfig::VaultTransit(Box::new(VaultTransitConfig { + address: address.to_string(), + auth_method: auth_method(), + namespace: None, + mount_path: "transit".to_string(), + metadata_kv_mount: "secret".to_string(), + metadata_key_prefix: "rustfs/kms/ha-transit-metadata".to_string(), + tls: None, + })), + ) +} + +fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool { + labels.iter().all(|(label, expected)| { + key.labels() + .any(|candidate| candidate.key() == *label && candidate.value() == *expected) + }) +} + +fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 { + snapshot + .iter() + .filter_map(|(composite, _, _, value)| { + let matches = composite.kind() == MetricKind::Counter + && composite.key().name() == name + && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Counter(count)) => Some(*count), + _ => None, + } + }) + .sum() +} + +fn gauge_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Option { + snapshot.iter().find_map(|(composite, _, _, value)| { + let matches = + composite.kind() == MetricKind::Gauge && composite.key().name() == name && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Gauge(value)) => Some(value.into_inner()), + _ => None, + } + }) +} + +fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec { + snapshot + .iter() + .filter_map(|(composite, _, _, value)| { + let matches = composite.kind() == MetricKind::Histogram + && composite.key().name() == name + && labels_match(composite.key(), labels); + match (matches, value) { + (true, DebugValue::Histogram(values)) => Some(values), + _ => None, + } + }) + .flatten() + .map(|value| value.into_inner()) + .collect() +} + +fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 { + ["retryable_conn", "retryable_status", "attempt_timeout"] + .into_iter() + .map(|error_class| { + counter_value( + snapshot, + ATTEMPT_FAILURES_TOTAL, + &[("operation", operation), ("error_class", error_class)], + ) + }) + .sum() +} + +async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) { + tokio::time::timeout(Duration::from_secs(20), async { + while counter.load(Ordering::SeqCst) < minimum { + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("timed out waiting for {description}")); +} + +async fn wait_for_file(path: &Path, description: &str) { + tokio::time::timeout(Duration::from_secs(70), async { + while !path.exists() { + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("timed out waiting for {description}")); +} + +async fn decrypt_loop( + backend: Arc, + request: DecryptRequest, + expected: Vec, + completed: Arc, + failed: Arc, + stop: CancellationToken, +) { + while !stop.is_cancelled() { + match backend.decrypt(request.clone()).await { + Ok(response) if response.plaintext == expected => { + completed.fetch_add(1, Ordering::SeqCst); + } + Ok(_) | Err(_) => { + failed.store(true, Ordering::SeqCst); + return; + } + } + } +} + +async fn exercise_failover(snapshotter: &Snapshotter) { + let address = required_env("RUSTFS_TEST_VAULT_ADDRESS"); + let marker = PathBuf::from(required_env("RUSTFS_TEST_VAULT_FAILOVER_MARKER")); + let elected = marker.with_extension("elected"); + let old_leader = required_env("RUSTFS_TEST_VAULT_OLD_LEADER"); + + let kv2 = Arc::new(VaultKmsBackend::new(kv2_config(&address)).await.expect("build KV2 backend")); + let transit = Arc::new( + VaultTransitKmsBackend::new(transit_config(&address)) + .await + .expect("build Transit backend"), + ); + let context = HashMap::from([("live".to_string(), "vault-ha-failover".to_string())]); + + let kv2_key = format!("rustfs-ha-kv2-{}", uuid::Uuid::new_v4()); + kv2.create_key(CreateKeyRequest { + key_name: Some(kv2_key.clone()), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + }) + .await + .expect("create KV2 key"); + let kv2_data_key = kv2 + .generate_data_key(GenerateDataKeyRequest { + key_id: kv2_key, + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("generate KV2 data key"); + + let transit_key = format!("rustfs-ha-transit-{}", uuid::Uuid::new_v4()); + transit + .create_key(CreateKeyRequest { + key_name: Some(transit_key.clone()), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + }) + .await + .expect("create Transit key"); + let transit_data_key = transit + .generate_data_key(GenerateDataKeyRequest { + key_id: transit_key, + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("generate Transit data key"); + + let kv2_request = DecryptRequest { + ciphertext: kv2_data_key.ciphertext_blob, + encryption_context: context.clone(), + grant_tokens: Vec::new(), + }; + let transit_request = DecryptRequest { + ciphertext: transit_data_key.ciphertext_blob, + encryption_context: context, + grant_tokens: Vec::new(), + }; + + for _ in 0..2 { + let kv2_response = kv2 + .decrypt(kv2_request.clone()) + .await + .expect("healthy KV2 decrypt before failover"); + assert!( + kv2_response.plaintext == kv2_data_key.plaintext_key, + "healthy KV2 decrypt returned unexpected plaintext" + ); + let transit_response = transit + .decrypt(transit_request.clone()) + .await + .expect("healthy Transit decrypt before failover"); + assert!( + transit_response.plaintext == transit_data_key.plaintext_key, + "healthy Transit decrypt returned unexpected plaintext" + ); + } + let baseline = snapshotter.snapshot().into_vec(); + assert_eq!( + retryable_failures(&baseline, "vault_kv2_read_key"), + 0, + "healthy KV2 baseline must not retry" + ); + assert_eq!( + retryable_failures(&baseline, "vault_transit_decrypt"), + 0, + "healthy Transit baseline must not retry" + ); + + let stop = CancellationToken::new(); + let failed = Arc::new(AtomicBool::new(false)); + let kv2_completed = Arc::new(AtomicU64::new(0)); + let transit_completed = Arc::new(AtomicU64::new(0)); + let kv2_worker = tokio::spawn(decrypt_loop( + Arc::clone(&kv2), + kv2_request, + kv2_data_key.plaintext_key, + Arc::clone(&kv2_completed), + Arc::clone(&failed), + stop.clone(), + )); + let transit_worker = tokio::spawn(decrypt_loop( + Arc::clone(&transit), + transit_request, + transit_data_key.plaintext_key, + Arc::clone(&transit_completed), + Arc::clone(&failed), + stop.clone(), + )); + + wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await; + wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await; + std::fs::write(&marker, b"ready").expect("publish failover readiness marker"); + + wait_for_file(&elected, "the replacement Vault leader").await; + let new_leader = std::fs::read_to_string(&elected).expect("read replacement Vault leader marker"); + assert_ne!(new_leader.trim(), old_leader, "the killed active node cannot remain leader"); + + let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2; + let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2; + wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await; + wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await; + + stop.cancel(); + kv2_worker.await.expect("KV2 decrypt worker must join"); + transit_worker.await.expect("Transit decrypt worker must join"); + assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext"); +} + +#[test] +#[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"] +fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime must build") + .block_on(exercise_failover(&snapshotter)); + }); + let snapshot = snapshotter.snapshot().into_vec(); + + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]), + 0, + "a bounded leader election must not open the circuit" + ); + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), + 0, + "leader failover must recover within the retry budget" + ); + + for (backend, operation) in [ + ("vault-kv2", "vault_kv2_read_key"), + ("vault-transit", "vault_transit_decrypt"), + ] { + assert!( + retryable_failures(&snapshot, operation) > 0, + "{operation} must observe the killed leader as a retryable attempt failure" + ); + let attempts = histogram_values(&snapshot, OPERATION_ATTEMPTS, &[("operation", operation), ("outcome", "success")]); + assert!(!attempts.is_empty(), "{operation} must record successful attempts"); + assert!( + attempts + .iter() + .all(|attempts| (1.0..=f64::from(MAX_ATTEMPTS)).contains(attempts)), + "{operation} attempts must stay within the configured budget: {attempts:?}" + ); + assert_eq!( + gauge_value(&snapshot, IN_FLIGHT, &[("backend", backend), ("scope", "operations")]), + Some(0.0), + "{backend} must release every in-flight permit" + ); + assert_eq!( + gauge_value(&snapshot, CIRCUIT_OPEN, &[("backend", backend), ("scope", "operations")]), + Some(0.0), + "{backend} circuit must remain closed after recovery" + ); + } +} diff --git a/scripts/test/vault_ha_kms_live.sh b/scripts/test/vault_ha_kms_live.sh new file mode 100755 index 000000000..59b2ff2fe --- /dev/null +++ b/scripts/test/vault_ha_kms_live.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the ignored RustFS KMS failover test against an ephemeral, official +# three-node Vault cluster using integrated Raft storage. The active container +# is killed only after KV2 and Transit decrypt loops report ready. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +VAULT_IMAGE="${RUSTFS_TEST_VAULT_IMAGE:-hashicorp/vault:1.17.6}" + +for command in docker jq; do + if ! command -v "$command" >/dev/null 2>&1; then + echo "$command is required for the live Vault HA test" >&2 + exit 1 + fi +done +if ! docker info >/dev/null 2>&1; then + echo "Docker is not available" >&2 + exit 1 +fi +if ! docker image inspect "$VAULT_IMAGE" >/dev/null 2>&1; then + docker pull "$VAULT_IMAGE" >/dev/null +fi + +TMP_DIR="$(mktemp -d -t rustfs-vault-ha-live.XXXXXX)" +RUN_ID="rustfs-kms-ha-$$-${RANDOM}" +NETWORK="${RUN_ID}" +MARKER="${TMP_DIR}/failover-ready" +ROOT_TOKEN="" +UNSEAL_KEY="" +TEST_PID="" +declare -a NODES=("${RUN_ID}-1" "${RUN_ID}-2" "${RUN_ID}-3") + +cleanup() { + if [[ -n "$TEST_PID" ]]; then + kill "$TEST_PID" 2>/dev/null || true + wait "$TEST_PID" 2>/dev/null || true + fi + for node in "${NODES[@]}"; do + docker rm -f "$node" >/dev/null 2>&1 || true + done + docker network rm "$NETWORK" >/dev/null 2>&1 || true + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +docker network create "$NETWORK" >/dev/null + +start_node() { + local index="$1" + local node="${NODES[$((index - 1))]}" + local config + config="$(jq -cn \ + --arg api "http://${node}:8200" \ + --arg cluster "http://${node}:8201" \ + --arg node_id "node${index}" \ + '{ + ui: false, + disable_mlock: true, + api_addr: $api, + cluster_addr: $cluster, + listener: [{tcp: { + address: "0.0.0.0:8200", + cluster_address: "0.0.0.0:8201", + tls_disable: true + }}], + storage: {raft: {path: "/vault/file", node_id: $node_id, performance_multiplier: 1}} + }')" + docker run -d \ + --name "$node" \ + --network "$NETWORK" \ + --cap-add IPC_LOCK \ + -p 127.0.0.1::8200 \ + -e VAULT_LOCAL_CONFIG="$config" \ + "$VAULT_IMAGE" server >/dev/null +} + +vault_exec() { + local node="$1" + shift + docker exec \ + -e VAULT_ADDR=http://127.0.0.1:8200 \ + -e VAULT_TOKEN="$ROOT_TOKEN" \ + "$node" vault "$@" +} + +wait_started() { + local node="$1" + local status + for _ in $(seq 1 120); do + status="$(docker exec -e VAULT_ADDR=http://127.0.0.1:8200 "$node" vault status -format=json 2>/dev/null || true)" + if jq -e '.initialized == false or .sealed == true or .sealed == false' >/dev/null 2>&1 <<<"$status"; then + return 0 + fi + sleep 0.25 + done + echo "$node did not start" >&2 + docker logs "$node" >&2 || true + return 1 +} + +active_node() { + local node status address + for node in "${NODES[@]}"; do + if ! docker inspect "$node" >/dev/null 2>&1; then + continue + fi + status="$(docker exec -e VAULT_ADDR=http://127.0.0.1:8200 "$node" vault status -format=json 2>/dev/null || true)" + address="http://${node}:8200" + if jq -e --arg address "$address" \ + '.ha_enabled == true and .sealed == false and (.is_self == true or (.leader_address == $address and .active_time != "0001-01-01T00:00:00Z"))' \ + >/dev/null 2>&1 <<<"$status"; then + printf '%s\n' "$node" + return 0 + fi + done + return 1 +} + +host_address() { + local node="$1" + local port + port="$(docker port "$node" 8200/tcp | awk -F: 'NR == 1 {print $NF}')" + if [[ ! "$port" =~ ^[0-9]+$ ]]; then + echo "failed to resolve host port for $node" >&2 + return 1 + fi + printf 'http://127.0.0.1:%s\n' "$port" +} + +for index in 1 2 3; do + start_node "$index" + wait_started "${NODES[$((index - 1))]}" +done + +INIT_JSON="$(docker exec -e VAULT_ADDR=http://127.0.0.1:8200 "${NODES[0]}" vault operator init -format=json -key-shares=1 -key-threshold=1)" +ROOT_TOKEN="$(jq -r '.root_token' <<<"$INIT_JSON")" +UNSEAL_KEY="$(jq -r '.unseal_keys_b64[0]' <<<"$INIT_JSON")" +if [[ -z "$ROOT_TOKEN" || -z "$UNSEAL_KEY" || "$ROOT_TOKEN" == null || "$UNSEAL_KEY" == null ]]; then + echo "Vault initialization did not return the expected credentials" >&2 + exit 1 +fi + +vault_exec "${NODES[0]}" operator unseal "$UNSEAL_KEY" >/dev/null +for node in "${NODES[@]:1}"; do + vault_exec "$node" operator raft join "http://${NODES[0]}:8200" >/dev/null + vault_exec "$node" operator unseal "$UNSEAL_KEY" >/dev/null +done + +for _ in $(seq 1 240); do + if vault_exec "${NODES[0]}" operator raft list-peers -format=json 2>/dev/null \ + | jq -e '.data.config.servers | length == 3 and all(.[]; .voter == true)' >/dev/null; then + break + fi + sleep 0.25 +done +if ! vault_exec "${NODES[0]}" operator raft list-peers -format=json \ + | jq -e '.data.config.servers | length == 3 and all(.[]; .voter == true)' >/dev/null; then + echo "Vault Raft cluster did not stabilize with three voters" >&2 + vault_exec "${NODES[0]}" operator raft list-peers -format=json >&2 || true + exit 1 +fi + +OLD_LEADER="$(active_node)" +if [[ -z "$OLD_LEADER" ]]; then + echo "Vault did not elect an active node" >&2 + exit 1 +fi +STANDBY="" +for node in "${NODES[@]}"; do + if [[ "$node" != "$OLD_LEADER" ]]; then + STANDBY="$node" + break + fi +done +VAULT_ADDR="$(host_address "$STANDBY")" + +vault_exec "$OLD_LEADER" audit enable file file_path=/tmp/vault-audit.log >/dev/null +vault_exec "$OLD_LEADER" secrets enable -path=secret kv-v2 >/dev/null +vault_exec "$OLD_LEADER" secrets enable -path=transit transit >/dev/null +vault_exec "$OLD_LEADER" auth enable approle >/dev/null + +POLICY_NAME="rustfs-kms-ha" +ROLE_NAME="rustfs-kms-ha" +POLICY_FILE="${TMP_DIR}/kms-policy.hcl" +cat >"$POLICY_FILE" <<'EOF' +path "secret/data/rustfs/kms/ha-kv2/*" { + capabilities = ["create", "read", "update"] +} +path "secret/metadata/rustfs/kms/ha-kv2/*" { + capabilities = ["list", "read", "delete"] +} +path "secret/metadata/rustfs/kms/ha-kv2" { + capabilities = ["list"] +} +path "secret/data/rustfs/kms/ha-transit-metadata/*" { + capabilities = ["create", "read", "update"] +} +path "secret/metadata/rustfs/kms/ha-transit-metadata/*" { + capabilities = ["list", "read", "delete"] +} +path "secret/metadata/rustfs/kms/ha-transit-metadata" { + capabilities = ["list"] +} +path "transit/keys" { + capabilities = ["list"] +} +path "transit/keys/*" { + capabilities = ["create", "read", "update"] +} +path "transit/encrypt/*" { + capabilities = ["update"] +} +path "transit/decrypt/*" { + capabilities = ["update"] +} +EOF +docker cp "$POLICY_FILE" "${OLD_LEADER}:/tmp/kms-policy.hcl" +vault_exec "$OLD_LEADER" policy write "$POLICY_NAME" /tmp/kms-policy.hcl >/dev/null +vault_exec "$OLD_LEADER" write "auth/approle/role/${ROLE_NAME}" \ + token_policies="$POLICY_NAME" \ + token_ttl=10m \ + token_max_ttl=20m \ + secret_id_ttl=30m \ + secret_id_num_uses=0 \ + >/dev/null +ROLE_ID="$(vault_exec "$OLD_LEADER" read -field=role_id "auth/approle/role/${ROLE_NAME}/role-id")" +SECRET_ID="$(vault_exec "$OLD_LEADER" write -f -field=secret_id "auth/approle/role/${ROLE_NAME}/secret-id")" + +cd "$PROJECT_ROOT" +cargo test -p rustfs-kms --test vault_ha_failover_live --no-run +env \ + -u RUSTFS_KMS_VAULT_TOKEN \ + -u RUSTFS_KMS_VAULT_TOKEN_FILE \ + HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ + NO_PROXY=127.0.0.1,localhost no_proxy=127.0.0.1,localhost \ + RUSTFS_TEST_VAULT_ADDRESS="$VAULT_ADDR" \ + RUSTFS_TEST_VAULT_ROLE_ID="$ROLE_ID" \ + RUSTFS_TEST_VAULT_SECRET_ID="$SECRET_ID" \ + RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \ + RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \ + cargo test -p rustfs-kms --test vault_ha_failover_live \ + vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \ + --ignored --nocapture --test-threads=1 & +TEST_PID=$! + +for _ in $(seq 1 240); do + if [[ -f "$MARKER" ]]; then + break + fi + if ! kill -0 "$TEST_PID" 2>/dev/null; then + wait "$TEST_PID" + fi + sleep 0.25 +done +if [[ ! -f "$MARKER" ]]; then + echo "KMS live test did not reach failover readiness" >&2 + exit 1 +fi + +docker kill "$OLD_LEADER" >/dev/null +NEW_LEADER="" +for _ in $(seq 1 240); do + NEW_LEADER="$(active_node || true)" + if [[ -n "$NEW_LEADER" && "$NEW_LEADER" != "$OLD_LEADER" ]]; then + break + fi + sleep 0.25 +done +if [[ -z "$NEW_LEADER" || "$NEW_LEADER" == "$OLD_LEADER" ]]; then + echo "Vault did not elect a replacement leader after killing $OLD_LEADER" >&2 + exit 1 +fi +printf '%s' "$NEW_LEADER" >"${MARKER%/*}/failover-ready.elected" + +wait "$TEST_PID" +TEST_PID="" + +for index in 1 2 3; do + docker cp "${NODES[$((index - 1))]}:/tmp/vault-audit.log" "${TMP_DIR}/vault-audit-${index}.log" >/dev/null 2>&1 || true +done +APPROLE_LOGINS="$(jq -s '[.[] | select(.type == "request" and .request.path == "auth/approle/login" and .request.operation == "update")] | length' \ + "${TMP_DIR}"/vault-audit-*.log 2>/dev/null || true)" +if [[ "$APPROLE_LOGINS" != 2 ]]; then + echo "expected exactly two AppRole logins (one per backend), got ${APPROLE_LOGINS:-unavailable}" >&2 + exit 1 +fi + +echo "Vault HA failover passed: 3 Raft voters, ${OLD_LEADER} killed, ${NEW_LEADER} elected, 2 AppRole logins"