rustfs/crates/kms/tests/behavior_lifecycle.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Black-box behavior: KMS service lifecycle, configuration, and redaction.
//!
//! Pins the contract the admin API depends on:
//!
//! * an unconfigured manager hands out nothing and refuses to start;
//! * `configure` is for the stopped state only — changing a running service is
//!   `reconfigure`, which never exposes a stopped interval;
//! * a configuration is validated *before* anything is published, and a failing
//!   persistence callback leaves the previous state completely intact;
//! * the Local backend's identity (directory, master key, dev-mode) is frozen
//!   once configured, because changing it silently orphans existing key files;
//! * redacted views never carry key material.

mod common;

use std::sync::Arc;
use std::time::Duration;

use common::{STATIC_KEY_ID, TestKms, assert_configuration_error, assert_no_secret_leak, static_secret_key};
use rustfs_kms::{BackendConfig, KmsConfig, KmsError, KmsServiceManager, KmsServiceStatus, KmsStartOutcome, LocalConfig};
use tempfile::TempDir;
use url::Url;

fn local_dev_config(dir: &TempDir) -> KmsConfig {
    KmsConfig::local(dir.path().to_path_buf()).with_insecure_development_defaults()
}

#[tokio::test]
async fn unconfigured_manager_exposes_nothing_and_refuses_to_start() {
    let manager = KmsServiceManager::new();

    assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
    assert!(manager.get_config().await.is_none(), "no config before configure");
    assert!(manager.get_redacted_config().await.is_none(), "no redacted config either");
    assert!(manager.get_manager().await.is_none(), "no manager handle before start");
    assert!(manager.get_encryption_service().await.is_none(), "no service handle before start");
    assert!(manager.get_service_version().await.is_none(), "no version before start");

    // A health check on a service that was never started is a definite "not
    // healthy", not an error: callers use it as a readiness probe.
    assert!(
        !manager.health_check().await.expect("health check must not error when idle"),
        "an unstarted service must report unhealthy"
    );

    assert_configuration_error(manager.start().await, "no configuration provided");

    // The failed start records the reason and still exposes no service.
    match manager.get_status().await {
        KmsServiceStatus::Error(message) => {
            assert!(message.contains("no configuration"), "error status should explain why: {message}")
        }
        other => panic!("expected Error status after starting without config, got {other:?}"),
    }
    assert!(manager.get_encryption_service().await.is_none());
    assert!(manager.get_service_version().await.is_none());
}

#[tokio::test]
async fn configure_start_stop_restart_state_machine() {
    let dir = TempDir::new().expect("temp dir");
    let manager = KmsServiceManager::new();
    let config = local_dev_config(&dir);

    manager.configure(config.clone()).await.expect("configure should succeed");
    assert_eq!(manager.get_status().await, KmsServiceStatus::Configured);
    assert!(
        manager.get_encryption_service().await.is_none(),
        "configured but not started must not expose a service"
    );
    assert!(manager.get_config().await.is_some(), "configure publishes the config");

    manager.start().await.expect("start should succeed");
    assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
    assert_eq!(manager.get_service_version().await, Some(1), "first start is version 1");
    assert!(manager.get_manager().await.is_some());
    assert!(manager.get_encryption_service().await.is_some());
    assert!(manager.health_check().await.expect("health check"), "a started local KMS is healthy");

    manager.stop().await.expect("stop should succeed");
    assert_eq!(
        manager.get_status().await,
        KmsServiceStatus::Configured,
        "stop falls back to Configured, not NotConfigured, because the config survives"
    );
    assert!(manager.get_encryption_service().await.is_none(), "stop withdraws the service");
    assert!(manager.get_service_version().await.is_none(), "stop clears the published version");
    assert!(manager.get_config().await.is_some(), "stop keeps the configuration");
    assert!(!manager.health_check().await.expect("health check"), "a stopped service is unhealthy");

    // Restart after stop reuses the surviving config and takes the next version.
    manager.start().await.expect("restart after stop should succeed");
    assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
    assert_eq!(
        manager.get_service_version().await,
        Some(2),
        "the version counter is monotonic across a stop/start cycle"
    );
}

#[tokio::test]
async fn start_or_restart_reports_the_action_it_took() {
    let dir = TempDir::new().expect("temp dir");
    let manager = KmsServiceManager::new();
    manager
        .configure(local_dev_config(&dir))
        .await
        .expect("configure should succeed");

    assert_eq!(
        manager.start_or_restart(false).await.expect("first start"),
        KmsStartOutcome::Started,
        "starting a stopped service reports Started"
    );
    assert_eq!(manager.get_service_version().await, Some(1));

    let service_v1 = manager.get_encryption_service().await.expect("service after start");

    assert_eq!(
        manager.start_or_restart(false).await.expect("non-forced start while running"),
        KmsStartOutcome::AlreadyRunning,
        "a non-forced start on a running service is a no-op"
    );
    assert_eq!(manager.get_service_version().await, Some(1), "the no-op path must not consume a version");
    assert!(
        Arc::ptr_eq(&service_v1, &manager.get_encryption_service().await.expect("service still running")),
        "a no-op start must not swap the live service instance"
    );

    assert_eq!(
        manager.start_or_restart(true).await.expect("forced restart"),
        KmsStartOutcome::Restarted,
        "a forced start on a running service reports Restarted"
    );
    assert_eq!(manager.get_service_version().await, Some(2), "a forced restart takes a new version");
    assert!(
        !Arc::ptr_eq(&service_v1, &manager.get_encryption_service().await.expect("service after restart")),
        "a forced restart must publish a different instance"
    );

    // The withdrawn instance keeps working for operations that already hold it.
    assert!(
        service_v1.health_check().await.expect("old handle health check"),
        "a replaced service stays usable while callers still hold it"
    );
}

#[tokio::test]
async fn configure_is_rejected_while_running() {
    let dir = TempDir::new().expect("temp dir");
    let manager = KmsServiceManager::new();
    let config = local_dev_config(&dir);
    manager.configure(config.clone()).await.expect("configure");
    manager.start().await.expect("start");

    assert_configuration_error(manager.configure(config.clone()).await, "use reconfigure instead");

    // The rejected call must not have disturbed the running service.
    assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
    assert_eq!(manager.get_service_version().await, Some(1), "a rejected configure takes no version");
}

#[tokio::test]
async fn reconfigure_swaps_the_service_without_a_stopped_interval() {
    let dir = TempDir::new().expect("temp dir");
    let manager = KmsServiceManager::new();
    let config = local_dev_config(&dir);
    manager.configure(config.clone()).await.expect("configure");
    manager.start().await.expect("start");

    let service_v1 = manager.get_encryption_service().await.expect("service v1");
    let manager_v1 = manager.get_manager().await.expect("manager v1");

    let mut next = config.clone();
    next.timeout = Duration::from_secs(45);
    manager.reconfigure(next).await.expect("reconfigure should succeed");

    assert_eq!(manager.get_status().await, KmsServiceStatus::Running, "reconfigure never reports stopped");
    assert_eq!(manager.get_service_version().await, Some(2));
    assert_eq!(
        manager.get_config().await.expect("config after reconfigure").timeout,
        Duration::from_secs(45),
        "the published config must be the new one"
    );

    let service_v2 = manager.get_encryption_service().await.expect("service v2");
    let manager_v2 = manager.get_manager().await.expect("manager v2");
    assert!(!Arc::ptr_eq(&service_v1, &service_v2), "reconfigure publishes a new service");
    assert!(!Arc::ptr_eq(&manager_v1, &manager_v2), "reconfigure publishes a new manager");

    // Both generations stay functional: that is the whole point of the swap.
    assert!(service_v1.health_check().await.expect("v1 health"), "old generation still serves");
    assert!(service_v2.health_check().await.expect("v2 health"), "new generation serves");
}

#[tokio::test]
async fn failed_persistence_does_not_publish_the_new_configuration() {
    let dir = TempDir::new().expect("temp dir");
    let manager = KmsServiceManager::new();

    // configure_with_persistence: nothing is published when persistence fails.
    let result = manager
        .configure_with_persistence(local_dev_config(&dir), || async {
            Err(KmsError::backend_error("simulated persistence failure"))
        })
        .await;
    assert!(
        matches!(result, Err(KmsError::BackendError { .. })),
        "the persistence error must propagate verbatim, got {result:?}"
    );
    assert_eq!(
        manager.get_status().await,
        KmsServiceStatus::NotConfigured,
        "a failed persist must leave the manager unconfigured"
    );
    assert!(manager.get_config().await.is_none(), "a failed persist must publish no config");

    // Now bring it up for real, then fail persistence on a reconfigure.
    let config = local_dev_config(&dir);
    manager.configure(config.clone()).await.expect("configure");
    manager.start().await.expect("start");
    let service_v1 = manager.get_encryption_service().await.expect("service v1");

    let mut next = config.clone();
    next.timeout = Duration::from_secs(99);
    let result = manager
        .reconfigure_with_persistence(next, || async { Err(KmsError::backend_error("simulated persistence failure")) })
        .await;
    assert!(result.is_err(), "reconfigure must fail when persistence fails, got {result:?}");

    assert_eq!(manager.get_status().await, KmsServiceStatus::Running, "the old service keeps running");
    assert_eq!(
        manager.get_config().await.expect("config").timeout,
        config.timeout,
        "the old configuration must still be the published one"
    );
    assert!(
        Arc::ptr_eq(&service_v1, &manager.get_encryption_service().await.expect("service")),
        "a failed reconfigure must not swap the live service"
    );
}

#[tokio::test]
async fn local_backend_identity_is_frozen_after_configuration() {
    let dir = TempDir::new().expect("temp dir");
    let other_dir = TempDir::new().expect("other temp dir");
    let manager = KmsServiceManager::new();
    let config = local_dev_config(&dir);
    manager.configure(config.clone()).await.expect("configure");
    manager.start().await.expect("start");

    // Moving the key directory would orphan every existing key file.
    let mut moved = config.clone();
    moved.backend_config = BackendConfig::Local(LocalConfig {
        key_dir: other_dir.path().to_path_buf(),
        master_key: None,
        file_permissions: Some(0o600),
    });
    assert_configuration_error(manager.reconfigure(moved).await, "key directory cannot be changed");

    // Changing the at-rest master key would make stored material undecryptable.
    let mut rekeyed = config.clone();
    rekeyed.backend_config = BackendConfig::Local(LocalConfig {
        key_dir: dir.path().to_path_buf(),
        master_key: Some("a-different-master-key".to_string()),
        file_permissions: Some(0o600),
    });
    assert_configuration_error(manager.reconfigure(rekeyed).await, "master key cannot be changed");

    // Flipping dev mode changes the at-rest protection of the same directory.
    let mut hardened = config.clone();
    hardened.allow_insecure_dev_defaults = false;
    hardened.backend_config = BackendConfig::Local(LocalConfig {
        key_dir: dir.path().to_path_buf(),
        master_key: Some("a-master-key".to_string()),
        file_permissions: Some(0o600),
    });
    assert!(
        manager.reconfigure(hardened).await.is_err(),
        "flipping development mode on a configured local backend must be refused"
    );

    // Switching away from Local entirely is refused for the same reason.
    let switched = KmsConfig::static_kms(STATIC_KEY_ID.to_string(), static_secret_key());
    assert_configuration_error(manager.reconfigure(switched).await, "backend cannot be changed");

    // Every rejection above left the original service untouched.
    assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
    assert_eq!(
        manager.get_service_version().await,
        Some(1),
        "rejected transitions must not consume service versions"
    );
}

#[tokio::test]
async fn invalid_configurations_are_rejected_before_anything_starts() {
    let dir = TempDir::new().expect("temp dir");

    // A zero timeout or zero retry budget would make every operation fail.
    let mut zero_timeout = local_dev_config(&dir);
    zero_timeout.timeout = Duration::ZERO;
    assert_configuration_error(zero_timeout.validate(), "Timeout must be greater than 0");

    let mut zero_retries = local_dev_config(&dir);
    zero_retries.retry_attempts = 0;
    assert_configuration_error(zero_retries.validate(), "Retry attempts must be greater than 0");

    // Cache enabled with no capacity is contradictory.
    let mut empty_cache = local_dev_config(&dir);
    empty_cache.enable_cache = true;
    empty_cache.cache_config.max_keys = 0;
    assert_configuration_error(empty_cache.validate(), "max_keys must be greater than 0");

    // A relative key directory is ambiguous relative to the server's cwd.
    let mut relative = local_dev_config(&dir);
    relative.backend_config = BackendConfig::Local(LocalConfig {
        key_dir: "relative/kms".into(),
        master_key: None,
        file_permissions: Some(0o600),
    });
    assert_configuration_error(relative.validate(), "must be an absolute path");

    // Production-shaped Local config: no master key is refused without opt-in.
    let production_without_master_key = KmsConfig::local("/var/lib/rustfs/kms-behavior".into());
    assert_configuration_error(production_without_master_key.validate(), "requires a master key");

    // ...and so is a key directory under the process temp dir.
    let temp_backed = KmsConfig {
        backend_config: BackendConfig::Local(LocalConfig {
            key_dir: dir.path().to_path_buf(),
            master_key: Some("a-master-key".to_string()),
            file_permissions: Some(0o600),
        }),
        ..KmsConfig::local(dir.path().to_path_buf())
    };
    assert_configuration_error(temp_backed.validate(), "temp directory");

    // Static backend: the secret must be base64 and exactly 32 bytes.
    assert_configuration_error(
        KmsConfig::static_kms("k".to_string(), "not base64!!".to_string()).validate(),
        "not valid base64",
    );
    assert_configuration_error(
        KmsConfig::static_kms("k".to_string(), base64_of(&[0u8; 16])).validate(),
        "exactly 32 bytes",
    );
    assert_configuration_error(
        KmsConfig::static_kms(String::new(), static_secret_key()).validate(),
        "key_id cannot be empty",
    );
    assert_configuration_error(
        KmsConfig::static_kms("k".to_string(), String::new()).validate(),
        "secret_key cannot be empty",
    );

    // Vault: plaintext HTTP and the built-in dev token are dev-only.
    let http = Url::parse("http://127.0.0.1:8200").expect("url");
    assert_configuration_error(KmsConfig::vault(http.clone(), "a-real-token".to_string()).validate(), "requires HTTPS");
    let https = Url::parse("https://vault.example.com:8200").expect("url");
    assert_configuration_error(
        KmsConfig::vault(https.clone(), "dev-token".to_string()).validate(),
        "dev-token is not allowed",
    );
    // An AppRole with no credential at all cannot authenticate.
    let approle = KmsConfig::vault_approle(https, "role".to_string(), String::new());
    assert_configuration_error(approle.validate(), "requires a secret_id");

    // None of the above may be startable through the manager either.
    let manager = KmsServiceManager::new();
    assert_configuration_error(manager.configure(zero_timeout).await, "Timeout must be greater than 0");
    assert_eq!(
        manager.get_status().await,
        KmsServiceStatus::NotConfigured,
        "a rejected configure must not move the manager out of NotConfigured"
    );
}

#[tokio::test]
async fn redacted_views_never_carry_key_material() {
    let secret = static_secret_key();
    let manager = KmsServiceManager::new();
    manager
        .configure(KmsConfig::static_kms(STATIC_KEY_ID.to_string(), secret.clone()))
        .await
        .expect("configure static");
    manager.start().await.expect("start static");

    let redacted = manager.get_redacted_config().await.expect("redacted config");
    let static_config = redacted.static_config().expect("static config");
    assert!(static_config.secret_key.is_empty(), "the redacted view must zero the secret key");
    assert_eq!(static_config.key_id, STATIC_KEY_ID, "the key id is not secret and must survive");

    let (status, from_state) = manager.get_redacted_state().await;
    assert_eq!(status, KmsServiceStatus::Running, "redacted state carries the live status");
    let from_state = from_state.expect("redacted state config");
    assert!(
        from_state.static_config().expect("static config").secret_key.is_empty(),
        "get_redacted_state must redact exactly like get_redacted_config"
    );

    // Neither the Debug rendering nor a JSON serialization may carry material.
    assert_no_secret_leak(&format!("{redacted:?}"), &[&secret]);
    assert_no_secret_leak(&serde_json::to_string(&redacted).expect("redacted config serializes"), &[&secret]);

    // Even the unredacted config must keep the secret out of Debug and JSON:
    // `Debug` is what ends up in logs, and `secret_key` is `skip_serializing`.
    let live = manager.get_config().await.expect("live config");
    assert_no_secret_leak(&format!("{live:?}"), &[&secret]);
    assert_no_secret_leak(&serde_json::to_string(&live).expect("live config serializes"), &[&secret]);

    // A Vault token is subject to the same rule in Debug output.
    let vault = KmsConfig::vault(
        Url::parse("https://vault.example.com:8200").expect("url"),
        "super-secret-vault-token".to_string(),
    );
    assert_no_secret_leak(&format!("{vault:?}"), &["super-secret-vault-token"]);
}

#[tokio::test]
async fn harness_restart_brings_the_service_back_over_the_same_state() {
    let mut kms = TestKms::local().await;
    assert_eq!(kms.manager().get_service_version().await, Some(1));

    kms.restart().await;

    assert_eq!(kms.manager().get_status().await, KmsServiceStatus::Running);
    assert_eq!(
        kms.manager().get_service_version().await,
        Some(1),
        "a restarted process starts its version counter over"
    );
    assert!(
        kms.manager().health_check().await.expect("health check"),
        "the restarted service is healthy"
    );
}

fn base64_of(bytes: &[u8]) -> String {
    use base64::Engine as _;
    base64::engine::general_purpose::STANDARD.encode(bytes)
}