feat(kms): add Vault Agent token file authentication

Add the TokenFile source: the token is read from an agent-managed sink
file (RUSTFS_KMS_VAULT_TOKEN_FILE or the TokenFile auth config) and
re-read once per poll interval (default 30s) through the existing
renewal loop, so a token rotated by the agent installs a new client
generation within one poll of the atomic replace. Each successful read
extends the token's observed validity to twice the poll interval; a
file that disappears or turns empty keeps failing the refresh until the
fail-closed window trips, and heals the provider as soon as it is
restored.

Reads are strict and never contact Vault on failure: the file must be
non-empty after trimming, and on Unix group/other permission bits are a
hard error (mirroring the SFTP host-key rule). Rotation detection uses
a content digest; the token itself is never stored on the source and
the crate-owned copy is zeroized.

Configuring the token file together with AppRole or an explicit static
token is rejected as a configuration error. All new config fields are
serde(default) and the strict admin-configure deserializer accepts the
new variant.

Covered by paused-clock tests (atomic replacement installs a new
generation next cycle, deletion fails closed and recovers, prompt task
recycling) plus negatives for missing/empty/over-permissive files and a
Debug leak regression.
This commit is contained in:
overtrue
2026-07-31 01:29:46 +08:00
parent 124b32742c
commit ee1aee0248
3 changed files with 476 additions and 3 deletions
+18
View File
@@ -249,6 +249,13 @@ enum StrictVaultAuthMethod {
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
TokenFile {
path: std::path::PathBuf,
#[serde(default)]
poll_interval_secs: Option<u64>,
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
}
impl From<StrictVaultAuthMethod> for VaultAuthMethod {
@@ -268,6 +275,15 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
refresh_safety_window_secs,
},
StrictVaultAuthMethod::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
} => Self::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
},
}
}
}
@@ -427,6 +443,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
},
has_stored_credentials: true,
namespace: vault_config.namespace.clone(),
@@ -440,6 +457,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
},
has_stored_credentials: true,
namespace: vault_config.namespace.clone(),
+304 -3
View File
@@ -35,9 +35,10 @@ use std::time::Duration;
use arc_swap::ArcSwap;
use async_trait::async_trait;
use sha2::Digest;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::warn;
use tracing::{info, warn};
use vaultrs::client::{VaultClient, VaultClientSettingsBuilder};
use zeroize::{Zeroize, ZeroizeOnDrop};
@@ -53,6 +54,9 @@ type AttemptResult<T> = std::result::Result<T, AttemptError>;
/// retries inside one [`policy::execute`] call).
const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5);
/// Default seconds between token file re-reads for [`TokenFileSource`].
const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30;
/// A crate-owned secret value, zeroized on drop and redacted in Debug output.
#[derive(Clone, Zeroize, ZeroizeOnDrop)]
pub(crate) struct SecretString(String);
@@ -147,8 +151,8 @@ fn attempt_error(operation: &str, error: vaultrs::error::ClientError) -> Attempt
///
/// Implementations perform one attempt per call; bounded retries and backoff
/// are owned by the caller through [`policy::execute`] with [`OpClass::Auth`].
/// Current sources are [`StaticToken`] and [`AppRoleLogin`]; an agent-managed
/// `TokenFile` source is planned as a follow-up.
/// Current sources are [`StaticToken`], [`AppRoleLogin`], and the agent-managed
/// [`TokenFileSource`].
#[async_trait]
pub(crate) trait TokenSource: fmt::Debug + Send + Sync {
/// One login attempt yielding a token for a new client generation.
@@ -286,6 +290,124 @@ impl fmt::Debug for AppRoleLogin {
}
}
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
/// token file (for example a Vault Agent auto-auth sink).
///
/// The agent owns the token's lifecycle; this source only tracks the file.
/// Every read grants the token an observed validity of `validity`, so the
/// renewal loop re-reads the file at half that interval and installs a new
/// client generation from whatever the file holds. A file that disappears or
/// turns empty keeps failing the refresh until the fail-closed window trips,
/// and heals the provider as soon as it is restored.
pub(crate) struct TokenFileSource {
path: PathBuf,
/// Observed validity granted per successful read; the renewal loop
/// re-reads at half this value.
validity: Duration,
/// Digest of the last token read, to tell agent-driven rotations apart
/// from plain refreshes in the logs. Never holds the token itself.
last_digest: std::sync::Mutex<Option<[u8; 32]>>,
}
impl TokenFileSource {
pub(crate) fn new(path: PathBuf, validity: Duration) -> Self {
Self {
path,
validity,
last_digest: std::sync::Mutex::new(None),
}
}
fn fatal(error: KmsError) -> AttemptError {
AttemptError {
class: ErrorClass::Fatal,
error,
}
}
/// Reject a token file readable by group or other, mirroring the SFTP
/// host-key rule: a Vault token grants use of the KMS keys, so a mode
/// wider than owner-only is a deployment error, not a warning.
#[cfg(unix)]
fn check_permissions(&self, metadata: &std::fs::Metadata) -> AttemptResult<()> {
use std::os::unix::fs::PermissionsExt;
let mode = metadata.permissions().mode() & 0o777;
if mode & 0o077 != 0 {
return Err(Self::fatal(KmsError::configuration_error(format!(
"Vault token file {} has insecure permissions {mode:#o} (group and other permission bits must be unset)",
self.path.display()
))));
}
Ok(())
}
}
#[async_trait]
impl TokenSource for TokenFileSource {
async fn acquire(&self) -> AttemptResult<TokenLease> {
// Synchronous reads on purpose: the token file is tiny, this runs at
// the renewal cadence, and staying off the blocking pool keeps the
// paused-clock tests of the renewal timing deterministic.
let metadata = std::fs::metadata(&self.path).map_err(|error| {
Self::fatal(KmsError::configuration_error(format!(
"Failed to read Vault token file {}: {error}",
self.path.display()
)))
})?;
#[cfg(unix)]
self.check_permissions(&metadata)?;
let mut raw = std::fs::read_to_string(&self.path).map_err(|error| {
Self::fatal(KmsError::configuration_error(format!(
"Failed to read Vault token file {}: {error}",
self.path.display()
)))
})?;
let trimmed = raw.trim();
if trimmed.is_empty() {
raw.zeroize();
return Err(Self::fatal(KmsError::configuration_error(format!(
"Vault token file {} is empty",
self.path.display()
))));
}
let digest: [u8; 32] = sha2::Sha256::digest(trimmed.as_bytes()).into();
let previous = self
.last_digest
.lock()
.expect("token file digest mutex poisoned")
.replace(digest);
if previous.is_some_and(|previous| previous != digest) {
info!(
path = %self.path.display(),
modified = ?metadata.modified().ok(),
"Vault token file rotated; installing a new client generation"
);
}
let token = trimmed.to_string();
raw.zeroize();
Ok(TokenLease::new(
token,
Some(LeaseInfo {
ttl: self.validity,
renewable: false,
}),
))
}
}
impl fmt::Debug for TokenFileSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// The token itself is never stored on the source; only its digest is.
f.debug_struct("TokenFileSource")
.field("path", &self.path)
.field("validity", &self.validity)
.finish_non_exhaustive()
}
}
/// Map the configured auth method onto a token source.
pub(crate) fn token_source_for(
auth_method: &VaultAuthMethod,
@@ -306,6 +428,18 @@ pub(crate) fn token_source_for(
secret_id.clone(),
secret_id_file.clone(),
)?)),
VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
..
} => {
let poll_interval = Duration::from_secs(poll_interval_secs.unwrap_or(DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS).max(1));
// Validity is twice the poll interval because the renewal loop
// fires at half the lease TTL: the file is then re-read once per
// poll interval, and a file that stops being readable expires the
// token after roughly two missed polls minus the safety window.
Ok(Box::new(TokenFileSource::new(path.clone(), poll_interval * 2)))
}
}
}
@@ -372,6 +506,10 @@ impl VaultCredentialPolicy {
VaultAuthMethod::AppRole {
refresh_safety_window_secs: Some(secs),
..
}
| VaultAuthMethod::TokenFile {
refresh_safety_window_secs: Some(secs),
..
} => Duration::from_secs(*secs),
_ => retry.attempt_timeout,
};
@@ -1053,4 +1191,167 @@ mod tests {
assert!(approle_rendered.contains("leak-test-role-id"), "role_id is not a secret");
assert!(approle_rendered.contains(REDACTED_SECRET));
}
/// Write a token file with owner-only permissions, as a Vault Agent sink
/// would.
fn write_owner_only(path: &std::path::Path, contents: &str) {
std::fs::write(path, contents).expect("write token file");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).expect("chmod token file");
}
}
async fn token_file_provider(path: std::path::PathBuf, policy: VaultCredentialPolicy) -> Arc<VaultCredentialProvider> {
Arc::new(
VaultCredentialProvider::new(test_settings(), Box::new(TokenFileSource::new(path, Duration::from_secs(60))), policy)
.await
.expect("token file provider must build without a live Vault"),
)
}
#[tokio::test]
async fn test_token_file_source_reads_and_trims_token() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("vault-token");
write_owner_only(&path, &format!(" {TEST_TOKEN}\n"));
let source = TokenFileSource::new(path, Duration::from_secs(60));
let lease = source.acquire().await.expect("readable token file must resolve");
assert_eq!(lease.expose(), TEST_TOKEN);
let lease_info = lease.lease_info().expect("token file leases carry an observed validity");
assert_eq!(lease_info.ttl, Duration::from_secs(60));
assert!(!lease_info.renewable, "agent-managed tokens are refreshed by re-reading, not renew-self");
}
#[tokio::test]
async fn test_token_file_missing_fails_fatally() {
let dir = tempfile::tempdir().expect("tempdir");
let source = TokenFileSource::new(dir.path().join("absent-token"), Duration::from_secs(60));
let failure = source.acquire().await.expect_err("missing token file must fail the attempt");
assert_eq!(failure.class, ErrorClass::Fatal);
assert!(matches!(failure.error, KmsError::ConfigurationError { .. }));
}
#[tokio::test]
async fn test_token_file_empty_fails_fatally() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("vault-token");
write_owner_only(&path, " \n\t\n");
let source = TokenFileSource::new(path, Duration::from_secs(60));
let failure = source
.acquire()
.await
.expect_err("effectively empty token file must fail the attempt");
assert_eq!(failure.class, ErrorClass::Fatal);
assert!(failure.error.to_string().contains("is empty"));
}
#[cfg(unix)]
#[tokio::test]
async fn test_token_file_rejects_group_or_other_permission_bits() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("vault-token");
write_owner_only(&path, TEST_TOKEN);
let source = TokenFileSource::new(path.clone(), Duration::from_secs(60));
for mode in [0o640u32, 0o604, 0o622, 0o660] {
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).expect("chmod");
let failure = source
.acquire()
.await
.expect_err("token file readable or writable beyond the owner must be rejected");
assert_eq!(failure.class, ErrorClass::Fatal, "mode {mode:#o}");
let rendered = failure.error.to_string();
assert!(rendered.contains("insecure permissions"), "mode {mode:#o}: {rendered}");
assert!(!rendered.contains(TEST_TOKEN), "permission errors must not echo the token");
}
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).expect("chmod");
source.acquire().await.expect("owner-only token file must resolve");
}
#[tokio::test(start_paused = true)]
async fn test_token_file_replacement_installs_new_generation_next_cycle() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("vault-token");
write_owner_only(&path, "agent-token-v1");
let provider = token_file_provider(path.clone(), test_policy(Duration::from_secs(10), Duration::from_secs(5))).await;
let task = provider.spawn_renewal_task().expect("token file credentials are lease-bound");
tokio::time::sleep(Duration::from_secs(29)).await;
assert_eq!(provider.snapshot().generation, 0, "no re-read before half the observed validity");
// Atomic replacement, as a Vault Agent sink writes: temp file + rename.
let staged = dir.path().join("vault-token.tmp");
write_owner_only(&staged, "agent-token-v2");
std::fs::rename(&staged, &path).expect("atomic replace");
tokio::time::sleep(Duration::from_secs(2)).await;
assert_eq!(
provider.snapshot().generation,
1,
"the poll after a rotation must install a new generation"
);
// A poll without a rotation still installs a fresh generation: each
// successful read re-extends the token's observed validity.
tokio::time::sleep(Duration::from_secs(30)).await;
assert_eq!(provider.snapshot().generation, 2);
task.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn test_token_file_deletion_fails_closed_and_recovers() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("vault-token");
write_owner_only(&path, "agent-token-v1");
let provider = token_file_provider(path.clone(), test_policy(Duration::from_secs(10), Duration::from_secs(5))).await;
let task = provider.spawn_renewal_task().expect("renewal task");
std::fs::remove_file(&path).expect("remove token file");
// Polls at 30s, 35s, ... keep failing; the last read keeps the token
// usable until 50s (60s observed validity minus the 10s safety window).
tokio::time::sleep(Duration::from_secs(49)).await;
provider
.current()
.expect("token outside the safety window must still be served");
tokio::time::sleep(Duration::from_secs(2)).await;
let error = provider
.current()
.expect_err("token inside the safety window must be refused");
assert!(
matches!(error, KmsError::CredentialsUnavailable { .. }),
"expected CredentialsUnavailable, got {error:?}"
);
// Restoring the file heals the provider on the next retry cycle.
write_owner_only(&path, "agent-token-v2");
tokio::time::sleep(Duration::from_secs(6)).await;
let handle = provider.current().expect("provider must recover once the token file is back");
assert!(handle.generation >= 1);
task.shutdown().await;
}
#[tokio::test]
async fn test_token_file_debug_redacts_token() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("vault-token");
write_owner_only(&path, TEST_TOKEN);
let source = TokenFileSource::new(path.clone(), Duration::from_secs(60));
source.acquire().await.expect("token file must resolve");
let rendered = format!("{source:?}");
assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}");
assert!(rendered.contains("vault-token"), "the file path is not a secret");
}
}
+154
View File
@@ -33,6 +33,7 @@ pub const ENV_KMS_VAULT_APPROLE_ROLE_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_ROLE_I
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID";
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
@@ -418,6 +419,22 @@ pub enum VaultAuthMethod {
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
/// the token is read from `path` and re-read periodically so a token
/// rotated by the agent is picked up without a restart.
TokenFile {
path: PathBuf,
/// Seconds between token file re-reads. Each successful read also
/// extends the token's observed validity to twice this value, so a
/// file that stops being readable eventually trips the fail-closed
/// window. Defaults to 30 seconds.
#[serde(default)]
poll_interval_secs: Option<u64>,
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
/// per-attempt timeout.
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
}
impl VaultAuthMethod {
@@ -431,6 +448,15 @@ impl VaultAuthMethod {
refresh_safety_window_secs: None,
}
}
/// Agent-managed token file with the default poll interval.
pub fn token_file(path: PathBuf) -> Self {
Self::TokenFile {
path,
poll_interval_secs: None,
refresh_safety_window_secs: None,
}
}
}
impl fmt::Debug for VaultAuthMethod {
@@ -451,6 +477,16 @@ impl fmt::Debug for VaultAuthMethod {
.field("mount", mount)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
Self::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
} => f
.debug_struct("TokenFile")
.field("path", path)
.field("poll_interval_secs", poll_interval_secs)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
}
}
}
@@ -935,6 +971,23 @@ fn is_under_temp_dir(path: &Path) -> bool {
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
/// file taking precedence. Without a role id the legacy token flow applies.
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
// A token file names one authoritative credential source; combining it
// with another one would leave the effective identity ambiguous, so
// that is a configuration error rather than a precedence rule.
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
)));
}
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
)));
}
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
}
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
return Ok(VaultAuthMethod::Token {
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
@@ -981,6 +1034,21 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
}
Ok(())
}
VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
..
} => {
if path.as_os_str().is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} token file path cannot be empty")));
}
if poll_interval_secs == &Some(0) {
return Err(KmsError::configuration_error(format!(
"{backend_name} token file poll interval must be greater than 0"
)));
}
Ok(())
}
}
}
@@ -1488,6 +1556,92 @@ mod tests {
);
}
#[test]
fn test_from_env_selects_token_file() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")),
(ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
let vault = config.vault_config().expect("vault backend config");
let VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
} = &vault.auth_method
else {
panic!("token file in the environment must select TokenFile auth, got {:?}", vault.auth_method);
};
assert_eq!(path, std::path::Path::new("/run/vault-agent/token"));
assert_eq!(poll_interval_secs, &None);
assert_eq!(refresh_safety_window_secs, &None);
},
);
}
#[test]
fn test_from_env_token_file_is_mutually_exclusive_with_other_auth() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
(ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
],
|| {
let error = KmsConfig::from_env().expect_err("token file combined with approle must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_TOKEN_FILE));
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
},
);
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
(ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")),
("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")),
],
|| {
let error = KmsConfig::from_env().expect_err("token file combined with a static token must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_TOKEN_FILE));
assert!(error.to_string().contains("RUSTFS_KMS_VAULT_TOKEN"));
},
);
}
#[test]
fn test_validate_rejects_bad_token_file_settings() {
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
backend: KmsBackend::VaultKv2,
backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig {
address: "https://vault.example.com:8200".to_string(),
auth_method,
..Default::default()
})),
..Default::default()
};
let error = vault_config(VaultAuthMethod::token_file(PathBuf::new()))
.validate()
.expect_err("empty token file path must be rejected");
assert!(error.to_string().contains("path"));
let error = vault_config(VaultAuthMethod::TokenFile {
path: PathBuf::from("/run/vault-agent/token"),
poll_interval_secs: Some(0),
refresh_safety_window_secs: None,
})
.validate()
.expect_err("zero poll interval must be rejected");
assert!(error.to_string().contains("poll interval"));
vault_config(VaultAuthMethod::token_file(PathBuf::from("/run/vault-agent/token")))
.validate()
.expect("well-formed token file auth must validate");
}
#[test]
fn test_approle_config_deserializes_legacy_shape_with_defaults() {
// Persisted configurations from before the AppRole implementation only