Compare commits

...

3 Commits

Author SHA1 Message Date
唐小鸭 21c2fb42bb refactor(replication): split four oversized hot-path functions into focused helpers
Pure-move decomposition of the four oversized functions flagged by the
replication compatibility review (P1-18), unblocking migration milestone
M2 which requires resyncer moves to stay mechanical:

- resync_bucket (522 lines -> 61-line step sequence): leader lock,
  target resolution, walk/collector/worker spawning, and dispatch loop
  extracted into focused helpers; pure decision helpers (DTO builders,
  HEAD-result classification) separated from IO orchestration.
- replicate_all (411 lines -> 113-line main body): initial target-info
  seeding, read/stat option builders, skip-path notes, target HEAD
  action resolution, and the multipart/single-put payload transport
  extracted as private free functions.
- start_mrf_processor (306 lines -> 46-line spawn body): recovery guard,
  ledger load, per-entry replay (delete/object/metadata), and retained
  entry resolution extracted; retry bookkeeping semantics preserved
  exactly (inner continue-paths push inside helpers, outer Missed push
  stays in the loop).
- apply_iam_item (255 lines -> match dispatch skeleton): one helper per
  IAM item type.

No behavior change: log texts, error paths, event emissions, and metric
counts are byte-identical; existing tests unchanged and green (238
ecstore replication/mrf/resync + 232 rustfs site-replication).
2026-08-17 17:08:45 +08:00
唐小鸭 d091554ffe fix(kms): resolve Vault auth from the environment at startup and add Kubernetes auth (#6095) 2026-08-17 14:12:39 +08:00
唐小鸭 c04ee41cf0 feat(site-replication): drain the retry queue from the reconcile tick (#6131) 2026-08-17 14:12:04 +08:00
14 changed files with 3470 additions and 1249 deletions
@@ -667,6 +667,368 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
Err(EcstoreError::PreconditionFailed)
}
/// Acquires the MRF recovery leader lock for the startup replay.
/// Returns `None` (after logging) when the lock cannot be created or another
/// node is already processing the backlog.
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
let recovery_lock = match storage
.new_ns_lock(
ReplicationMetadataStore::rustfs_meta_bucket(),
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
)
.await
{
Ok(lock) => lock,
Err(error) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to create the MRF recovery leader lock"
);
return None;
}
};
match recovery_lock
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
.await
{
Ok(guard) => Some(guard),
Err(_) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
"Another node is already processing the MRF recovery backlog"
);
None
}
}
}
/// Reads and decodes the on-disk MRF recovery file.
/// Returns `None` when there is nothing to replay: missing file (publishes an
/// empty available summary), read failure, or corrupt data (quarantined).
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
Ok(d) => d,
Err(EcstoreError::ConfigNotFound) => {
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
available: true,
buckets: Vec::new(),
});
return None;
}
Err(e) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %e,
"Failed to load MRF recovery file"
);
return None;
}
};
match decode_mrf_file(&data) {
Ok(v) => Some(v),
Err(e) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %e,
"Failed to decode MRF recovery file — preserving corrupt data"
);
quarantine_mrf_file(storage, &data).await;
None
}
}
}
/// Replays one MRF recovery entry by operation kind.
/// Returns `None` when the entry is skipped entirely (no admission outcome);
/// entries that must be retried later are pushed onto `retry_entries`.
async fn replay_mrf_entry<S: ReplicationStorage>(
entry: &MrfReplicateEntry,
storage: &Arc<S>,
retry_entries: &mut Vec<MrfReplicateEntry>,
) -> Option<ReplicationQueueAdmission> {
match entry.op {
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
replay_mrf_object_entry(entry, storage, retry_entries).await
}
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
}
}
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
/// stale force-delete generations are skipped, and plain deletes are
/// reconstructed as heal deletes.
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
entry: &MrfReplicateEntry,
storage: &Arc<S>,
retry_entries: &mut Vec<MrfReplicateEntry>,
) -> Option<ReplicationQueueAdmission> {
if should_replay_force_delete_intent(entry) {
let operation_id = entry.force_delete_id?;
let delete = force_delete_heal_replication_info(entry, operation_id);
if replicate_delete_with_outcome(delete, storage.clone()).await {
Some(ReplicationQueueAdmission::Queued)
} else {
Some(ReplicationQueueAdmission::Missed)
}
} else if entry.force_delete_id.is_some() {
Some(ReplicationQueueAdmission::Skipped)
} else {
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
}
}
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
DeletedObjectReplicationInfo {
delete_object: ReplicationDeletedObject {
object_name: entry.object.clone(),
force_delete: true,
force_delete_id: Some(operation_id),
force_delete_target_arns: entry.target_arns.clone(),
force_delete_generation: entry.force_delete_generation,
..Default::default()
},
bucket: entry.bucket.clone(),
op_type: ReplicationType::Heal,
event_type: REPLICATE_HEAL_DELETE.to_string(),
..Default::default()
}
}
/// Reconstruct a heal delete and re-queue it. We do NOT call
/// get_object_info here because the delete-marker or version may
/// already be absent from the local store — that is expected.
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
entry: &MrfReplicateEntry,
storage: &Arc<S>,
retry_entries: &mut Vec<MrfReplicateEntry>,
) -> Option<ReplicationQueueAdmission> {
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
let oi = ObjectInfo {
bucket: entry.bucket.clone(),
name: entry.object.clone(),
version_id: entry.version_id,
delete_marker: entry.delete_marker,
..Default::default()
};
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
if replicate_delete_with_outcome(dv, storage.clone()).await {
Some(ReplicationQueueAdmission::Queued)
} else {
Some(ReplicationQueueAdmission::Missed)
}
}
/// The MRF entry does not persist the replication decision and the
/// source object is gone, so re-derive the decision from the live
/// bucket config (mirroring get_heal_replicate_object_info) and set
/// it on the reconstructed delete. Without this the decision string
/// is empty and the delete replicates to zero targets — a silent
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
async fn resolve_mrf_delete_replicate_decision(
entry: &MrfReplicateEntry,
oi: &ObjectInfo,
versioned: bool,
retry_entries: &mut Vec<MrfReplicateEntry>,
) -> Option<ReplicateDecision> {
if entry.target_arns.is_empty() {
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
Ok(None) => None,
Err(_) => {
retry_entries.push(entry.clone());
None
}
Ok(Some(_)) => match check_replicate_delete_strict(
&entry.bucket,
&ObjectToDelete {
object_name: entry.object.clone(),
version_id: entry.version_id,
..Default::default()
},
oi,
&ObjectOptions {
versioned,
..Default::default()
},
None,
)
.await
{
Ok(dsc) => Some(dsc),
Err(_) => {
retry_entries.push(entry.clone());
None
}
},
}
} else {
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
}
}
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
/// replication decision.
fn reconstructed_heal_delete_info(
entry: &MrfReplicateEntry,
oi: &ObjectInfo,
dsc: &ReplicateDecision,
) -> DeletedObjectReplicationInfo {
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let delete_marker_mtime = entry
.delete_marker_mtime
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
DeletedObjectReplicationInfo {
delete_object: ReplicationDeletedObject {
object_name: entry.object.clone(),
version_id: entry.version_id,
delete_marker_version_id: entry.delete_marker_version_id,
delete_marker: entry.delete_marker,
delete_marker_mtime,
force_delete: entry.force_delete,
replication_state: Some(rstate),
..Default::default()
},
bucket: entry.bucket.clone(),
op_type: ReplicationType::Heal,
event_type: REPLICATE_HEAL_DELETE.to_string(),
..Default::default()
}
}
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
async fn replay_mrf_object_entry<S: ReplicationStorage>(
entry: &MrfReplicateEntry,
storage: &Arc<S>,
retry_entries: &mut Vec<MrfReplicateEntry>,
) -> Option<ReplicationQueueAdmission> {
let opts = ObjectOptions {
version_id: entry.version_id.map(|u| u.to_string()),
..Default::default()
};
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
Ok(oi) => oi,
Err(e) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %entry.bucket,
object = %entry.object,
error = %e,
"MRF recovery: source object lookup failed"
);
if should_retry_mrf_source_lookup(&e) {
retry_entries.push(entry.clone());
}
return None;
}
};
if entry.target_arns.is_empty() {
// Legacy entries predate target admission persistence. They cannot
// be safely attributed, so retain the old live-config fallback.
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
} else {
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
Some(ReplicationQueueAdmission::Queued)
} else {
Some(ReplicationQueueAdmission::Missed)
}
}
}
/// Replays a metadata-kind MRF entry against the live source object.
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
entry: &MrfReplicateEntry,
storage: &Arc<S>,
retry_entries: &mut Vec<MrfReplicateEntry>,
) -> Option<ReplicationQueueAdmission> {
let opts = ObjectOptions {
version_id: entry.version_id.map(|u| u.to_string()),
..Default::default()
};
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
Ok(oi) => oi,
Err(e) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %entry.bucket,
object = %entry.object,
error = %e,
"MRF metadata recovery: source object lookup failed"
);
if should_retry_mrf_source_lookup(&e) {
retry_entries.push(entry.clone());
}
return None;
}
};
if entry.target_arns.is_empty() {
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
} else {
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
Some(ReplicationQueueAdmission::Queued)
} else {
Some(ReplicationQueueAdmission::Missed)
}
}
}
/// Pure DTO construction: replicate-object info for an entry with persisted
/// admitted targets, carrying over the entry's retry count.
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
roi.retry_count = entry.retry_count.max(0) as u32;
roi
}
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
/// On acknowledgement failure the backlog is preserved for the next startup and
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
storage: &Arc<S>,
recovery_guard: &rustfs_lock::NamespaceLockGuard,
entries: &[MrfReplicateEntry],
retry_entries: &[MrfReplicateEntry],
) -> Vec<MrfReplicateEntry> {
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
Ok(retained) => retained,
Err(error) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
Err(read_error) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %read_error,
"Failed to refresh the MRF backlog after acknowledgement failure"
);
entries.to_vec()
}
}
}
}
}
#[derive(Debug, thiserror::Error)]
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
struct ResyncActiveConflictError {
@@ -1221,71 +1583,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let storage = self.storage.clone();
let handle = tokio::spawn(async move {
let recovery_lock = match storage
.new_ns_lock(
ReplicationMetadataStore::rustfs_meta_bucket(),
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
)
.await
{
Ok(lock) => lock,
Err(error) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to create the MRF recovery leader lock"
);
return;
}
};
let recovery_guard = match recovery_lock
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
.await
{
Ok(guard) => guard,
Err(_) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
"Another node is already processing the MRF recovery backlog"
);
return;
}
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
return;
};
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
Ok(d) => d,
Err(EcstoreError::ConfigNotFound) => {
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
available: true,
buckets: Vec::new(),
});
return;
}
Err(e) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %e,
"Failed to load MRF recovery file"
);
return;
}
};
let entries = match decode_mrf_file(&data) {
Ok(v) => v,
Err(e) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %e,
"Failed to decode MRF recovery file — preserving corrupt data"
);
quarantine_mrf_file(&storage, &data).await;
return;
}
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
return;
};
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
@@ -1294,187 +1597,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let mut retry_entries = Vec::new();
for entry in entries.iter() {
let admission = match entry.op {
MrfOpKind::Delete => {
if should_replay_force_delete_intent(entry) {
let Some(operation_id) = entry.force_delete_id else {
continue;
};
let delete = DeletedObjectReplicationInfo {
delete_object: ReplicationDeletedObject {
object_name: entry.object.clone(),
force_delete: true,
force_delete_id: Some(operation_id),
force_delete_target_arns: entry.target_arns.clone(),
force_delete_generation: entry.force_delete_generation,
..Default::default()
},
bucket: entry.bucket.clone(),
op_type: ReplicationType::Heal,
event_type: REPLICATE_HEAL_DELETE.to_string(),
..Default::default()
};
if replicate_delete_with_outcome(delete, storage.clone()).await {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
}
} else if entry.force_delete_id.is_some() {
ReplicationQueueAdmission::Skipped
} else {
// Reconstruct a heal delete and re-queue it. We do NOT call
// get_object_info here because the delete-marker or version may
// already be absent from the local store — that is expected.
//
// The MRF entry does not persist the replication decision and the
// source object is gone, so re-derive the decision from the live
// bucket config (mirroring get_heal_replicate_object_info) and set
// it on the reconstructed delete. Without this the decision string
// is empty and the delete replicates to zero targets — a silent
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
let oi = ObjectInfo {
bucket: entry.bucket.clone(),
name: entry.object.clone(),
version_id: entry.version_id,
delete_marker: entry.delete_marker,
..Default::default()
};
let dsc = if entry.target_arns.is_empty() {
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
Ok(None) => continue,
Err(_) => {
retry_entries.push(entry.clone());
continue;
}
Ok(Some(_)) => match check_replicate_delete_strict(
&entry.bucket,
&ObjectToDelete {
object_name: entry.object.clone(),
version_id: entry.version_id,
..Default::default()
},
&oi,
&ObjectOptions {
versioned,
..Default::default()
},
None,
)
.await
{
Ok(dsc) => dsc,
Err(_) => {
retry_entries.push(entry.clone());
continue;
}
},
}
} else {
replicate_decision_for_admitted_targets(&entry.target_arns)
};
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let delete_marker_mtime = entry
.delete_marker_mtime
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
let dv = DeletedObjectReplicationInfo {
delete_object: ReplicationDeletedObject {
object_name: entry.object.clone(),
version_id: entry.version_id,
delete_marker_version_id: entry.delete_marker_version_id,
delete_marker: entry.delete_marker,
delete_marker_mtime,
force_delete: entry.force_delete,
replication_state: Some(rstate),
..Default::default()
},
bucket: entry.bucket.clone(),
op_type: ReplicationType::Heal,
event_type: REPLICATE_HEAL_DELETE.to_string(),
..Default::default()
};
if replicate_delete_with_outcome(dv, storage.clone()).await {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
}
}
}
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
let opts = ObjectOptions {
version_id: entry.version_id.map(|u| u.to_string()),
..Default::default()
};
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
Ok(oi) => oi,
Err(e) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %entry.bucket,
object = %entry.object,
error = %e,
"MRF recovery: source object lookup failed"
);
if should_retry_mrf_source_lookup(&e) {
retry_entries.push(entry.clone());
}
continue;
}
};
if entry.target_arns.is_empty() {
// Legacy entries predate target admission persistence. They cannot
// be safely attributed, so retain the old live-config fallback.
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
} else {
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
roi.retry_count = entry.retry_count.max(0) as u32;
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
}
}
}
MrfOpKind::Metadata => {
let opts = ObjectOptions {
version_id: entry.version_id.map(|u| u.to_string()),
..Default::default()
};
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
Ok(oi) => oi,
Err(e) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %entry.bucket,
object = %entry.object,
error = %e,
"MRF metadata recovery: source object lookup failed"
);
if should_retry_mrf_source_lookup(&e) {
retry_entries.push(entry.clone());
}
continue;
}
};
if entry.target_arns.is_empty() {
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
} else {
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
roi.retry_count = entry.retry_count.max(0) as u32;
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
}
}
}
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
continue;
};
if admission == ReplicationQueueAdmission::Missed {
@@ -1484,29 +1608,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
}
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
Ok(retained) => retained,
Err(error) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %error,
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
);
match read_mrf_entries(storage.clone()).await {
Ok(current) => current,
Err(read_error) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
error = %read_error,
"Failed to refresh the MRF backlog after acknowledgement failure"
);
entries.clone()
}
}
}
};
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
let retained_count = retained.len();
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
File diff suppressed because it is too large Load Diff
+58
View File
@@ -293,6 +293,15 @@ enum StrictVaultAuthMethod {
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
Kubernetes {
role: String,
#[serde(default)]
mount: Option<String>,
#[serde(default)]
jwt_path: Option<std::path::PathBuf>,
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
TokenFile {
path: std::path::PathBuf,
#[serde(default)]
@@ -319,6 +328,17 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
refresh_safety_window_secs,
},
StrictVaultAuthMethod::Kubernetes {
role,
mount,
jwt_path,
refresh_safety_window_secs,
} => Self::Kubernetes {
role,
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
refresh_safety_window_secs,
},
StrictVaultAuthMethod::TokenFile {
path,
poll_interval_secs,
@@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
},
has_stored_credentials: true,
@@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
},
has_stored_credentials: true,
@@ -901,6 +923,42 @@ mod tests {
assert!(request.to_kms_config().validate().is_ok());
}
/// The admin API reaches Kubernetes auth with the role alone; the mount and
/// the projected token path fall back to the cluster defaults, so a Tenant
/// manifest carries no credential and no cluster-specific paths.
#[test]
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
let raw = serde_json::json!({
"backend_type": "vault-transit",
"address": "https://vault.example.com:8200",
"mount_path": "rustfs",
"auth_method": { "Kubernetes": { "role": "rustfs" } }
});
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
let config = request.to_kms_config();
config.validate().expect("kubernetes auth must validate");
let vault = config.vault_transit_config().expect("vault transit backend config");
let VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} = &vault.auth_method
else {
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
};
assert_eq!(role, "rustfs");
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
let unknown_field = serde_json::json!({
"backend_type": "vault-transit",
"address": "https://vault.example.com:8200",
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
});
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
.expect_err("an unknown auth field must be rejected rather than silently dropped");
}
#[test]
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
+1
View File
@@ -550,6 +550,7 @@ impl VaultKmsClient {
address: config.address.clone(),
namespace: config.namespace.clone(),
attempt_timeout: kms_config.effective_timeout(),
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
};
let source = token_source_for(&config.auth_method, &settings)?;
let policy = VaultCredentialPolicy::from_kms_config(
+271 -5
View File
@@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin {
}
}
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
/// projected ServiceAccount token for a lease-bound Vault token.
///
/// The JWT is re-read on every login because the kubelet rotates a projected
/// token well inside the pod's lifetime; caching it would strand the source on
/// an expired assertion once the current Vault token can no longer be renewed.
///
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
/// the projected token and mounts it world-readable by default, so rejecting
/// group/other bits would refuse every standard pod rather than catch a
/// deployment error.
pub(crate) struct KubernetesLogin {
/// Unauthenticated client used only for the login exchange.
login_client: VaultClient,
mount: String,
role: String,
jwt_path: PathBuf,
}
impl KubernetesLogin {
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
Ok(Self {
login_client: settings.build_login_client()?,
mount,
role,
jwt_path,
})
}
/// Read the ServiceAccount token for one login attempt.
///
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
/// has not projected yet heals the source without a restart.
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
.await
.map_err(|error| AttemptError {
class: ErrorClass::Fatal,
error: KmsError::configuration_error(format!(
"Failed to read Kubernetes ServiceAccount token {}: {error}",
self.jwt_path.display()
)),
})?;
let trimmed = raw.trim();
if trimmed.is_empty() {
raw.zeroize();
return Err(AttemptError {
class: ErrorClass::Fatal,
error: KmsError::configuration_error(format!(
"Kubernetes ServiceAccount token {} is empty",
self.jwt_path.display()
)),
});
}
let jwt = SecretString::new(trimmed.to_string());
raw.zeroize();
Ok(jwt)
}
}
#[async_trait]
impl TokenSource for KubernetesLogin {
async fn acquire(&self) -> AttemptResult<TokenLease> {
let jwt = self.resolve_jwt().await?;
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
.await
.map_err(|error| attempt_error("Kubernetes login", error))?;
Ok(TokenLease::from_auth(auth))
}
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
let auth = vaultrs::token::renew_self(client, None)
.await
.map_err(|error| attempt_error("token renewal", error))?;
Ok(TokenLease::from_auth(auth))
}
}
impl fmt::Debug for KubernetesLogin {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// The login client embeds Vault client settings and must stay out of
// Debug output; the role name is not a secret, and the JWT is never held.
f.debug_struct("KubernetesLogin")
.field("mount", &self.mount)
.field("role", &self.role)
.field("jwt_path", &self.jwt_path)
.finish_non_exhaustive()
}
}
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
/// token file (for example a Vault Agent auto-auth sink).
///
@@ -464,6 +555,9 @@ pub(crate) fn token_source_for(
secret_id.clone(),
secret_id_file.clone(),
)?)),
VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
@@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings {
pub(crate) namespace: Option<String>,
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
pub(crate) attempt_timeout: Duration,
/// Whether to accept an unverified Vault server certificate. Gated on
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
pub(crate) skip_tls_verify: bool,
}
impl VaultConnectionSettings {
@@ -499,6 +596,11 @@ impl VaultConnectionSettings {
// operation-level retry policy.
settings_builder.timeout(Some(self.attempt_timeout));
settings_builder.token(token);
// Always set explicitly: left unset, vaultrs derives this from its own
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
// disable certificate verification behind the KMS configuration and its
// insecure-defaults gate.
settings_builder.verify(!self.skip_tls_verify);
if let Some(namespace) = &self.namespace {
settings_builder.namespace(Some(namespace.clone()));
@@ -551,6 +653,10 @@ impl VaultCredentialPolicy {
refresh_safety_window_secs: Some(secs),
..
}
| VaultAuthMethod::Kubernetes {
refresh_safety_window_secs: Some(secs),
..
}
| VaultAuthMethod::TokenFile {
refresh_safety_window_secs: Some(secs),
..
@@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle {
impl VaultClientHandle {
/// Absolute expiry of this generation's token.
///
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
/// a value too large to add to `issued_at` would panic on the bare `+`. A
/// TTL that cannot be represented is indistinguishable from no expiry, so it
/// collapses to `None` — the same answer already given for the zero-lease
/// tokens Vault issues, which keeps the token in use and still fully
/// validated by Vault on every call.
fn expires_at(&self) -> Option<Instant> {
self.lease.map(|lease| self.issued_at + lease.ttl)
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
}
/// When the renewal task should refresh this generation: half the TTL,
/// leaving the second half as budget for retries before the fail-closed
/// window is reached.
///
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
/// leaving a token that never expires with nothing to renew.
fn renew_at(&self) -> Option<Instant> {
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
}
}
@@ -662,7 +778,7 @@ impl VaultCredentialProvider {
let handle = self.current.load_full();
if let Some(expires_at) = handle.expires_at() {
let now = Instant::now();
if now + self.policy.safety_window >= expires_at {
if self.inside_safety_window(now, expires_at) {
return Err(KmsError::credentials_unavailable(format!(
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
handle.generation, self.policy.safety_window
@@ -672,6 +788,18 @@ impl VaultCredentialProvider {
Ok(handle)
}
/// Whether the token expiring at `expires_at` is close enough to refuse.
///
/// `safety_window` reaches here from persisted configuration, so it is not
/// guaranteed to have passed this version's validation: a window too large
/// to add to the current instant would panic on the bare `+`. Such a window
/// means every token is always inside it, so saturating to "refuse" is both
/// the fail-closed answer and the one the arithmetic was reaching for.
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
now.checked_add(self.policy.safety_window)
.is_none_or(|deadline| deadline >= expires_at)
}
/// Publish the credential gauges for the generation currently installed.
///
/// The fail-closed gauge re-evaluates the very gate
@@ -683,7 +811,7 @@ impl VaultCredentialProvider {
let fail_closed = match handle.expires_at() {
Some(expires_at) => {
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
now + self.policy.safety_window >= expires_at
self.inside_safety_window(now, expires_at)
}
// A generation without an expiry has no remaining TTL to report
// and can never lapse, so it can never fail closed either.
@@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::REDACTED_SECRET;
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
@@ -871,6 +999,7 @@ mod tests {
address: "http://127.0.0.1:8200".to_string(),
namespace: Some("team-namespace".to_string()),
attempt_timeout: Duration::from_secs(30),
skip_tls_verify: false,
}
}
@@ -1057,6 +1186,143 @@ mod tests {
assert!(format!("{source:?}").contains("AppRoleLogin"));
}
#[tokio::test]
async fn test_kubernetes_auth_method_maps_to_login_source() {
let settings = test_settings();
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
.expect("kubernetes auth must map to a login source");
assert!(format!("{source:?}").contains("KubernetesLogin"));
}
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
/// path from persisted configuration, so the fail-closed comparison must
/// survive a window too large to add to the current instant. Before the
/// checked arithmetic this panicked with "overflow when adding duration to
/// instant" on the first request after a lease-bearing login.
#[tokio::test]
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
let (provider, _state) = scripted_provider(
Duration::from_secs(60),
true,
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
)
.await;
let error = provider
.current()
.expect_err("a window wider than any lease must refuse the token");
assert!(
matches!(error, KmsError::CredentialsUnavailable { .. }),
"expected CredentialsUnavailable, got {error:?}"
);
}
/// `lease_duration` is a bare u64 straight off the Vault response and forms
/// the other side of the same comparison, so an absurd one must not panic
/// either. It is indistinguishable from a non-expiring token, which is how
/// the zero-lease case already behaves.
#[tokio::test]
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
let (provider, _state) = scripted_provider(
Duration::from_secs(u64::MAX),
true,
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
)
.await;
provider
.current()
.expect("a token whose expiry cannot be represented must stay usable");
}
/// The configured flag has to reach the HTTP client, not just the config
/// struct: every generation (authenticated and login) builds its own client,
/// and a Vault with a self-signed certificate fails the handshake unless
/// each one carries the setting.
#[test]
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
for skip_tls_verify in [false, true] {
let settings = VaultConnectionSettings {
address: "https://vault.example.com:8200".to_string(),
namespace: None,
attempt_timeout: Duration::from_secs(30),
skip_tls_verify,
};
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
let login = settings.build_login_client().expect("login client must build");
assert_eq!(login.settings.verify, !skip_tls_verify);
}
}
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
/// builder leaves it unset, which would disable certificate verification
/// without passing the KMS insecure-defaults gate.
#[test]
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
assert!(
client.settings.verify,
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
);
});
}
/// The projected token is read fresh per login attempt and trimmed, so a
/// kubelet rotation is picked up without a restart and a trailing newline
/// does not corrupt the assertion sent to Vault.
#[tokio::test]
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("token");
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
let login = KubernetesLogin::new(
&test_settings(),
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
"rustfs".to_string(),
path.clone(),
)
.expect("login source must build");
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
assert_eq!(
login.resolve_jwt().await.expect("second read").expose(),
"rotated-jwt",
"a rotated projected token must be picked up without a restart"
);
}
/// The ServiceAccount token is re-read per attempt, so an unreadable or
/// empty one fails that attempt without reaching Vault; the refresh loop
/// keeps retrying, which is what lets a late projection heal the source.
#[tokio::test]
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
let dir = tempfile::tempdir().expect("temp dir");
let missing = dir.path().join("absent-token");
let empty = dir.path().join("empty-token");
tokio::fs::write(&empty, " \n").await.expect("write empty token");
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
let login =
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
.expect("login source must build");
let error = login
.acquire()
.await
.expect_err("an unusable ServiceAccount token must fail the attempt");
assert!(matches!(error.class, ErrorClass::Fatal));
assert!(error.error.to_string().contains(expected), "got {}", error.error);
}
}
#[tokio::test(start_paused = true)]
async fn test_renewal_task_renews_at_half_ttl() {
let (provider, state) = scripted_provider(
+1
View File
@@ -415,6 +415,7 @@ impl VaultTransitKmsClient {
address: config.address.clone(),
namespace: config.namespace.clone(),
attempt_timeout: kms_config.effective_timeout(),
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
};
let source = token_source_for(&config.auth_method, &settings)?;
let policy = VaultCredentialPolicy::from_kms_config(
+4
View File
@@ -450,6 +450,10 @@ impl VaultRestoreClient {
address: target.address.clone(),
namespace: target.namespace.clone(),
attempt_timeout: kms_config.effective_timeout(),
// A restore target carries no TLS settings, so certificates are
// always verified: recovery is the last path that should accept an
// unauthenticated Vault.
skip_tls_verify: false,
};
let source = token_source_for(&target.auth_method, &settings)?;
let policy = VaultCredentialPolicy::from_kms_config(
+295 -54
View File
@@ -25,6 +25,10 @@ use url::Url;
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
@@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
/// Age in whole seconds beyond which a key is reported as due for rotation;
@@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
/// Where the kubelet projects a pod's ServiceAccount token by default.
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
///
@@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String {
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
}
fn default_vault_kubernetes_mount() -> String {
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
}
fn default_vault_kubernetes_jwt_path() -> PathBuf {
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
}
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
@@ -490,6 +508,23 @@ pub enum VaultAuthMethod {
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
/// for a lease-bound Vault token that is renewed in the background.
Kubernetes {
/// Vault role bound to this ServiceAccount.
role: String,
/// Kubernetes auth engine mount path.
#[serde(default = "default_vault_kubernetes_mount")]
mount: String,
/// Projected ServiceAccount token to present. Re-read on every login so
/// a token the kubelet rotates is picked up without a restart.
#[serde(default = "default_vault_kubernetes_jwt_path")]
jwt_path: PathBuf,
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
/// per-attempt timeout.
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
/// the token is read from `path` and re-read periodically so a token
/// rotated by the agent is picked up without a restart.
@@ -520,6 +555,16 @@ impl VaultAuthMethod {
}
}
/// Kubernetes authentication with the default mount and projected token path.
pub fn kubernetes(role: String) -> Self {
Self::Kubernetes {
role,
mount: default_vault_kubernetes_mount(),
jwt_path: default_vault_kubernetes_jwt_path(),
refresh_safety_window_secs: None,
}
}
/// Agent-managed token file with the default poll interval.
pub fn token_file(path: PathBuf) -> Self {
Self::TokenFile {
@@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod {
.field("mount", mount)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
// No redaction: the role and mount name a Vault binding, and the
// ServiceAccount token itself is never held on this type.
Self::Kubernetes {
role,
mount,
jwt_path,
refresh_safety_window_secs,
} => f
.debug_struct("Kubernetes")
.field("role", role)
.field("mount", mount)
.field("jwt_path", jwt_path)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
Self::TokenFile {
path,
poll_interval_secs,
@@ -1028,50 +1087,12 @@ impl KmsConfig {
});
}
KmsBackend::VaultKv2 => {
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
let auth_method = vault_auth_method_from_env()?;
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
Some(path) => {
tracing::warn!(
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
);
path
}
None => default_vault_kv2_mount_path(),
};
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
address,
auth_method,
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path,
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
tls: vault_tls_config(skip_tls_verify),
}));
config.backend_config =
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
}
KmsBackend::VaultTransit => {
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
let auth_method = vault_auth_method_from_env()?;
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
address,
auth_method,
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
metadata_kv_mount: get_env_str(
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
),
metadata_key_prefix: get_env_str(
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
),
tls: vault_tls_config(skip_tls_verify),
}));
config.backend_config =
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
}
KmsBackend::Static => {
// Read from file first, then fall back to direct env var
@@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool {
path.starts_with(std::env::temp_dir())
}
/// Command-line values that take precedence over the matching environment
/// variables when assembling a Vault backend configuration.
///
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
/// reads, so these are only set when the operator passed an explicit flag.
///
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
/// redacting `Debug` impls elsewhere in this module exist because a derived one
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
/// error instead of a leak.
#[derive(Default, Clone, Copy)]
pub struct VaultCliOverrides<'a> {
pub address: Option<&'a str>,
pub token: Option<&'a str>,
pub mount_path: Option<&'a str>,
}
/// Assemble the Vault KV2 backend configuration from the environment.
///
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
/// so both resolve the same auth method, namespace, TLS and mount settings.
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
let mount_path = match overrides
.mount_path
.map(str::to_string)
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
{
Some(path) => {
tracing::warn!(
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
);
path
}
None => default_vault_kv2_mount_path(),
};
Ok(VaultConfig {
address: vault_address_from_env(overrides.address),
auth_method: vault_auth_method_from_env(overrides.token)?,
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
mount_path,
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
})
}
/// Assemble the Vault Transit backend configuration from the environment.
///
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
/// points share it.
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
Ok(VaultTransitConfig {
address: vault_address_from_env(overrides.address),
auth_method: vault_auth_method_from_env(overrides.token)?,
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
mount_path: overrides
.mount_path
.map(str::to_string)
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
})
}
fn vault_address_from_env(override_value: Option<&str>) -> String {
override_value
.map(str::to_string)
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
}
/// Resolve the Vault auth method from environment variables.
///
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
@@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool {
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
/// file taking precedence. Without a role id the legacy token flow applies.
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
///
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
/// presents the pod's projected ServiceAccount token.
///
/// `token_override` carries a token supplied on the command line; it stands in
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
/// so a flag and the variable it mirrors select the same method.
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
let token = token_override
.map(str::to_string)
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
// A token file names one authoritative credential source; combining it
// with another one would leave the effective identity ambiguous, so
// that is a configuration error rather than a precedence rule.
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
)));
}
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
)));
for (name, configured) in [
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
(ENV_KMS_VAULT_TOKEN, token.is_some()),
] {
if configured {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
)));
}
}
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
}
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
if let Some(role) = kubernetes_role {
// Unlike a leftover static token, a second login method is never a
// stale remnant: both were configured deliberately and neither can be
// ranked over the other.
if role_id.is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
)));
}
return Ok(VaultAuthMethod::Kubernetes {
role,
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
refresh_safety_window_secs: None,
});
}
let Some(role_id) = role_id else {
return Ok(VaultAuthMethod::Token {
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
token: token.unwrap_or_else(|| "dev-token".to_string()),
});
};
@@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
}
Ok(())
}
VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} => {
if role.is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
}
if mount.is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
}
if jwt_path.as_os_str().is_empty() {
return Err(KmsError::configuration_error(format!(
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
)));
}
Ok(())
}
VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
@@ -1976,6 +2117,106 @@ mod tests {
.expect("well-formed token file auth must validate");
}
/// A Kubernetes role alone configures the method: the credential is the
/// pod's projected ServiceAccount token, so nothing secret is in the
/// environment and the mount and token path fall back to the cluster
/// defaults.
#[test]
fn test_from_env_selects_kubernetes() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
(ENV_KMS_VAULT_TOKEN, None),
(ENV_KMS_VAULT_TOKEN_FILE, None),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
let vault = config.vault_transit_config().expect("vault transit backend config");
let VaultAuthMethod::Kubernetes {
role,
mount,
jwt_path,
refresh_safety_window_secs,
} = &vault.auth_method
else {
panic!(
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
vault.auth_method
);
};
assert_eq!(role, "rustfs");
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
assert_eq!(refresh_safety_window_secs, &None);
},
);
}
#[test]
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
(ENV_KMS_VAULT_TOKEN, None),
(ENV_KMS_VAULT_TOKEN_FILE, None),
],
|| {
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
},
);
}
#[test]
fn test_validate_rejects_bad_kubernetes_settings() {
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
backend: KmsBackend::VaultTransit,
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
address: "https://vault.example.com:8200".to_string(),
auth_method,
..Default::default()
})),
..Default::default()
};
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
.validate()
.expect_err("an empty kubernetes role must be rejected");
assert!(error.to_string().contains("role"), "got {error}");
let error = vault_config(VaultAuthMethod::Kubernetes {
role: "rustfs".to_string(),
mount: String::new(),
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
refresh_safety_window_secs: None,
})
.validate()
.expect_err("an empty kubernetes mount must be rejected");
assert!(error.to_string().contains("mount"), "got {error}");
let error = vault_config(VaultAuthMethod::Kubernetes {
role: "rustfs".to_string(),
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
jwt_path: PathBuf::new(),
refresh_safety_window_secs: None,
})
.validate()
.expect_err("an empty ServiceAccount token path must be rejected");
assert!(error.to_string().contains("token path"), "got {error}");
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
.validate()
.expect("well-formed kubernetes auth must validate");
}
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
/// empty one names a path no Vault engine answers. The Transit backend
/// already rejects its own empty mounts; this closes the same gap on the
+3 -3
View File
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
pub api_version: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct SRIAMUser {
#[serde(rename = "accessKey", default)]
pub access_key: String,
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
pub api_version: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct SRGroupInfo {
#[serde(rename = "updateReq", default)]
pub update_req: GroupAddRemove,
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
pub api_version: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct SRIAMItem {
#[serde(default)]
pub r#type: String,
+1 -1
View File
@@ -2,7 +2,7 @@
RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
## Backend comparison
+48 -6
View File
@@ -8,9 +8,12 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti
| --- | --- | --- | --- | --- |
| Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments |
| AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar |
| Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute |
| Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth |
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous.
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity.
All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`.
The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification.
@@ -56,7 +59,44 @@ Deliver the SecretID out of band — a secrets-manager-mounted file, an init-con
The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login.
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart.
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart.
## Kubernetes authentication
On Kubernetes this is the method to prefer: the pod's own ServiceAccount is the identity, so there is no credential to distribute, rotate, or leak into a Secret.
### Vault-side setup
```shell
vault auth enable kubernetes
vault write auth/kubernetes/config \
kubernetes_host="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT"
vault write auth/kubernetes/role/rustfs \
bound_service_account_names=rustfs \
bound_service_account_namespaces=rustfs \
token_policies=rustfs-kms \
token_ttl=1h
```
As with AppRole, keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s).
### RustFS configuration
```shell
RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend
RUSTFS_KMS_VAULT_ADDRESS=https://vault.vault.svc.cluster.local:8200
RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs
# Optional, defaults to "kubernetes":
# RUSTFS_KMS_VAULT_KUBERNETES_MOUNT=kubernetes
# Optional, defaults to the kubelet's projected token path:
# RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
```
RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart.
A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own.
## Vault Agent token file
@@ -101,13 +141,13 @@ If the agent stops refreshing the file that is fine — RustFS re-reads the same
## Fail-closed window
For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it.
- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration.
- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration.
- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise.
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file).
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file).
### Troubleshooting
@@ -117,6 +157,8 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r
| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file |
| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider |
| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider |
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file |
| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server |
| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider |
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file |
When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault.
+5 -1
View File
@@ -286,6 +286,7 @@ fn auth_method_kind(auth: &VaultAuthMethod) -> String {
match auth {
VaultAuthMethod::Token { .. } => "token",
VaultAuthMethod::AppRole { .. } => "approle",
VaultAuthMethod::Kubernetes { .. } => "kubernetes",
VaultAuthMethod::TokenFile { .. } => "token-file",
}
.to_string()
@@ -484,7 +485,10 @@ fn business_trust_root_secrets(config: &KmsConfig) -> Vec<Zeroizing<String>> {
secrets.push(Zeroizing::new(role_id.clone()));
secrets.push(Zeroizing::new(secret_id.clone()));
}
VaultAuthMethod::TokenFile { .. } => {}
// Kubernetes and TokenFile hold no inline plaintext credential: the
// ServiceAccount token and the agent-managed token live in files, and
// the role names a Vault binding rather than half a credential pair.
VaultAuthMethod::Kubernetes { .. } | VaultAuthMethod::TokenFile { .. } => {}
};
match &config.backend_config {
File diff suppressed because it is too large Load Diff
+178 -37
View File
@@ -304,30 +304,37 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
Ok(kms_config)
}
/// Collect the Vault settings the command line owns.
///
/// Everything else — auth method, namespace, TLS, KV mount and metadata paths —
/// is resolved from the environment by the KMS crate, so this path and
/// [`rustfs_kms::config::KmsConfig::from_env`] cannot drift apart. The address
/// stays required here so a missing one is still named instead of silently
/// falling back to the crate's localhost default.
fn vault_cli_overrides<'a>(
cfg: &'a config::Config,
backend_name: &str,
) -> std::io::Result<rustfs_kms::config::VaultCliOverrides<'a>> {
let address = cfg
.kms_vault_address
.as_deref()
.ok_or_else(|| Error::other(format!("Vault address is required for {backend_name} backend")))?;
Ok(rustfs_kms::config::VaultCliOverrides {
address: Some(address),
token: cfg.kms_vault_token.as_deref(),
mount_path: cfg.kms_vault_mount_path.as_deref(),
})
}
/// Build KMS configuration for Vault backend
fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
let vault_address = cfg
.kms_vault_address
.as_ref()
.ok_or_else(|| Error::other("Vault address is required for vault backend"))?;
let vault_token = cfg
.kms_vault_token
.as_ref()
.ok_or_else(|| Error::other("Vault token is required for vault backend"))?;
let backend_config = rustfs_kms::config::vault_kv2_config_from_env(vault_cli_overrides(cfg, "vault")?)
.map_err(|e| Error::other(format!("Vault KMS configuration failed: {e}")))?;
let kms_config = rustfs_kms::config::KmsConfig {
backend: rustfs_kms::config::KmsBackend::VaultKv2,
backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(rustfs_kms::config::VaultConfig {
address: vault_address.clone(),
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
token: vault_token.clone(),
},
namespace: None,
mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()),
kv_mount: "secret".to_string(),
key_path_prefix: "rustfs/kms/keys".to_string(),
tls: None,
})),
backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(backend_config)),
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
default_key_id: cfg.kms_default_key_id.clone(),
@@ -344,26 +351,12 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
/// Build KMS configuration for Vault Transit backend
fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
let vault_address = cfg
.kms_vault_address
.as_ref()
.ok_or_else(|| Error::other("Vault address is required for vault-transit backend"))?;
let vault_token = cfg
.kms_vault_token
.as_ref()
.ok_or_else(|| Error::other("Vault token is required for vault-transit backend"))?;
let backend_config = rustfs_kms::config::vault_transit_config_from_env(vault_cli_overrides(cfg, "vault-transit")?)
.map_err(|e| Error::other(format!("Vault Transit KMS configuration failed: {e}")))?;
let kms_config = rustfs_kms::config::KmsConfig {
backend: rustfs_kms::config::KmsBackend::VaultTransit,
backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(rustfs_kms::config::VaultTransitConfig {
address: vault_address.clone(),
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
token: vault_token.clone(),
},
namespace: None,
mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()),
..rustfs_kms::config::VaultTransitConfig::default()
})),
backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(backend_config)),
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
default_key_id: cfg.kms_default_key_id.clone(),
@@ -1405,7 +1398,10 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
#[cfg(test)]
mod tests {
use super::{build_aws_kms_config, notification_config_to_event_rules, resolve_buffer_profile_config};
use super::{
build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config, notification_config_to_event_rules,
resolve_buffer_profile_config,
};
use crate::config::{BufferConfig, WorkloadProfile};
use rustfs_config::KI_B;
use rustfs_s3_types::EventName;
@@ -1499,6 +1495,151 @@ mod tests {
assert!(err.to_string().contains("Invalid ARN"), "unexpected error: {err}");
}
fn vault_kms_test_config(backend: &str) -> crate::config::Config {
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-vault-kms".to_string()]);
config.kms_enable = true;
config.kms_backend = backend.to_string();
config.kms_vault_address = Some("https://vault.example.com:8200".to_string());
config
}
/// The Vault auth method and the settings the CLI has no flag for come from
/// the environment, so startup and `KmsConfig::from_env` cannot disagree.
/// Regression: startup used to hardcode token auth and require a token,
/// which made every non-token method unreachable through `RUSTFS_KMS_ENABLE`.
#[test]
fn build_vault_transit_kms_config_resolves_auth_and_mounts_from_env() {
let config = temp_env::with_vars(
[
("RUSTFS_KMS_VAULT_TOKEN", None),
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", Some("env-role-id")),
("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID", Some("env-secret-id")),
("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE", None),
("RUSTFS_KMS_VAULT_NAMESPACE", Some("team-a")),
("RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT", Some("rustfs-kv")),
],
|| {
build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
.expect("vault transit KMS configuration should build")
},
);
let vault = config.vault_transit_config().expect("vault transit backend config");
let rustfs_kms::config::VaultAuthMethod::AppRole { role_id, secret_id, .. } = &vault.auth_method else {
panic!("approle in the environment must select AppRole auth, got {:?}", vault.auth_method);
};
assert_eq!(role_id, "env-role-id");
assert_eq!(secret_id, "env-secret-id");
assert_eq!(vault.namespace.as_deref(), Some("team-a"));
assert_eq!(vault.metadata_kv_mount, "rustfs-kv");
}
/// Kubernetes auth needs no credential in the environment at all: the role
/// selects it and the pod's projected ServiceAccount token supplies the rest.
#[test]
fn build_vault_transit_kms_config_selects_kubernetes_auth() {
let config = temp_env::with_vars(
[
("RUSTFS_KMS_VAULT_TOKEN", None),
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")),
("RUSTFS_KMS_VAULT_KUBERNETES_MOUNT", None),
("RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH", None),
],
|| {
build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
.expect("vault transit KMS configuration should build")
},
);
let vault = config.vault_transit_config().expect("vault transit backend config");
let rustfs_kms::config::VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} = &vault.auth_method
else {
panic!(
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
vault.auth_method
);
};
assert_eq!(role, "rustfs");
assert_eq!(mount, rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
assert_eq!(jwt_path, std::path::Path::new(rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
}
/// Two credential sources leave the effective identity ambiguous, so
/// startup refuses rather than picking one.
#[test]
fn build_vault_kms_config_refuses_two_auth_methods() {
temp_env::with_vars(
[
("RUSTFS_KMS_VAULT_TOKEN", None),
("RUSTFS_KMS_VAULT_TOKEN_FILE", Some("/run/vault-agent/token")),
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")),
],
|| {
let error = build_vault_kms_config(&vault_kms_test_config("vault"))
.expect_err("two Vault auth methods must not start the server");
assert!(error.to_string().contains("exactly one"), "unexpected error: {error}");
},
);
}
/// The KV2 backend has its own builder, so the key-location settings have
/// to be proven separately from the Transit one: pointing at the wrong KV
/// mount or prefix makes existing keys look absent.
#[test]
fn build_vault_kms_config_resolves_kv_mount_and_prefix_from_env() {
let config = temp_env::with_vars(
[
("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")),
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
("RUSTFS_KMS_VAULT_KV_MOUNT", Some("rustfs-kv")),
("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")),
],
|| build_vault_kms_config(&vault_kms_test_config("vault")).expect("vault KV2 KMS configuration should build"),
);
let vault = config.vault_config().expect("vault kv2 backend config");
assert_eq!(vault.kv_mount, "rustfs-kv");
assert_eq!(vault.key_path_prefix, "tenant/keys");
}
/// Skipping TLS verification was silently dropped on this path before, so
/// an operator who asked for it still got a verified connection. Now that it
/// is honoured it must fail closed without the development opt-in, rather
/// than quietly downgrading the Vault connection.
#[test]
fn build_vault_transit_kms_config_refuses_skip_tls_verify_without_opt_in() {
let vars = [
("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")),
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
("RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY", Some("true")),
];
temp_env::with_vars(vars, || {
let error = build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
.expect_err("skipping TLS verification must not start the server");
assert!(error.to_string().contains("TLS"), "unexpected error: {error}");
});
temp_env::with_vars(vars, || {
let mut cfg = vault_kms_test_config("vault-transit");
cfg.kms_allow_insecure_dev_defaults = true;
let config = build_vault_transit_kms_config(&cfg).expect("the development opt-in should accept skip-verify");
let vault = config.vault_transit_config().expect("vault transit backend config");
assert!(vault.tls.as_ref().is_some_and(|tls| tls.skip_verify));
});
}
fn aws_kms_test_config() -> crate::config::Config {
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]);
config.kms_enable = true;