mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 18:27:49 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 21c2fb42bb | |||
| d091554ffe | |||
| c04ee41cf0 |
@@ -67,9 +67,6 @@ type MetricValues = Arc<Mutex<BTreeMap<String, MetricPointVersions>>>;
|
|||||||
|
|
||||||
const KIB: usize = 1024;
|
const KIB: usize = 1024;
|
||||||
const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total";
|
const READER_PATH_COUNTER: &str = "rustfs_io_get_object_reader_path_by_size_total";
|
||||||
/// Physical bytes the erasure layer pulled from disk, emitted per shard read by
|
|
||||||
/// `crates/ecstore/src/erasure/coding/decode.rs`.
|
|
||||||
const SHARD_READ_BYTES_COUNTER: &str = "rustfs_io_get_object_shard_read_observed_bytes_total";
|
|
||||||
const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
const MSGPACK_JSON_DECODE_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||||
const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
const MSGPACK_JSON_FALLBACK_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||||
const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
const MSGPACK_JSON_DECODE_ERROR_COUNTER: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||||
@@ -149,7 +146,6 @@ struct OtlpMetricCollector {
|
|||||||
decode_values: MetricValues,
|
decode_values: MetricValues,
|
||||||
fallback_values: MetricValues,
|
fallback_values: MetricValues,
|
||||||
decode_error_values: MetricValues,
|
decode_error_values: MetricValues,
|
||||||
shard_read_values: MetricValues,
|
|
||||||
task: JoinHandle<()>,
|
task: JoinHandle<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,12 +157,10 @@ impl OtlpMetricCollector {
|
|||||||
let decode_values = Arc::new(Mutex::new(BTreeMap::new()));
|
let decode_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||||
let fallback_values = Arc::new(Mutex::new(BTreeMap::new()));
|
let fallback_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||||
let decode_error_values = Arc::new(Mutex::new(BTreeMap::new()));
|
let decode_error_values = Arc::new(Mutex::new(BTreeMap::new()));
|
||||||
let shard_read_values = Arc::new(Mutex::new(BTreeMap::new()));
|
|
||||||
let task_values = values.clone();
|
let task_values = values.clone();
|
||||||
let task_decode_values = decode_values.clone();
|
let task_decode_values = decode_values.clone();
|
||||||
let task_fallback_values = fallback_values.clone();
|
let task_fallback_values = fallback_values.clone();
|
||||||
let task_decode_error_values = decode_error_values.clone();
|
let task_decode_error_values = decode_error_values.clone();
|
||||||
let task_shard_read_values = shard_read_values.clone();
|
|
||||||
let task = tokio::spawn(async move {
|
let task = tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
let Ok((stream, _)) = listener.accept().await else {
|
let Ok((stream, _)) = listener.accept().await else {
|
||||||
@@ -176,7 +170,6 @@ impl OtlpMetricCollector {
|
|||||||
let decode_values = task_decode_values.clone();
|
let decode_values = task_decode_values.clone();
|
||||||
let fallback_values = task_fallback_values.clone();
|
let fallback_values = task_fallback_values.clone();
|
||||||
let decode_error_values = task_decode_error_values.clone();
|
let decode_error_values = task_decode_error_values.clone();
|
||||||
let shard_read_values = task_shard_read_values.clone();
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _ = hyper::server::conn::http1::Builder::new()
|
let _ = hyper::server::conn::http1::Builder::new()
|
||||||
.serve_connection(
|
.serve_connection(
|
||||||
@@ -188,7 +181,6 @@ impl OtlpMetricCollector {
|
|||||||
decode_values.clone(),
|
decode_values.clone(),
|
||||||
fallback_values.clone(),
|
fallback_values.clone(),
|
||||||
decode_error_values.clone(),
|
decode_error_values.clone(),
|
||||||
shard_read_values.clone(),
|
|
||||||
)
|
)
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
@@ -202,48 +194,10 @@ impl OtlpMetricCollector {
|
|||||||
decode_values,
|
decode_values,
|
||||||
fallback_values,
|
fallback_values,
|
||||||
decode_error_values,
|
decode_error_values,
|
||||||
shard_read_values,
|
|
||||||
task,
|
task,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Total physical bytes read from disk across every shard-read label set.
|
|
||||||
async fn shard_read_bytes_total(&self) -> u64 {
|
|
||||||
self.shard_read_values
|
|
||||||
.lock()
|
|
||||||
.await
|
|
||||||
.values()
|
|
||||||
.map(|versions| versions.values().map(|(_, value)| *value).sum::<u64>())
|
|
||||||
.sum()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits until the shard-read counter stops advancing so a measurement window
|
|
||||||
/// is not polluted by exports still in flight.
|
|
||||||
///
|
|
||||||
/// Requires several consecutive equal samples spanning more than one export
|
|
||||||
/// interval (`RUSTFS_OBS_METER_INTERVAL=1`): a single unchanged sample only
|
|
||||||
/// proves the latest export has not landed yet, which silently reads as "no
|
|
||||||
/// disk reads happened" and makes any upper-bound assertion vacuous.
|
|
||||||
async fn wait_for_shard_read_bytes_to_settle(&self) -> TestResult<u64> {
|
|
||||||
const REQUIRED_STABLE_SAMPLES: usize = 5;
|
|
||||||
let mut last = self.shard_read_bytes_total().await;
|
|
||||||
let mut stable = 0;
|
|
||||||
for _ in 0..60 {
|
|
||||||
sleep(Duration::from_millis(500)).await;
|
|
||||||
let current = self.shard_read_bytes_total().await;
|
|
||||||
if current == last {
|
|
||||||
stable += 1;
|
|
||||||
if stable >= REQUIRED_STABLE_SAMPLES {
|
|
||||||
return Ok(current);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
stable = 0;
|
|
||||||
last = current;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err("timed out waiting for shard-read byte counter to settle".into())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 {
|
async fn reader_path_total(&self, path: &str, object_class: &str, size_bucket: &str) -> u64 {
|
||||||
self.reader_path_values(path, object_class, size_bucket).await.values().sum()
|
self.reader_path_values(path, object_class, size_bucket).await.values().sum()
|
||||||
}
|
}
|
||||||
@@ -367,7 +321,6 @@ async fn handle_metric_export(
|
|||||||
decode_values: MetricValues,
|
decode_values: MetricValues,
|
||||||
fallback_values: MetricValues,
|
fallback_values: MetricValues,
|
||||||
decode_error_values: MetricValues,
|
decode_error_values: MetricValues,
|
||||||
shard_read_values: MetricValues,
|
|
||||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||||
if request.uri().path() != "/v1/metrics" {
|
if request.uri().path() != "/v1/metrics" {
|
||||||
return Ok(response(StatusCode::NOT_FOUND));
|
return Ok(response(StatusCode::NOT_FOUND));
|
||||||
@@ -401,9 +354,7 @@ async fn handle_metric_export(
|
|||||||
let mut decode_values = decode_values.lock().await;
|
let mut decode_values = decode_values.lock().await;
|
||||||
let mut fallback_values = fallback_values.lock().await;
|
let mut fallback_values = fallback_values.lock().await;
|
||||||
let mut decode_error_values = decode_error_values.lock().await;
|
let mut decode_error_values = decode_error_values.lock().await;
|
||||||
let mut shard_read_values = shard_read_values.lock().await;
|
|
||||||
record_reader_path_metrics(&export, &mut values);
|
record_reader_path_metrics(&export, &mut values);
|
||||||
record_shard_read_bytes_metrics(&export, &mut shard_read_values);
|
|
||||||
record_msgpack_decode_metrics(&export, &mut decode_values);
|
record_msgpack_decode_metrics(&export, &mut decode_values);
|
||||||
record_msgpack_fallback_metrics(&export, &mut fallback_values);
|
record_msgpack_fallback_metrics(&export, &mut fallback_values);
|
||||||
record_msgpack_decode_error_metrics(&export, &mut decode_error_values);
|
record_msgpack_decode_error_metrics(&export, &mut decode_error_values);
|
||||||
@@ -424,50 +375,6 @@ fn reader_path_metric_key(path: &str, object_class: &str, size_bucket: &str) ->
|
|||||||
format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}")
|
format!("{path}\u{1f}{object_class}\u{1f}{size_bucket}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Accumulates `SHARD_READ_BYTES_COUNTER` across all label sets. Only the total
|
|
||||||
/// matters: it is the number of physical bytes the erasure layer actually pulled
|
|
||||||
/// from disk, which is what separates a bounded per-part read from a decode of
|
|
||||||
/// the whole object.
|
|
||||||
fn record_shard_read_bytes_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
|
||||||
for resource_metrics in &export.resource_metrics {
|
|
||||||
for scope_metrics in &resource_metrics.scope_metrics {
|
|
||||||
for metric in &scope_metrics.metrics {
|
|
||||||
if metric.name != SHARD_READ_BYTES_COUNTER {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let Some(metric::Data::Sum(sum)) = &metric.data else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
for point in &sum.data_points {
|
|
||||||
let Some(number_data_point::Value::AsInt(value)) = point.value.as_ref() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let value = u64::try_from(*value).unwrap_or_default();
|
|
||||||
// Keyed by labels, not by position: point order within an export
|
|
||||||
// is not guaranteed stable, so an index key would alias distinct
|
|
||||||
// series across batches.
|
|
||||||
let key = format!(
|
|
||||||
"{}\u{1f}{}\u{1f}{}",
|
|
||||||
attribute_string(&point.attributes, "path").unwrap_or_default(),
|
|
||||||
attribute_string(&point.attributes, "role").unwrap_or_default(),
|
|
||||||
attribute_string(&point.attributes, "outcome").unwrap_or_default(),
|
|
||||||
);
|
|
||||||
values
|
|
||||||
.entry(key)
|
|
||||||
.or_default()
|
|
||||||
.entry(point.start_time_unix_nano)
|
|
||||||
.and_modify(|current| {
|
|
||||||
if point.time_unix_nano >= current.0 {
|
|
||||||
*current = (point.time_unix_nano, value);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.or_insert((point.time_unix_nano, value));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
fn record_reader_path_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, MetricPointVersions>) {
|
||||||
for resource_metrics in &export.resource_metrics {
|
for resource_metrics in &export.resource_metrics {
|
||||||
for scope_metrics in &resource_metrics.scope_metrics {
|
for scope_metrics in &resource_metrics.scope_metrics {
|
||||||
@@ -1957,86 +1864,6 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A tail range over a compressed multipart object must read only the physical
|
|
||||||
/// data it needs, not decode the object from byte zero.
|
|
||||||
///
|
|
||||||
/// The byte-exactness tests around this one stay green even if the seek path
|
|
||||||
/// regresses into decoding from the start of the object: the bytes returned are
|
|
||||||
/// still correct, only the read amplification explodes. This asserts the cost
|
|
||||||
/// side, using `SHARD_READ_BYTES_COUNTER` — already emitted per shard read by the
|
|
||||||
/// erasure layer, so no production code is instrumented for the test.
|
|
||||||
///
|
|
||||||
/// `get_compressed_offsets` skips whole preceding parts by their stored size and
|
|
||||||
/// then seeks inside the covering part via its compression index, so a bounded
|
|
||||||
/// read costs on the order of the covering part's block size against a ~5 MiB
|
|
||||||
/// object.
|
|
||||||
#[tokio::test]
|
|
||||||
#[serial]
|
|
||||||
async fn four_node_compressed_multipart_tail_range_reads_are_bounded() -> TestResult {
|
|
||||||
init_logging();
|
|
||||||
|
|
||||||
let collector = OtlpMetricCollector::start().await?;
|
|
||||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
|
||||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
|
||||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
|
||||||
cluster.start().await?;
|
|
||||||
|
|
||||||
let bucket = "inline-multipart-compression-tail-range";
|
|
||||||
cluster.create_test_bucket(bucket).await?;
|
|
||||||
let client = cluster.create_s3_client(0)?;
|
|
||||||
let key = "multipart/tail-range.txt";
|
|
||||||
let (body, _second_part, etag) = put_two_part_multipart(&client, bucket, key).await?;
|
|
||||||
|
|
||||||
// Establish that the object really took the compressed read path; otherwise a
|
|
||||||
// small delta below would only prove compression never happened.
|
|
||||||
assert_reader_path(
|
|
||||||
&collector,
|
|
||||||
&client,
|
|
||||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, COMPRESSED),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let baseline = collector.wait_for_shard_read_bytes_to_settle().await?;
|
|
||||||
|
|
||||||
let tail_len = 4 * KIB;
|
|
||||||
let start = body.len() - tail_len;
|
|
||||||
let end = body.len() - 1;
|
|
||||||
let range = client
|
|
||||||
.get_object()
|
|
||||||
.bucket(bucket)
|
|
||||||
.key(key)
|
|
||||||
.range(format!("bytes={start}-{end}"))
|
|
||||||
.send()
|
|
||||||
.await?;
|
|
||||||
let tail = range.body.collect().await?.into_bytes();
|
|
||||||
assert_eq!(tail.as_ref(), &body[start..], "tail range returned wrong bytes");
|
|
||||||
|
|
||||||
let after = collector.wait_for_shard_read_bytes_to_settle().await?;
|
|
||||||
let read_bytes = after.saturating_sub(baseline);
|
|
||||||
|
|
||||||
// A zero delta means the window caught nothing — an unexported counter, or a
|
|
||||||
// read served without touching the erasure layer — which would make the upper
|
|
||||||
// bound vacuously true. Fail instead of passing blind.
|
|
||||||
assert!(
|
|
||||||
read_bytes > 0,
|
|
||||||
"no shard reads observed for the tail range; the budget assertion below would be vacuous"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Part 1 alone is MPU_PART_1_SIZE, so a whole-object decode cannot come in
|
|
||||||
// under it. Half the logical size leaves generous headroom for erasure padding
|
|
||||||
// and unrelated background reads while still failing loudly on a full decode.
|
|
||||||
let budget = (body.len() / 2) as u64;
|
|
||||||
assert!(
|
|
||||||
read_bytes < budget,
|
|
||||||
"tail range read {read_bytes} physical bytes for a {tail_len}-byte range (budget {budget}, object {} bytes): \
|
|
||||||
the read is not bounded to the covering part",
|
|
||||||
body.len()
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> TestResult {
|
||||||
|
|||||||
@@ -667,6 +667,368 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
|||||||
Err(EcstoreError::PreconditionFailed)
|
Err(EcstoreError::PreconditionFailed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||||
|
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||||
|
/// node is already processing the backlog.
|
||||||
|
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||||
|
let recovery_lock = match storage
|
||||||
|
.new_ns_lock(
|
||||||
|
ReplicationMetadataStore::rustfs_meta_bucket(),
|
||||||
|
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(lock) => lock,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to create the MRF recovery leader lock"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match recovery_lock
|
||||||
|
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(guard) => Some(guard),
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
"Another node is already processing the MRF recovery backlog"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reads and decodes the on-disk MRF recovery file.
|
||||||
|
/// Returns `None` when there is nothing to replay: missing file (publishes an
|
||||||
|
/// empty available summary), read failure, or corrupt data (quarantined).
|
||||||
|
async fn load_mrf_recovery_entries<S: ReplicationStorage>(storage: &Arc<S>) -> Option<Vec<MrfReplicateEntry>> {
|
||||||
|
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(EcstoreError::ConfigNotFound) => {
|
||||||
|
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||||
|
available: true,
|
||||||
|
buckets: Vec::new(),
|
||||||
|
});
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %e,
|
||||||
|
"Failed to load MRF recovery file"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match decode_mrf_file(&data) {
|
||||||
|
Ok(v) => Some(v),
|
||||||
|
Err(e) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %e,
|
||||||
|
"Failed to decode MRF recovery file — preserving corrupt data"
|
||||||
|
);
|
||||||
|
quarantine_mrf_file(storage, &data).await;
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays one MRF recovery entry by operation kind.
|
||||||
|
/// Returns `None` when the entry is skipped entirely (no admission outcome);
|
||||||
|
/// entries that must be retried later are pushed onto `retry_entries`.
|
||||||
|
async fn replay_mrf_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
match entry.op {
|
||||||
|
MrfOpKind::Delete => replay_mrf_delete_entry(entry, storage, retry_entries).await,
|
||||||
|
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
||||||
|
replay_mrf_object_entry(entry, storage, retry_entries).await
|
||||||
|
}
|
||||||
|
MrfOpKind::Metadata => replay_mrf_metadata_entry(entry, storage, retry_entries).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays a delete-kind MRF entry: force-delete intents replay directly,
|
||||||
|
/// stale force-delete generations are skipped, and plain deletes are
|
||||||
|
/// reconstructed as heal deletes.
|
||||||
|
async fn replay_mrf_delete_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
if should_replay_force_delete_intent(entry) {
|
||||||
|
let operation_id = entry.force_delete_id?;
|
||||||
|
let delete = force_delete_heal_replication_info(entry, operation_id);
|
||||||
|
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
} else if entry.force_delete_id.is_some() {
|
||||||
|
Some(ReplicationQueueAdmission::Skipped)
|
||||||
|
} else {
|
||||||
|
replay_mrf_reconstructed_delete(entry, storage, retry_entries).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: heal replication info for a replayed force-delete intent.
|
||||||
|
fn force_delete_heal_replication_info(entry: &MrfReplicateEntry, operation_id: uuid::Uuid) -> DeletedObjectReplicationInfo {
|
||||||
|
DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
force_delete: true,
|
||||||
|
force_delete_id: Some(operation_id),
|
||||||
|
force_delete_target_arns: entry.target_arns.clone(),
|
||||||
|
force_delete_generation: entry.force_delete_generation,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a heal delete and re-queue it. We do NOT call
|
||||||
|
/// get_object_info here because the delete-marker or version may
|
||||||
|
/// already be absent from the local store — that is expected.
|
||||||
|
async fn replay_mrf_reconstructed_delete<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
||||||
|
let oi = ObjectInfo {
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
delete_marker: entry.delete_marker,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let dsc = resolve_mrf_delete_replicate_decision(entry, &oi, versioned, retry_entries).await?;
|
||||||
|
let dv = reconstructed_heal_delete_info(entry, &oi, &dsc);
|
||||||
|
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The MRF entry does not persist the replication decision and the
|
||||||
|
/// source object is gone, so re-derive the decision from the live
|
||||||
|
/// bucket config (mirroring get_heal_replicate_object_info) and set
|
||||||
|
/// it on the reconstructed delete. Without this the decision string
|
||||||
|
/// is empty and the delete replicates to zero targets — a silent
|
||||||
|
/// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
||||||
|
async fn resolve_mrf_delete_replicate_decision(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
oi: &ObjectInfo,
|
||||||
|
versioned: bool,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicateDecision> {
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
||||||
|
Ok(None) => None,
|
||||||
|
Err(_) => {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Ok(Some(_)) => match check_replicate_delete_strict(
|
||||||
|
&entry.bucket,
|
||||||
|
&ObjectToDelete {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
oi,
|
||||||
|
&ObjectOptions {
|
||||||
|
versioned,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(dsc) => Some(dsc),
|
||||||
|
Err(_) => {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Some(replicate_decision_for_admitted_targets(&entry.target_arns))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: reconstructed heal delete carrying the re-derived
|
||||||
|
/// replication decision.
|
||||||
|
fn reconstructed_heal_delete_info(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
oi: &ObjectInfo,
|
||||||
|
dsc: &ReplicateDecision,
|
||||||
|
) -> DeletedObjectReplicationInfo {
|
||||||
|
let mut rstate = oi.replication_state();
|
||||||
|
rstate.replicate_decision_str = dsc.to_string();
|
||||||
|
|
||||||
|
let delete_marker_mtime = entry
|
||||||
|
.delete_marker_mtime
|
||||||
|
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
||||||
|
|
||||||
|
DeletedObjectReplicationInfo {
|
||||||
|
delete_object: ReplicationDeletedObject {
|
||||||
|
object_name: entry.object.clone(),
|
||||||
|
version_id: entry.version_id,
|
||||||
|
delete_marker_version_id: entry.delete_marker_version_id,
|
||||||
|
delete_marker: entry.delete_marker,
|
||||||
|
delete_marker_mtime,
|
||||||
|
force_delete: entry.force_delete,
|
||||||
|
replication_state: Some(rstate),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
bucket: entry.bucket.clone(),
|
||||||
|
op_type: ReplicationType::Heal,
|
||||||
|
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays an Object/Heal/ExistingObject MRF entry against the live source object.
|
||||||
|
async fn replay_mrf_object_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_id: entry.version_id.map(|u| u.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||||
|
Ok(oi) => oi,
|
||||||
|
Err(e) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
bucket = %entry.bucket,
|
||||||
|
object = %entry.object,
|
||||||
|
error = %e,
|
||||||
|
"MRF recovery: source object lookup failed"
|
||||||
|
);
|
||||||
|
if should_retry_mrf_source_lookup(&e) {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
// Legacy entries predate target admission persistence. They cannot
|
||||||
|
// be safely attributed, so retain the old live-config fallback.
|
||||||
|
Some(queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||||
|
} else {
|
||||||
|
let roi = admitted_mrf_replicate_object(oi, entry, entry.op.replication_type());
|
||||||
|
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replays a metadata-kind MRF entry against the live source object.
|
||||||
|
async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||||
|
entry: &MrfReplicateEntry,
|
||||||
|
storage: &Arc<S>,
|
||||||
|
retry_entries: &mut Vec<MrfReplicateEntry>,
|
||||||
|
) -> Option<ReplicationQueueAdmission> {
|
||||||
|
let opts = ObjectOptions {
|
||||||
|
version_id: entry.version_id.map(|u| u.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
||||||
|
Ok(oi) => oi,
|
||||||
|
Err(e) => {
|
||||||
|
debug!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
bucket = %entry.bucket,
|
||||||
|
object = %entry.object,
|
||||||
|
error = %e,
|
||||||
|
"MRF metadata recovery: source object lookup failed"
|
||||||
|
);
|
||||||
|
if should_retry_mrf_source_lookup(&e) {
|
||||||
|
retry_entries.push(entry.clone());
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if entry.target_arns.is_empty() {
|
||||||
|
Some(queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await)
|
||||||
|
} else {
|
||||||
|
let roi = admitted_mrf_replicate_object(oi, entry, ReplicationType::Metadata);
|
||||||
|
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
||||||
|
Some(ReplicationQueueAdmission::Queued)
|
||||||
|
} else {
|
||||||
|
Some(ReplicationQueueAdmission::Missed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure DTO construction: replicate-object info for an entry with persisted
|
||||||
|
/// admitted targets, carrying over the entry's retry count.
|
||||||
|
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||||
|
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||||
|
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||||
|
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||||
|
roi
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||||
|
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||||
|
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||||
|
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||||
|
storage: &Arc<S>,
|
||||||
|
recovery_guard: &rustfs_lock::NamespaceLockGuard,
|
||||||
|
entries: &[MrfReplicateEntry],
|
||||||
|
retry_entries: &[MrfReplicateEntry],
|
||||||
|
) -> Vec<MrfReplicateEntry> {
|
||||||
|
match acknowledge_mrf_recovery(storage.clone(), recovery_guard, entries, retry_entries).await {
|
||||||
|
Ok(retained) => retained,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %error,
|
||||||
|
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||||
|
);
|
||||||
|
match read_mrf_entries(storage.clone()).await {
|
||||||
|
Ok(current) => current,
|
||||||
|
Err(read_error) => {
|
||||||
|
warn!(
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||||
|
error = %read_error,
|
||||||
|
"Failed to refresh the MRF backlog after acknowledgement failure"
|
||||||
|
);
|
||||||
|
entries.to_vec()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||||
struct ResyncActiveConflictError {
|
struct ResyncActiveConflictError {
|
||||||
@@ -1221,71 +1583,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
let storage = self.storage.clone();
|
let storage = self.storage.clone();
|
||||||
|
|
||||||
let handle = tokio::spawn(async move {
|
let handle = tokio::spawn(async move {
|
||||||
let recovery_lock = match storage
|
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||||
.new_ns_lock(
|
return;
|
||||||
ReplicationMetadataStore::rustfs_meta_bucket(),
|
|
||||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(lock) => lock,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %error,
|
|
||||||
"Failed to create the MRF recovery leader lock"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let recovery_guard = match recovery_lock
|
|
||||||
.get_write_lock_quiet(ReplicationLockTiming::acquire_timeout())
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(guard) => guard,
|
|
||||||
Err(_) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
"Another node is already processing the MRF recovery backlog"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||||
Ok(d) => d,
|
return;
|
||||||
Err(EcstoreError::ConfigNotFound) => {
|
|
||||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
|
||||||
available: true,
|
|
||||||
buckets: Vec::new(),
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %e,
|
|
||||||
"Failed to load MRF recovery file"
|
|
||||||
);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let entries = match decode_mrf_file(&data) {
|
|
||||||
Ok(v) => v,
|
|
||||||
Err(e) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %e,
|
|
||||||
"Failed to decode MRF recovery file — preserving corrupt data"
|
|
||||||
);
|
|
||||||
quarantine_mrf_file(&storage, &data).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||||
|
|
||||||
@@ -1294,187 +1597,8 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
let mut retry_entries = Vec::new();
|
let mut retry_entries = Vec::new();
|
||||||
|
|
||||||
for entry in entries.iter() {
|
for entry in entries.iter() {
|
||||||
let admission = match entry.op {
|
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||||
MrfOpKind::Delete => {
|
continue;
|
||||||
if should_replay_force_delete_intent(entry) {
|
|
||||||
let Some(operation_id) = entry.force_delete_id else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let delete = DeletedObjectReplicationInfo {
|
|
||||||
delete_object: ReplicationDeletedObject {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
force_delete: true,
|
|
||||||
force_delete_id: Some(operation_id),
|
|
||||||
force_delete_target_arns: entry.target_arns.clone(),
|
|
||||||
force_delete_generation: entry.force_delete_generation,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
op_type: ReplicationType::Heal,
|
|
||||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
if replicate_delete_with_outcome(delete, storage.clone()).await {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
} else if entry.force_delete_id.is_some() {
|
|
||||||
ReplicationQueueAdmission::Skipped
|
|
||||||
} else {
|
|
||||||
// Reconstruct a heal delete and re-queue it. We do NOT call
|
|
||||||
// get_object_info here because the delete-marker or version may
|
|
||||||
// already be absent from the local store — that is expected.
|
|
||||||
//
|
|
||||||
// The MRF entry does not persist the replication decision and the
|
|
||||||
// source object is gone, so re-derive the decision from the live
|
|
||||||
// bucket config (mirroring get_heal_replicate_object_info) and set
|
|
||||||
// it on the reconstructed delete. Without this the decision string
|
|
||||||
// is empty and the delete replicates to zero targets — a silent
|
|
||||||
// no-op that leaves replicas diverged (backlog#858 / #799 B9).
|
|
||||||
let versioned = ReplicationVersioningStore::prefix_enabled(&entry.bucket, &entry.object).await;
|
|
||||||
let oi = ObjectInfo {
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
delete_marker: entry.delete_marker,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let dsc = if entry.target_arns.is_empty() {
|
|
||||||
match ReplicationMetadataStore::optional_replication_config(&entry.bucket).await {
|
|
||||||
Ok(None) => continue,
|
|
||||||
Err(_) => {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Ok(Some(_)) => match check_replicate_delete_strict(
|
|
||||||
&entry.bucket,
|
|
||||||
&ObjectToDelete {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&oi,
|
|
||||||
&ObjectOptions {
|
|
||||||
versioned,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(dsc) => dsc,
|
|
||||||
Err(_) => {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
replicate_decision_for_admitted_targets(&entry.target_arns)
|
|
||||||
};
|
|
||||||
let mut rstate = oi.replication_state();
|
|
||||||
rstate.replicate_decision_str = dsc.to_string();
|
|
||||||
|
|
||||||
let delete_marker_mtime = entry
|
|
||||||
.delete_marker_mtime
|
|
||||||
.and_then(|nanos| OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).ok());
|
|
||||||
|
|
||||||
let dv = DeletedObjectReplicationInfo {
|
|
||||||
delete_object: ReplicationDeletedObject {
|
|
||||||
object_name: entry.object.clone(),
|
|
||||||
version_id: entry.version_id,
|
|
||||||
delete_marker_version_id: entry.delete_marker_version_id,
|
|
||||||
delete_marker: entry.delete_marker,
|
|
||||||
delete_marker_mtime,
|
|
||||||
force_delete: entry.force_delete,
|
|
||||||
replication_state: Some(rstate),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
bucket: entry.bucket.clone(),
|
|
||||||
op_type: ReplicationType::Heal,
|
|
||||||
event_type: REPLICATE_HEAL_DELETE.to_string(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
if replicate_delete_with_outcome(dv, storage.clone()).await {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MrfOpKind::Object | MrfOpKind::Heal | MrfOpKind::ExistingObject => {
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
version_id: entry.version_id.map(|u| u.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
|
||||||
Ok(oi) => oi,
|
|
||||||
Err(e) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
bucket = %entry.bucket,
|
|
||||||
object = %entry.object,
|
|
||||||
error = %e,
|
|
||||||
"MRF recovery: source object lookup failed"
|
|
||||||
);
|
|
||||||
if should_retry_mrf_source_lookup(&e) {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if entry.target_arns.is_empty() {
|
|
||||||
// Legacy entries predate target admission persistence. They cannot
|
|
||||||
// be safely attributed, so retain the old live-config fallback.
|
|
||||||
queue_replication_heal(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
|
||||||
} else {
|
|
||||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
|
||||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, entry.op.replication_type());
|
|
||||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
|
||||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
MrfOpKind::Metadata => {
|
|
||||||
let opts = ObjectOptions {
|
|
||||||
version_id: entry.version_id.map(|u| u.to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
|
|
||||||
Ok(oi) => oi,
|
|
||||||
Err(e) => {
|
|
||||||
debug!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
bucket = %entry.bucket,
|
|
||||||
object = %entry.object,
|
|
||||||
error = %e,
|
|
||||||
"MRF metadata recovery: source object lookup failed"
|
|
||||||
);
|
|
||||||
if should_retry_mrf_source_lookup(&e) {
|
|
||||||
retry_entries.push(entry.clone());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if entry.target_arns.is_empty() {
|
|
||||||
queue_replication_metadata(&entry.bucket, oi, entry.retry_count.max(0) as u32).await
|
|
||||||
} else {
|
|
||||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
|
||||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
|
||||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
|
||||||
if replicate_object_with_outcome(roi, storage.clone()).await.1 {
|
|
||||||
ReplicationQueueAdmission::Queued
|
|
||||||
} else {
|
|
||||||
ReplicationQueueAdmission::Missed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if admission == ReplicationQueueAdmission::Missed {
|
if admission == ReplicationQueueAdmission::Missed {
|
||||||
@@ -1484,29 +1608,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let retained = match acknowledge_mrf_recovery(storage.clone(), &recovery_guard, &entries, &retry_entries).await {
|
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||||
Ok(retained) => retained,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %error,
|
|
||||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
|
||||||
);
|
|
||||||
match read_mrf_entries(storage.clone()).await {
|
|
||||||
Ok(current) => current,
|
|
||||||
Err(read_error) => {
|
|
||||||
warn!(
|
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
|
||||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
|
||||||
error = %read_error,
|
|
||||||
"Failed to refresh the MRF backlog after acknowledgement failure"
|
|
||||||
);
|
|
||||||
entries.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let retained_count = retained.len();
|
let retained_count = retained.len();
|
||||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -293,6 +293,15 @@ enum StrictVaultAuthMethod {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
refresh_safety_window_secs: Option<u64>,
|
refresh_safety_window_secs: Option<u64>,
|
||||||
},
|
},
|
||||||
|
Kubernetes {
|
||||||
|
role: String,
|
||||||
|
#[serde(default)]
|
||||||
|
mount: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
jwt_path: Option<std::path::PathBuf>,
|
||||||
|
#[serde(default)]
|
||||||
|
refresh_safety_window_secs: Option<u64>,
|
||||||
|
},
|
||||||
TokenFile {
|
TokenFile {
|
||||||
path: std::path::PathBuf,
|
path: std::path::PathBuf,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -319,6 +328,17 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
|
|||||||
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
|
||||||
refresh_safety_window_secs,
|
refresh_safety_window_secs,
|
||||||
},
|
},
|
||||||
|
StrictVaultAuthMethod::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount,
|
||||||
|
jwt_path,
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
} => Self::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
|
||||||
|
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
},
|
||||||
StrictVaultAuthMethod::TokenFile {
|
StrictVaultAuthMethod::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -499,6 +519,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
|||||||
auth_method_type: match &vault_config.auth_method {
|
auth_method_type: match &vault_config.auth_method {
|
||||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||||
|
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||||
},
|
},
|
||||||
has_stored_credentials: true,
|
has_stored_credentials: true,
|
||||||
@@ -513,6 +534,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
|||||||
auth_method_type: match &vault_config.auth_method {
|
auth_method_type: match &vault_config.auth_method {
|
||||||
VaultAuthMethod::Token { .. } => "token".to_string(),
|
VaultAuthMethod::Token { .. } => "token".to_string(),
|
||||||
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
|
||||||
|
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
|
||||||
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
|
||||||
},
|
},
|
||||||
has_stored_credentials: true,
|
has_stored_credentials: true,
|
||||||
@@ -901,6 +923,42 @@ mod tests {
|
|||||||
assert!(request.to_kms_config().validate().is_ok());
|
assert!(request.to_kms_config().validate().is_ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The admin API reaches Kubernetes auth with the role alone; the mount and
|
||||||
|
/// the projected token path fall back to the cluster defaults, so a Tenant
|
||||||
|
/// manifest carries no credential and no cluster-specific paths.
|
||||||
|
#[test]
|
||||||
|
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
|
||||||
|
let raw = serde_json::json!({
|
||||||
|
"backend_type": "vault-transit",
|
||||||
|
"address": "https://vault.example.com:8200",
|
||||||
|
"mount_path": "rustfs",
|
||||||
|
"auth_method": { "Kubernetes": { "role": "rustfs" } }
|
||||||
|
});
|
||||||
|
|
||||||
|
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
|
||||||
|
let config = request.to_kms_config();
|
||||||
|
config.validate().expect("kubernetes auth must validate");
|
||||||
|
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
let VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} = &vault.auth_method
|
||||||
|
else {
|
||||||
|
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
|
||||||
|
};
|
||||||
|
assert_eq!(role, "rustfs");
|
||||||
|
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||||
|
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||||
|
|
||||||
|
let unknown_field = serde_json::json!({
|
||||||
|
"backend_type": "vault-transit",
|
||||||
|
"address": "https://vault.example.com:8200",
|
||||||
|
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
|
||||||
|
});
|
||||||
|
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
|
||||||
|
.expect_err("an unknown auth field must be rejected rather than silently dropped");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
fn test_deserialize_aws_configure_request_accepts_type_aliases() {
|
||||||
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
|
||||||
|
|||||||
@@ -550,6 +550,7 @@ impl VaultKmsClient {
|
|||||||
address: config.address.clone(),
|
address: config.address.clone(),
|
||||||
namespace: config.namespace.clone(),
|
namespace: config.namespace.clone(),
|
||||||
attempt_timeout: kms_config.effective_timeout(),
|
attempt_timeout: kms_config.effective_timeout(),
|
||||||
|
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||||
};
|
};
|
||||||
let source = token_source_for(&config.auth_method, &settings)?;
|
let source = token_source_for(&config.auth_method, &settings)?;
|
||||||
let policy = VaultCredentialPolicy::from_kms_config(
|
let policy = VaultCredentialPolicy::from_kms_config(
|
||||||
|
|||||||
@@ -326,6 +326,97 @@ impl fmt::Debug for AppRoleLogin {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
|
||||||
|
/// projected ServiceAccount token for a lease-bound Vault token.
|
||||||
|
///
|
||||||
|
/// The JWT is re-read on every login because the kubelet rotates a projected
|
||||||
|
/// token well inside the pod's lifetime; caching it would strand the source on
|
||||||
|
/// an expired assertion once the current Vault token can no longer be renewed.
|
||||||
|
///
|
||||||
|
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
|
||||||
|
/// the projected token and mounts it world-readable by default, so rejecting
|
||||||
|
/// group/other bits would refuse every standard pod rather than catch a
|
||||||
|
/// deployment error.
|
||||||
|
pub(crate) struct KubernetesLogin {
|
||||||
|
/// Unauthenticated client used only for the login exchange.
|
||||||
|
login_client: VaultClient,
|
||||||
|
mount: String,
|
||||||
|
role: String,
|
||||||
|
jwt_path: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl KubernetesLogin {
|
||||||
|
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
|
||||||
|
Ok(Self {
|
||||||
|
login_client: settings.build_login_client()?,
|
||||||
|
mount,
|
||||||
|
role,
|
||||||
|
jwt_path,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the ServiceAccount token for one login attempt.
|
||||||
|
///
|
||||||
|
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
|
||||||
|
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
|
||||||
|
/// has not projected yet heals the source without a restart.
|
||||||
|
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
|
||||||
|
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
|
||||||
|
.await
|
||||||
|
.map_err(|error| AttemptError {
|
||||||
|
class: ErrorClass::Fatal,
|
||||||
|
error: KmsError::configuration_error(format!(
|
||||||
|
"Failed to read Kubernetes ServiceAccount token {}: {error}",
|
||||||
|
self.jwt_path.display()
|
||||||
|
)),
|
||||||
|
})?;
|
||||||
|
let trimmed = raw.trim();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
raw.zeroize();
|
||||||
|
return Err(AttemptError {
|
||||||
|
class: ErrorClass::Fatal,
|
||||||
|
error: KmsError::configuration_error(format!(
|
||||||
|
"Kubernetes ServiceAccount token {} is empty",
|
||||||
|
self.jwt_path.display()
|
||||||
|
)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let jwt = SecretString::new(trimmed.to_string());
|
||||||
|
raw.zeroize();
|
||||||
|
Ok(jwt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl TokenSource for KubernetesLogin {
|
||||||
|
async fn acquire(&self) -> AttemptResult<TokenLease> {
|
||||||
|
let jwt = self.resolve_jwt().await?;
|
||||||
|
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
|
||||||
|
.await
|
||||||
|
.map_err(|error| attempt_error("Kubernetes login", error))?;
|
||||||
|
Ok(TokenLease::from_auth(auth))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
|
||||||
|
let auth = vaultrs::token::renew_self(client, None)
|
||||||
|
.await
|
||||||
|
.map_err(|error| attempt_error("token renewal", error))?;
|
||||||
|
Ok(TokenLease::from_auth(auth))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for KubernetesLogin {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
// The login client embeds Vault client settings and must stay out of
|
||||||
|
// Debug output; the role name is not a secret, and the JWT is never held.
|
||||||
|
f.debug_struct("KubernetesLogin")
|
||||||
|
.field("mount", &self.mount)
|
||||||
|
.field("role", &self.role)
|
||||||
|
.field("jwt_path", &self.jwt_path)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
|
||||||
/// token file (for example a Vault Agent auto-auth sink).
|
/// token file (for example a Vault Agent auto-auth sink).
|
||||||
///
|
///
|
||||||
@@ -464,6 +555,9 @@ pub(crate) fn token_source_for(
|
|||||||
secret_id.clone(),
|
secret_id.clone(),
|
||||||
secret_id_file.clone(),
|
secret_id_file.clone(),
|
||||||
)?)),
|
)?)),
|
||||||
|
VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
|
||||||
VaultAuthMethod::TokenFile {
|
VaultAuthMethod::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -486,6 +580,9 @@ pub(crate) struct VaultConnectionSettings {
|
|||||||
pub(crate) namespace: Option<String>,
|
pub(crate) namespace: Option<String>,
|
||||||
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
/// Per-attempt HTTP timeout applied to the underlying reqwest client.
|
||||||
pub(crate) attempt_timeout: Duration,
|
pub(crate) attempt_timeout: Duration,
|
||||||
|
/// Whether to accept an unverified Vault server certificate. Gated on
|
||||||
|
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
|
||||||
|
pub(crate) skip_tls_verify: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VaultConnectionSettings {
|
impl VaultConnectionSettings {
|
||||||
@@ -499,6 +596,11 @@ impl VaultConnectionSettings {
|
|||||||
// operation-level retry policy.
|
// operation-level retry policy.
|
||||||
settings_builder.timeout(Some(self.attempt_timeout));
|
settings_builder.timeout(Some(self.attempt_timeout));
|
||||||
settings_builder.token(token);
|
settings_builder.token(token);
|
||||||
|
// Always set explicitly: left unset, vaultrs derives this from its own
|
||||||
|
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
|
||||||
|
// disable certificate verification behind the KMS configuration and its
|
||||||
|
// insecure-defaults gate.
|
||||||
|
settings_builder.verify(!self.skip_tls_verify);
|
||||||
|
|
||||||
if let Some(namespace) = &self.namespace {
|
if let Some(namespace) = &self.namespace {
|
||||||
settings_builder.namespace(Some(namespace.clone()));
|
settings_builder.namespace(Some(namespace.clone()));
|
||||||
@@ -551,6 +653,10 @@ impl VaultCredentialPolicy {
|
|||||||
refresh_safety_window_secs: Some(secs),
|
refresh_safety_window_secs: Some(secs),
|
||||||
..
|
..
|
||||||
}
|
}
|
||||||
|
| VaultAuthMethod::Kubernetes {
|
||||||
|
refresh_safety_window_secs: Some(secs),
|
||||||
|
..
|
||||||
|
}
|
||||||
| VaultAuthMethod::TokenFile {
|
| VaultAuthMethod::TokenFile {
|
||||||
refresh_safety_window_secs: Some(secs),
|
refresh_safety_window_secs: Some(secs),
|
||||||
..
|
..
|
||||||
@@ -584,15 +690,25 @@ pub(crate) struct VaultClientHandle {
|
|||||||
|
|
||||||
impl VaultClientHandle {
|
impl VaultClientHandle {
|
||||||
/// Absolute expiry of this generation's token.
|
/// Absolute expiry of this generation's token.
|
||||||
|
///
|
||||||
|
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
|
||||||
|
/// a value too large to add to `issued_at` would panic on the bare `+`. A
|
||||||
|
/// TTL that cannot be represented is indistinguishable from no expiry, so it
|
||||||
|
/// collapses to `None` — the same answer already given for the zero-lease
|
||||||
|
/// tokens Vault issues, which keeps the token in use and still fully
|
||||||
|
/// validated by Vault on every call.
|
||||||
fn expires_at(&self) -> Option<Instant> {
|
fn expires_at(&self) -> Option<Instant> {
|
||||||
self.lease.map(|lease| self.issued_at + lease.ttl)
|
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// When the renewal task should refresh this generation: half the TTL,
|
/// When the renewal task should refresh this generation: half the TTL,
|
||||||
/// leaving the second half as budget for retries before the fail-closed
|
/// leaving the second half as budget for retries before the fail-closed
|
||||||
/// window is reached.
|
/// window is reached.
|
||||||
|
///
|
||||||
|
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
|
||||||
|
/// leaving a token that never expires with nothing to renew.
|
||||||
fn renew_at(&self) -> Option<Instant> {
|
fn renew_at(&self) -> Option<Instant> {
|
||||||
self.lease.map(|lease| self.issued_at + lease.ttl / 2)
|
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -662,7 +778,7 @@ impl VaultCredentialProvider {
|
|||||||
let handle = self.current.load_full();
|
let handle = self.current.load_full();
|
||||||
if let Some(expires_at) = handle.expires_at() {
|
if let Some(expires_at) = handle.expires_at() {
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
if now + self.policy.safety_window >= expires_at {
|
if self.inside_safety_window(now, expires_at) {
|
||||||
return Err(KmsError::credentials_unavailable(format!(
|
return Err(KmsError::credentials_unavailable(format!(
|
||||||
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
|
||||||
handle.generation, self.policy.safety_window
|
handle.generation, self.policy.safety_window
|
||||||
@@ -672,6 +788,18 @@ impl VaultCredentialProvider {
|
|||||||
Ok(handle)
|
Ok(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the token expiring at `expires_at` is close enough to refuse.
|
||||||
|
///
|
||||||
|
/// `safety_window` reaches here from persisted configuration, so it is not
|
||||||
|
/// guaranteed to have passed this version's validation: a window too large
|
||||||
|
/// to add to the current instant would panic on the bare `+`. Such a window
|
||||||
|
/// means every token is always inside it, so saturating to "refuse" is both
|
||||||
|
/// the fail-closed answer and the one the arithmetic was reaching for.
|
||||||
|
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
|
||||||
|
now.checked_add(self.policy.safety_window)
|
||||||
|
.is_none_or(|deadline| deadline >= expires_at)
|
||||||
|
}
|
||||||
|
|
||||||
/// Publish the credential gauges for the generation currently installed.
|
/// Publish the credential gauges for the generation currently installed.
|
||||||
///
|
///
|
||||||
/// The fail-closed gauge re-evaluates the very gate
|
/// The fail-closed gauge re-evaluates the very gate
|
||||||
@@ -683,7 +811,7 @@ impl VaultCredentialProvider {
|
|||||||
let fail_closed = match handle.expires_at() {
|
let fail_closed = match handle.expires_at() {
|
||||||
Some(expires_at) => {
|
Some(expires_at) => {
|
||||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||||
now + self.policy.safety_window >= expires_at
|
self.inside_safety_window(now, expires_at)
|
||||||
}
|
}
|
||||||
// A generation without an expiry has no remaining TTL to report
|
// A generation without an expiry has no remaining TTL to report
|
||||||
// and can never lapse, so it can never fail closed either.
|
// and can never lapse, so it can never fail closed either.
|
||||||
@@ -860,7 +988,7 @@ impl Drop for CredentialTaskHandle {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::REDACTED_SECRET;
|
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET};
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||||
|
|
||||||
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
|
||||||
@@ -871,6 +999,7 @@ mod tests {
|
|||||||
address: "http://127.0.0.1:8200".to_string(),
|
address: "http://127.0.0.1:8200".to_string(),
|
||||||
namespace: Some("team-namespace".to_string()),
|
namespace: Some("team-namespace".to_string()),
|
||||||
attempt_timeout: Duration::from_secs(30),
|
attempt_timeout: Duration::from_secs(30),
|
||||||
|
skip_tls_verify: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1057,6 +1186,143 @@ mod tests {
|
|||||||
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
assert!(format!("{source:?}").contains("AppRoleLogin"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_kubernetes_auth_method_maps_to_login_source() {
|
||||||
|
let settings = test_settings();
|
||||||
|
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
|
||||||
|
.expect("kubernetes auth must map to a login source");
|
||||||
|
|
||||||
|
assert!(format!("{source:?}").contains("KubernetesLogin"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
|
||||||
|
/// path from persisted configuration, so the fail-closed comparison must
|
||||||
|
/// survive a window too large to add to the current instant. Before the
|
||||||
|
/// checked arithmetic this panicked with "overflow when adding duration to
|
||||||
|
/// instant" on the first request after a lease-bearing login.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
|
||||||
|
let (provider, _state) = scripted_provider(
|
||||||
|
Duration::from_secs(60),
|
||||||
|
true,
|
||||||
|
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let error = provider
|
||||||
|
.current()
|
||||||
|
.expect_err("a window wider than any lease must refuse the token");
|
||||||
|
assert!(
|
||||||
|
matches!(error, KmsError::CredentialsUnavailable { .. }),
|
||||||
|
"expected CredentialsUnavailable, got {error:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `lease_duration` is a bare u64 straight off the Vault response and forms
|
||||||
|
/// the other side of the same comparison, so an absurd one must not panic
|
||||||
|
/// either. It is indistinguishable from a non-expiring token, which is how
|
||||||
|
/// the zero-lease case already behaves.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
|
||||||
|
let (provider, _state) = scripted_provider(
|
||||||
|
Duration::from_secs(u64::MAX),
|
||||||
|
true,
|
||||||
|
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
provider
|
||||||
|
.current()
|
||||||
|
.expect("a token whose expiry cannot be represented must stay usable");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The configured flag has to reach the HTTP client, not just the config
|
||||||
|
/// struct: every generation (authenticated and login) builds its own client,
|
||||||
|
/// and a Vault with a self-signed certificate fails the handshake unless
|
||||||
|
/// each one carries the setting.
|
||||||
|
#[test]
|
||||||
|
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
|
||||||
|
for skip_tls_verify in [false, true] {
|
||||||
|
let settings = VaultConnectionSettings {
|
||||||
|
address: "https://vault.example.com:8200".to_string(),
|
||||||
|
namespace: None,
|
||||||
|
attempt_timeout: Duration::from_secs(30),
|
||||||
|
skip_tls_verify,
|
||||||
|
};
|
||||||
|
|
||||||
|
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
|
||||||
|
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
|
||||||
|
|
||||||
|
let login = settings.build_login_client().expect("login client must build");
|
||||||
|
assert_eq!(login.settings.verify, !skip_tls_verify);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
|
||||||
|
/// builder leaves it unset, which would disable certificate verification
|
||||||
|
/// without passing the KMS insecure-defaults gate.
|
||||||
|
#[test]
|
||||||
|
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
|
||||||
|
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
|
||||||
|
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
|
||||||
|
assert!(
|
||||||
|
client.settings.verify,
|
||||||
|
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The projected token is read fresh per login attempt and trimmed, so a
|
||||||
|
/// kubelet rotation is picked up without a restart and a trailing newline
|
||||||
|
/// does not corrupt the assertion sent to Vault.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
|
||||||
|
let dir = tempfile::tempdir().expect("temp dir");
|
||||||
|
let path = dir.path().join("token");
|
||||||
|
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
|
||||||
|
|
||||||
|
let login = KubernetesLogin::new(
|
||||||
|
&test_settings(),
|
||||||
|
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||||
|
"rustfs".to_string(),
|
||||||
|
path.clone(),
|
||||||
|
)
|
||||||
|
.expect("login source must build");
|
||||||
|
|
||||||
|
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
|
||||||
|
|
||||||
|
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
|
||||||
|
assert_eq!(
|
||||||
|
login.resolve_jwt().await.expect("second read").expose(),
|
||||||
|
"rotated-jwt",
|
||||||
|
"a rotated projected token must be picked up without a restart"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ServiceAccount token is re-read per attempt, so an unreadable or
|
||||||
|
/// empty one fails that attempt without reaching Vault; the refresh loop
|
||||||
|
/// keeps retrying, which is what lets a late projection heal the source.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
|
||||||
|
let dir = tempfile::tempdir().expect("temp dir");
|
||||||
|
let missing = dir.path().join("absent-token");
|
||||||
|
let empty = dir.path().join("empty-token");
|
||||||
|
tokio::fs::write(&empty, " \n").await.expect("write empty token");
|
||||||
|
|
||||||
|
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
|
||||||
|
let login =
|
||||||
|
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
|
||||||
|
.expect("login source must build");
|
||||||
|
|
||||||
|
let error = login
|
||||||
|
.acquire()
|
||||||
|
.await
|
||||||
|
.expect_err("an unusable ServiceAccount token must fail the attempt");
|
||||||
|
assert!(matches!(error.class, ErrorClass::Fatal));
|
||||||
|
assert!(error.error.to_string().contains(expected), "got {}", error.error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(start_paused = true)]
|
#[tokio::test(start_paused = true)]
|
||||||
async fn test_renewal_task_renews_at_half_ttl() {
|
async fn test_renewal_task_renews_at_half_ttl() {
|
||||||
let (provider, state) = scripted_provider(
|
let (provider, state) = scripted_provider(
|
||||||
|
|||||||
@@ -415,6 +415,7 @@ impl VaultTransitKmsClient {
|
|||||||
address: config.address.clone(),
|
address: config.address.clone(),
|
||||||
namespace: config.namespace.clone(),
|
namespace: config.namespace.clone(),
|
||||||
attempt_timeout: kms_config.effective_timeout(),
|
attempt_timeout: kms_config.effective_timeout(),
|
||||||
|
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
|
||||||
};
|
};
|
||||||
let source = token_source_for(&config.auth_method, &settings)?;
|
let source = token_source_for(&config.auth_method, &settings)?;
|
||||||
let policy = VaultCredentialPolicy::from_kms_config(
|
let policy = VaultCredentialPolicy::from_kms_config(
|
||||||
|
|||||||
@@ -450,6 +450,10 @@ impl VaultRestoreClient {
|
|||||||
address: target.address.clone(),
|
address: target.address.clone(),
|
||||||
namespace: target.namespace.clone(),
|
namespace: target.namespace.clone(),
|
||||||
attempt_timeout: kms_config.effective_timeout(),
|
attempt_timeout: kms_config.effective_timeout(),
|
||||||
|
// A restore target carries no TLS settings, so certificates are
|
||||||
|
// always verified: recovery is the last path that should accept an
|
||||||
|
// unauthenticated Vault.
|
||||||
|
skip_tls_verify: false,
|
||||||
};
|
};
|
||||||
let source = token_source_for(&target.auth_method, &settings)?;
|
let source = token_source_for(&target.auth_method, &settings)?;
|
||||||
let policy = VaultCredentialPolicy::from_kms_config(
|
let policy = VaultCredentialPolicy::from_kms_config(
|
||||||
|
|||||||
+295
-54
@@ -25,6 +25,10 @@ use url::Url;
|
|||||||
|
|
||||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||||
|
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
|
||||||
|
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
|
||||||
|
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
|
||||||
|
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
|
||||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||||
@@ -35,6 +39,9 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
|||||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||||
|
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
|
||||||
|
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
|
||||||
|
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
|
||||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||||
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
/// Age in whole seconds beyond which a key is reported as due for rotation;
|
||||||
@@ -45,6 +52,9 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
|
|||||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||||
|
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
|
||||||
|
/// Where the kubelet projects a pod's ServiceAccount token by default.
|
||||||
|
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||||
|
|
||||||
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
|
||||||
///
|
///
|
||||||
@@ -84,6 +94,14 @@ fn default_vault_approle_mount() -> String {
|
|||||||
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_vault_kubernetes_mount() -> String {
|
||||||
|
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_vault_kubernetes_jwt_path() -> PathBuf {
|
||||||
|
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
|
||||||
|
}
|
||||||
|
|
||||||
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
|
||||||
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
|
||||||
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
|
||||||
@@ -490,6 +508,23 @@ pub enum VaultAuthMethod {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
refresh_safety_window_secs: Option<u64>,
|
refresh_safety_window_secs: Option<u64>,
|
||||||
},
|
},
|
||||||
|
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
|
||||||
|
/// for a lease-bound Vault token that is renewed in the background.
|
||||||
|
Kubernetes {
|
||||||
|
/// Vault role bound to this ServiceAccount.
|
||||||
|
role: String,
|
||||||
|
/// Kubernetes auth engine mount path.
|
||||||
|
#[serde(default = "default_vault_kubernetes_mount")]
|
||||||
|
mount: String,
|
||||||
|
/// Projected ServiceAccount token to present. Re-read on every login so
|
||||||
|
/// a token the kubelet rotates is picked up without a restart.
|
||||||
|
#[serde(default = "default_vault_kubernetes_jwt_path")]
|
||||||
|
jwt_path: PathBuf,
|
||||||
|
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
|
||||||
|
/// per-attempt timeout.
|
||||||
|
#[serde(default)]
|
||||||
|
refresh_safety_window_secs: Option<u64>,
|
||||||
|
},
|
||||||
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
|
||||||
/// the token is read from `path` and re-read periodically so a token
|
/// the token is read from `path` and re-read periodically so a token
|
||||||
/// rotated by the agent is picked up without a restart.
|
/// rotated by the agent is picked up without a restart.
|
||||||
@@ -520,6 +555,16 @@ impl VaultAuthMethod {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Kubernetes authentication with the default mount and projected token path.
|
||||||
|
pub fn kubernetes(role: String) -> Self {
|
||||||
|
Self::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount: default_vault_kubernetes_mount(),
|
||||||
|
jwt_path: default_vault_kubernetes_jwt_path(),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Agent-managed token file with the default poll interval.
|
/// Agent-managed token file with the default poll interval.
|
||||||
pub fn token_file(path: PathBuf) -> Self {
|
pub fn token_file(path: PathBuf) -> Self {
|
||||||
Self::TokenFile {
|
Self::TokenFile {
|
||||||
@@ -548,6 +593,20 @@ impl fmt::Debug for VaultAuthMethod {
|
|||||||
.field("mount", mount)
|
.field("mount", mount)
|
||||||
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||||
.finish(),
|
.finish(),
|
||||||
|
// No redaction: the role and mount name a Vault binding, and the
|
||||||
|
// ServiceAccount token itself is never held on this type.
|
||||||
|
Self::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount,
|
||||||
|
jwt_path,
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
} => f
|
||||||
|
.debug_struct("Kubernetes")
|
||||||
|
.field("role", role)
|
||||||
|
.field("mount", mount)
|
||||||
|
.field("jwt_path", jwt_path)
|
||||||
|
.field("refresh_safety_window_secs", refresh_safety_window_secs)
|
||||||
|
.finish(),
|
||||||
Self::TokenFile {
|
Self::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -1028,50 +1087,12 @@ impl KmsConfig {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
KmsBackend::VaultKv2 => {
|
KmsBackend::VaultKv2 => {
|
||||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
config.backend_config =
|
||||||
let auth_method = vault_auth_method_from_env()?;
|
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?));
|
||||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
|
||||||
|
|
||||||
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
|
|
||||||
Some(path) => {
|
|
||||||
tracing::warn!(
|
|
||||||
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
|
||||||
);
|
|
||||||
path
|
|
||||||
}
|
|
||||||
None => default_vault_kv2_mount_path(),
|
|
||||||
};
|
|
||||||
|
|
||||||
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
|
|
||||||
address,
|
|
||||||
auth_method,
|
|
||||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
|
||||||
mount_path,
|
|
||||||
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
|
||||||
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
|
||||||
tls: vault_tls_config(skip_tls_verify),
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
KmsBackend::VaultTransit => {
|
KmsBackend::VaultTransit => {
|
||||||
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
|
config.backend_config =
|
||||||
let auth_method = vault_auth_method_from_env()?;
|
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?));
|
||||||
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
|
|
||||||
|
|
||||||
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
|
||||||
address,
|
|
||||||
auth_method,
|
|
||||||
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
|
|
||||||
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
|
|
||||||
metadata_kv_mount: get_env_str(
|
|
||||||
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
|
||||||
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
|
||||||
),
|
|
||||||
metadata_key_prefix: get_env_str(
|
|
||||||
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
|
|
||||||
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
|
|
||||||
),
|
|
||||||
tls: vault_tls_config(skip_tls_verify),
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
KmsBackend::Static => {
|
KmsBackend::Static => {
|
||||||
// Read from file first, then fall back to direct env var
|
// Read from file first, then fall back to direct env var
|
||||||
@@ -1202,6 +1223,78 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
|||||||
path.starts_with(std::env::temp_dir())
|
path.starts_with(std::env::temp_dir())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Command-line values that take precedence over the matching environment
|
||||||
|
/// variables when assembling a Vault backend configuration.
|
||||||
|
///
|
||||||
|
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
|
||||||
|
/// reads, so these are only set when the operator passed an explicit flag.
|
||||||
|
///
|
||||||
|
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
|
||||||
|
/// redacting `Debug` impls elsewhere in this module exist because a derived one
|
||||||
|
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
|
||||||
|
/// error instead of a leak.
|
||||||
|
#[derive(Default, Clone, Copy)]
|
||||||
|
pub struct VaultCliOverrides<'a> {
|
||||||
|
pub address: Option<&'a str>,
|
||||||
|
pub token: Option<&'a str>,
|
||||||
|
pub mount_path: Option<&'a str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Assemble the Vault KV2 backend configuration from the environment.
|
||||||
|
///
|
||||||
|
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
|
||||||
|
/// so both resolve the same auth method, namespace, TLS and mount settings.
|
||||||
|
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
|
||||||
|
let mount_path = match overrides
|
||||||
|
.mount_path
|
||||||
|
.map(str::to_string)
|
||||||
|
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
|
||||||
|
{
|
||||||
|
Some(path) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
|
||||||
|
);
|
||||||
|
path
|
||||||
|
}
|
||||||
|
None => default_vault_kv2_mount_path(),
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(VaultConfig {
|
||||||
|
address: vault_address_from_env(overrides.address),
|
||||||
|
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||||
|
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||||
|
mount_path,
|
||||||
|
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
|
||||||
|
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
|
||||||
|
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Assemble the Vault Transit backend configuration from the environment.
|
||||||
|
///
|
||||||
|
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
|
||||||
|
/// points share it.
|
||||||
|
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
|
||||||
|
Ok(VaultTransitConfig {
|
||||||
|
address: vault_address_from_env(overrides.address),
|
||||||
|
auth_method: vault_auth_method_from_env(overrides.token)?,
|
||||||
|
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
|
||||||
|
mount_path: overrides
|
||||||
|
.mount_path
|
||||||
|
.map(str::to_string)
|
||||||
|
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
|
||||||
|
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
|
||||||
|
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
|
||||||
|
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn vault_address_from_env(override_value: Option<&str>) -> String {
|
||||||
|
override_value
|
||||||
|
.map(str::to_string)
|
||||||
|
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolve the Vault auth method from environment variables.
|
/// Resolve the Vault auth method from environment variables.
|
||||||
///
|
///
|
||||||
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
|
||||||
@@ -1209,27 +1302,59 @@ fn is_under_temp_dir(path: &Path) -> bool {
|
|||||||
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
|
||||||
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
|
||||||
/// file taking precedence. Without a role id the legacy token flow applies.
|
/// file taking precedence. Without a role id the legacy token flow applies.
|
||||||
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
|
///
|
||||||
|
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
|
||||||
|
/// presents the pod's projected ServiceAccount token.
|
||||||
|
///
|
||||||
|
/// `token_override` carries a token supplied on the command line; it stands in
|
||||||
|
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
|
||||||
|
/// so a flag and the variable it mirrors select the same method.
|
||||||
|
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
|
||||||
|
let token = token_override
|
||||||
|
.map(str::to_string)
|
||||||
|
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
|
||||||
|
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
|
||||||
|
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
|
||||||
|
|
||||||
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
|
||||||
// A token file names one authoritative credential source; combining it
|
// A token file names one authoritative credential source; combining it
|
||||||
// with another one would leave the effective identity ambiguous, so
|
// with another one would leave the effective identity ambiguous, so
|
||||||
// that is a configuration error rather than a precedence rule.
|
// that is a configuration error rather than a precedence rule.
|
||||||
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
|
for (name, configured) in [
|
||||||
return Err(KmsError::configuration_error(format!(
|
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
|
||||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
|
||||||
)));
|
(ENV_KMS_VAULT_TOKEN, token.is_some()),
|
||||||
}
|
] {
|
||||||
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
|
if configured {
|
||||||
return Err(KmsError::configuration_error(format!(
|
return Err(KmsError::configuration_error(format!(
|
||||||
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
|
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method"
|
||||||
)));
|
)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
|
if let Some(role) = kubernetes_role {
|
||||||
|
// Unlike a leftover static token, a second login method is never a
|
||||||
|
// stale remnant: both were configured deliberately and neither can be
|
||||||
|
// ranked over the other.
|
||||||
|
if role_id.is_some() {
|
||||||
|
return Err(KmsError::configuration_error(format!(
|
||||||
|
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
return Ok(VaultAuthMethod::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
|
||||||
|
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
|
||||||
|
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some(role_id) = role_id else {
|
||||||
return Ok(VaultAuthMethod::Token {
|
return Ok(VaultAuthMethod::Token {
|
||||||
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
|
token: token.unwrap_or_else(|| "dev-token".to_string()),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1273,6 +1398,22 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
|
|||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} => {
|
||||||
|
if role.is_empty() {
|
||||||
|
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
|
||||||
|
}
|
||||||
|
if mount.is_empty() {
|
||||||
|
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
|
||||||
|
}
|
||||||
|
if jwt_path.as_os_str().is_empty() {
|
||||||
|
return Err(KmsError::configuration_error(format!(
|
||||||
|
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
VaultAuthMethod::TokenFile {
|
VaultAuthMethod::TokenFile {
|
||||||
path,
|
path,
|
||||||
poll_interval_secs,
|
poll_interval_secs,
|
||||||
@@ -1976,6 +2117,106 @@ mod tests {
|
|||||||
.expect("well-formed token file auth must validate");
|
.expect("well-formed token file auth must validate");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A Kubernetes role alone configures the method: the credential is the
|
||||||
|
/// pod's projected ServiceAccount token, so nothing secret is in the
|
||||||
|
/// environment and the mount and token path fall back to the cluster
|
||||||
|
/// defaults.
|
||||||
|
#[test]
|
||||||
|
fn test_from_env_selects_kubernetes() {
|
||||||
|
with_vars(
|
||||||
|
vec![
|
||||||
|
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||||
|
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
|
||||||
|
(ENV_KMS_VAULT_TOKEN, None),
|
||||||
|
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||||
|
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
let VaultAuthMethod::Kubernetes {
|
||||||
|
role,
|
||||||
|
mount,
|
||||||
|
jwt_path,
|
||||||
|
refresh_safety_window_secs,
|
||||||
|
} = &vault.auth_method
|
||||||
|
else {
|
||||||
|
panic!(
|
||||||
|
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||||
|
vault.auth_method
|
||||||
|
);
|
||||||
|
};
|
||||||
|
assert_eq!(role, "rustfs");
|
||||||
|
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||||
|
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||||
|
assert_eq!(refresh_safety_window_secs, &None);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
|
||||||
|
with_vars(
|
||||||
|
vec![
|
||||||
|
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
|
||||||
|
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
|
||||||
|
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
|
||||||
|
(ENV_KMS_VAULT_TOKEN, None),
|
||||||
|
(ENV_KMS_VAULT_TOKEN_FILE, None),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
|
||||||
|
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
|
||||||
|
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_validate_rejects_bad_kubernetes_settings() {
|
||||||
|
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
|
||||||
|
backend: KmsBackend::VaultTransit,
|
||||||
|
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||||
|
address: "https://vault.example.com:8200".to_string(),
|
||||||
|
auth_method,
|
||||||
|
..Default::default()
|
||||||
|
})),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
|
||||||
|
.validate()
|
||||||
|
.expect_err("an empty kubernetes role must be rejected");
|
||||||
|
assert!(error.to_string().contains("role"), "got {error}");
|
||||||
|
|
||||||
|
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||||
|
role: "rustfs".to_string(),
|
||||||
|
mount: String::new(),
|
||||||
|
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
})
|
||||||
|
.validate()
|
||||||
|
.expect_err("an empty kubernetes mount must be rejected");
|
||||||
|
assert!(error.to_string().contains("mount"), "got {error}");
|
||||||
|
|
||||||
|
let error = vault_config(VaultAuthMethod::Kubernetes {
|
||||||
|
role: "rustfs".to_string(),
|
||||||
|
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
|
||||||
|
jwt_path: PathBuf::new(),
|
||||||
|
refresh_safety_window_secs: None,
|
||||||
|
})
|
||||||
|
.validate()
|
||||||
|
.expect_err("an empty ServiceAccount token path must be rejected");
|
||||||
|
assert!(error.to_string().contains("token path"), "got {error}");
|
||||||
|
|
||||||
|
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
|
||||||
|
.validate()
|
||||||
|
.expect("well-formed kubernetes auth must validate");
|
||||||
|
}
|
||||||
|
|
||||||
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
|
||||||
/// empty one names a path no Vault engine answers. The Transit backend
|
/// empty one names a path no Vault engine answers. The Transit backend
|
||||||
/// already rejects its own empty mounts; this closes the same gap on the
|
/// already rejects its own empty mounts; this closes the same gap on the
|
||||||
|
|||||||
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
|
|||||||
pub api_version: Option<String>,
|
pub api_version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
pub struct SRIAMUser {
|
pub struct SRIAMUser {
|
||||||
#[serde(rename = "accessKey", default)]
|
#[serde(rename = "accessKey", default)]
|
||||||
pub access_key: String,
|
pub access_key: String,
|
||||||
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
|
|||||||
pub api_version: Option<String>,
|
pub api_version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
pub struct SRGroupInfo {
|
pub struct SRGroupInfo {
|
||||||
#[serde(rename = "updateReq", default)]
|
#[serde(rename = "updateReq", default)]
|
||||||
pub update_req: GroupAddRemove,
|
pub update_req: GroupAddRemove,
|
||||||
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
|
|||||||
pub api_version: Option<String>,
|
pub api_version: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
pub struct SRIAMItem {
|
pub struct SRIAMItem {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub r#type: String,
|
pub r#type: String,
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
|
RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
|
||||||
|
|
||||||
For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
|
For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first.
|
||||||
|
|
||||||
## Backend comparison
|
## Backend comparison
|
||||||
|
|
||||||
|
|||||||
@@ -8,9 +8,12 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti
|
|||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments |
|
| Static token | `Token` | Whatever the operator provisioned; RustFS never renews it | None | Development; short-lived experiments |
|
||||||
| AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar |
|
| AppRole | `AppRole` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production without a Vault Agent sidecar |
|
||||||
|
| Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute |
|
||||||
| Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth |
|
| Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth |
|
||||||
|
|
||||||
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` or an explicit `RUSTFS_KMS_VAULT_TOKEN` is rejected at startup with a configuration error, because the effective identity would be ambiguous.
|
Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity.
|
||||||
|
|
||||||
|
All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`.
|
||||||
|
|
||||||
The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification.
|
The default `dev-token` fallback for `RUSTFS_KMS_VAULT_TOKEN` is rejected outside explicit development mode (`RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`), as are plain-HTTP Vault addresses and disabled TLS verification.
|
||||||
|
|
||||||
@@ -56,7 +59,44 @@ Deliver the SecretID out of band — a secrets-manager-mounted file, an init-con
|
|||||||
|
|
||||||
The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login.
|
The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login.
|
||||||
|
|
||||||
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip) and is retried on the normal refresh cadence, so repairing the file heals the backend without a restart.
|
An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart.
|
||||||
|
|
||||||
|
## Kubernetes authentication
|
||||||
|
|
||||||
|
On Kubernetes this is the method to prefer: the pod's own ServiceAccount is the identity, so there is no credential to distribute, rotate, or leak into a Secret.
|
||||||
|
|
||||||
|
### Vault-side setup
|
||||||
|
|
||||||
|
```shell
|
||||||
|
vault auth enable kubernetes
|
||||||
|
|
||||||
|
vault write auth/kubernetes/config \
|
||||||
|
kubernetes_host="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT"
|
||||||
|
|
||||||
|
vault write auth/kubernetes/role/rustfs \
|
||||||
|
bound_service_account_names=rustfs \
|
||||||
|
bound_service_account_namespaces=rustfs \
|
||||||
|
token_policies=rustfs-kms \
|
||||||
|
token_ttl=1h
|
||||||
|
```
|
||||||
|
|
||||||
|
As with AppRole, keep `token_ttl` comfortably above the RustFS per-attempt timeout (default 30s).
|
||||||
|
|
||||||
|
### RustFS configuration
|
||||||
|
|
||||||
|
```shell
|
||||||
|
RUSTFS_KMS_BACKEND=vault-transit # or "vault" for the KV2 backend
|
||||||
|
RUSTFS_KMS_VAULT_ADDRESS=https://vault.vault.svc.cluster.local:8200
|
||||||
|
RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs
|
||||||
|
# Optional, defaults to "kubernetes":
|
||||||
|
# RUSTFS_KMS_VAULT_KUBERNETES_MOUNT=kubernetes
|
||||||
|
# Optional, defaults to the kubelet's projected token path:
|
||||||
|
# RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token
|
||||||
|
```
|
||||||
|
|
||||||
|
RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart.
|
||||||
|
|
||||||
|
A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own.
|
||||||
|
|
||||||
## Vault Agent token file
|
## Vault Agent token file
|
||||||
|
|
||||||
@@ -101,13 +141,13 @@ If the agent stops refreshing the file that is fine — RustFS re-reads the same
|
|||||||
|
|
||||||
## Fail-closed window
|
## Fail-closed window
|
||||||
|
|
||||||
For lease-bound credentials (AppRole tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
|
For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side.
|
||||||
|
|
||||||
- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it.
|
- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it.
|
||||||
- Override: `refresh_safety_window_secs` on the `AppRole` or `TokenFile` auth configuration.
|
- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration.
|
||||||
- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise.
|
- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise.
|
||||||
|
|
||||||
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole) or two poll intervals (token file).
|
The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file).
|
||||||
|
|
||||||
### Troubleshooting
|
### Troubleshooting
|
||||||
|
|
||||||
@@ -117,6 +157,8 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r
|
|||||||
| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file |
|
| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file |
|
||||||
| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider |
|
| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider |
|
||||||
| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider |
|
| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider |
|
||||||
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, token file |
|
| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server |
|
||||||
|
| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider |
|
||||||
|
| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file |
|
||||||
|
|
||||||
When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault.
|
When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault.
|
||||||
|
|||||||
@@ -286,6 +286,7 @@ fn auth_method_kind(auth: &VaultAuthMethod) -> String {
|
|||||||
match auth {
|
match auth {
|
||||||
VaultAuthMethod::Token { .. } => "token",
|
VaultAuthMethod::Token { .. } => "token",
|
||||||
VaultAuthMethod::AppRole { .. } => "approle",
|
VaultAuthMethod::AppRole { .. } => "approle",
|
||||||
|
VaultAuthMethod::Kubernetes { .. } => "kubernetes",
|
||||||
VaultAuthMethod::TokenFile { .. } => "token-file",
|
VaultAuthMethod::TokenFile { .. } => "token-file",
|
||||||
}
|
}
|
||||||
.to_string()
|
.to_string()
|
||||||
@@ -484,7 +485,10 @@ fn business_trust_root_secrets(config: &KmsConfig) -> Vec<Zeroizing<String>> {
|
|||||||
secrets.push(Zeroizing::new(role_id.clone()));
|
secrets.push(Zeroizing::new(role_id.clone()));
|
||||||
secrets.push(Zeroizing::new(secret_id.clone()));
|
secrets.push(Zeroizing::new(secret_id.clone()));
|
||||||
}
|
}
|
||||||
VaultAuthMethod::TokenFile { .. } => {}
|
// Kubernetes and TokenFile hold no inline plaintext credential: the
|
||||||
|
// ServiceAccount token and the agent-managed token live in files, and
|
||||||
|
// the role names a Vault binding rather than half a credential pair.
|
||||||
|
VaultAuthMethod::Kubernetes { .. } | VaultAuthMethod::TokenFile { .. } => {}
|
||||||
};
|
};
|
||||||
|
|
||||||
match &config.backend_config {
|
match &config.backend_config {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
+178
-37
@@ -304,30 +304,37 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
|||||||
Ok(kms_config)
|
Ok(kms_config)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Collect the Vault settings the command line owns.
|
||||||
|
///
|
||||||
|
/// Everything else — auth method, namespace, TLS, KV mount and metadata paths —
|
||||||
|
/// is resolved from the environment by the KMS crate, so this path and
|
||||||
|
/// [`rustfs_kms::config::KmsConfig::from_env`] cannot drift apart. The address
|
||||||
|
/// stays required here so a missing one is still named instead of silently
|
||||||
|
/// falling back to the crate's localhost default.
|
||||||
|
fn vault_cli_overrides<'a>(
|
||||||
|
cfg: &'a config::Config,
|
||||||
|
backend_name: &str,
|
||||||
|
) -> std::io::Result<rustfs_kms::config::VaultCliOverrides<'a>> {
|
||||||
|
let address = cfg
|
||||||
|
.kms_vault_address
|
||||||
|
.as_deref()
|
||||||
|
.ok_or_else(|| Error::other(format!("Vault address is required for {backend_name} backend")))?;
|
||||||
|
|
||||||
|
Ok(rustfs_kms::config::VaultCliOverrides {
|
||||||
|
address: Some(address),
|
||||||
|
token: cfg.kms_vault_token.as_deref(),
|
||||||
|
mount_path: cfg.kms_vault_mount_path.as_deref(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Build KMS configuration for Vault backend
|
/// Build KMS configuration for Vault backend
|
||||||
fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
|
fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
|
||||||
let vault_address = cfg
|
let backend_config = rustfs_kms::config::vault_kv2_config_from_env(vault_cli_overrides(cfg, "vault")?)
|
||||||
.kms_vault_address
|
.map_err(|e| Error::other(format!("Vault KMS configuration failed: {e}")))?;
|
||||||
.as_ref()
|
|
||||||
.ok_or_else(|| Error::other("Vault address is required for vault backend"))?;
|
|
||||||
let vault_token = cfg
|
|
||||||
.kms_vault_token
|
|
||||||
.as_ref()
|
|
||||||
.ok_or_else(|| Error::other("Vault token is required for vault backend"))?;
|
|
||||||
|
|
||||||
let kms_config = rustfs_kms::config::KmsConfig {
|
let kms_config = rustfs_kms::config::KmsConfig {
|
||||||
backend: rustfs_kms::config::KmsBackend::VaultKv2,
|
backend: rustfs_kms::config::KmsBackend::VaultKv2,
|
||||||
backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(rustfs_kms::config::VaultConfig {
|
backend_config: rustfs_kms::config::BackendConfig::VaultKv2(Box::new(backend_config)),
|
||||||
address: vault_address.clone(),
|
|
||||||
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
|
|
||||||
token: vault_token.clone(),
|
|
||||||
},
|
|
||||||
namespace: None,
|
|
||||||
mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()),
|
|
||||||
kv_mount: "secret".to_string(),
|
|
||||||
key_path_prefix: "rustfs/kms/keys".to_string(),
|
|
||||||
tls: None,
|
|
||||||
})),
|
|
||||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||||
default_key_id: cfg.kms_default_key_id.clone(),
|
default_key_id: cfg.kms_default_key_id.clone(),
|
||||||
@@ -344,26 +351,12 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
|||||||
|
|
||||||
/// Build KMS configuration for Vault Transit backend
|
/// Build KMS configuration for Vault Transit backend
|
||||||
fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
|
fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
|
||||||
let vault_address = cfg
|
let backend_config = rustfs_kms::config::vault_transit_config_from_env(vault_cli_overrides(cfg, "vault-transit")?)
|
||||||
.kms_vault_address
|
.map_err(|e| Error::other(format!("Vault Transit KMS configuration failed: {e}")))?;
|
||||||
.as_ref()
|
|
||||||
.ok_or_else(|| Error::other("Vault address is required for vault-transit backend"))?;
|
|
||||||
let vault_token = cfg
|
|
||||||
.kms_vault_token
|
|
||||||
.as_ref()
|
|
||||||
.ok_or_else(|| Error::other("Vault token is required for vault-transit backend"))?;
|
|
||||||
|
|
||||||
let kms_config = rustfs_kms::config::KmsConfig {
|
let kms_config = rustfs_kms::config::KmsConfig {
|
||||||
backend: rustfs_kms::config::KmsBackend::VaultTransit,
|
backend: rustfs_kms::config::KmsBackend::VaultTransit,
|
||||||
backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(rustfs_kms::config::VaultTransitConfig {
|
backend_config: rustfs_kms::config::BackendConfig::VaultTransit(Box::new(backend_config)),
|
||||||
address: vault_address.clone(),
|
|
||||||
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
|
|
||||||
token: vault_token.clone(),
|
|
||||||
},
|
|
||||||
namespace: None,
|
|
||||||
mount_path: cfg.kms_vault_mount_path.clone().unwrap_or_else(|| "transit".to_string()),
|
|
||||||
..rustfs_kms::config::VaultTransitConfig::default()
|
|
||||||
})),
|
|
||||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||||
default_key_id: cfg.kms_default_key_id.clone(),
|
default_key_id: cfg.kms_default_key_id.clone(),
|
||||||
@@ -1405,7 +1398,10 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{build_aws_kms_config, notification_config_to_event_rules, resolve_buffer_profile_config};
|
use super::{
|
||||||
|
build_aws_kms_config, build_vault_kms_config, build_vault_transit_kms_config, notification_config_to_event_rules,
|
||||||
|
resolve_buffer_profile_config,
|
||||||
|
};
|
||||||
use crate::config::{BufferConfig, WorkloadProfile};
|
use crate::config::{BufferConfig, WorkloadProfile};
|
||||||
use rustfs_config::KI_B;
|
use rustfs_config::KI_B;
|
||||||
use rustfs_s3_types::EventName;
|
use rustfs_s3_types::EventName;
|
||||||
@@ -1499,6 +1495,151 @@ mod tests {
|
|||||||
assert!(err.to_string().contains("Invalid ARN"), "unexpected error: {err}");
|
assert!(err.to_string().contains("Invalid ARN"), "unexpected error: {err}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn vault_kms_test_config(backend: &str) -> crate::config::Config {
|
||||||
|
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-vault-kms".to_string()]);
|
||||||
|
config.kms_enable = true;
|
||||||
|
config.kms_backend = backend.to_string();
|
||||||
|
config.kms_vault_address = Some("https://vault.example.com:8200".to_string());
|
||||||
|
config
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The Vault auth method and the settings the CLI has no flag for come from
|
||||||
|
/// the environment, so startup and `KmsConfig::from_env` cannot disagree.
|
||||||
|
/// Regression: startup used to hardcode token auth and require a token,
|
||||||
|
/// which made every non-token method unreachable through `RUSTFS_KMS_ENABLE`.
|
||||||
|
#[test]
|
||||||
|
fn build_vault_transit_kms_config_resolves_auth_and_mounts_from_env() {
|
||||||
|
let config = temp_env::with_vars(
|
||||||
|
[
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN", None),
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", Some("env-role-id")),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID", Some("env-secret-id")),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_NAMESPACE", Some("team-a")),
|
||||||
|
("RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT", Some("rustfs-kv")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
|
||||||
|
.expect("vault transit KMS configuration should build")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
let rustfs_kms::config::VaultAuthMethod::AppRole { role_id, secret_id, .. } = &vault.auth_method else {
|
||||||
|
panic!("approle in the environment must select AppRole auth, got {:?}", vault.auth_method);
|
||||||
|
};
|
||||||
|
assert_eq!(role_id, "env-role-id");
|
||||||
|
assert_eq!(secret_id, "env-secret-id");
|
||||||
|
assert_eq!(vault.namespace.as_deref(), Some("team-a"));
|
||||||
|
assert_eq!(vault.metadata_kv_mount, "rustfs-kv");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Kubernetes auth needs no credential in the environment at all: the role
|
||||||
|
/// selects it and the pod's projected ServiceAccount token supplies the rest.
|
||||||
|
#[test]
|
||||||
|
fn build_vault_transit_kms_config_selects_kubernetes_auth() {
|
||||||
|
let config = temp_env::with_vars(
|
||||||
|
[
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN", None),
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_MOUNT", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH", None),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
|
||||||
|
.expect("vault transit KMS configuration should build")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
let rustfs_kms::config::VaultAuthMethod::Kubernetes {
|
||||||
|
role, mount, jwt_path, ..
|
||||||
|
} = &vault.auth_method
|
||||||
|
else {
|
||||||
|
panic!(
|
||||||
|
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
|
||||||
|
vault.auth_method
|
||||||
|
);
|
||||||
|
};
|
||||||
|
assert_eq!(role, "rustfs");
|
||||||
|
assert_eq!(mount, rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
|
||||||
|
assert_eq!(jwt_path, std::path::Path::new(rustfs_kms::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Two credential sources leave the effective identity ambiguous, so
|
||||||
|
/// startup refuses rather than picking one.
|
||||||
|
#[test]
|
||||||
|
fn build_vault_kms_config_refuses_two_auth_methods() {
|
||||||
|
temp_env::with_vars(
|
||||||
|
[
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN", None),
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN_FILE", Some("/run/vault-agent/token")),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", Some("rustfs")),
|
||||||
|
],
|
||||||
|
|| {
|
||||||
|
let error = build_vault_kms_config(&vault_kms_test_config("vault"))
|
||||||
|
.expect_err("two Vault auth methods must not start the server");
|
||||||
|
assert!(error.to_string().contains("exactly one"), "unexpected error: {error}");
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The KV2 backend has its own builder, so the key-location settings have
|
||||||
|
/// to be proven separately from the Transit one: pointing at the wrong KV
|
||||||
|
/// mount or prefix makes existing keys look absent.
|
||||||
|
#[test]
|
||||||
|
fn build_vault_kms_config_resolves_kv_mount_and_prefix_from_env() {
|
||||||
|
let config = temp_env::with_vars(
|
||||||
|
[
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")),
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KV_MOUNT", Some("rustfs-kv")),
|
||||||
|
("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")),
|
||||||
|
],
|
||||||
|
|| build_vault_kms_config(&vault_kms_test_config("vault")).expect("vault KV2 KMS configuration should build"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let vault = config.vault_config().expect("vault kv2 backend config");
|
||||||
|
assert_eq!(vault.kv_mount, "rustfs-kv");
|
||||||
|
assert_eq!(vault.key_path_prefix, "tenant/keys");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Skipping TLS verification was silently dropped on this path before, so
|
||||||
|
/// an operator who asked for it still got a verified connection. Now that it
|
||||||
|
/// is honoured it must fail closed without the development opt-in, rather
|
||||||
|
/// than quietly downgrading the Vault connection.
|
||||||
|
#[test]
|
||||||
|
fn build_vault_transit_kms_config_refuses_skip_tls_verify_without_opt_in() {
|
||||||
|
let vars = [
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN", Some("a-real-token")),
|
||||||
|
("RUSTFS_KMS_VAULT_TOKEN_FILE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_APPROLE_ROLE_ID", None),
|
||||||
|
("RUSTFS_KMS_VAULT_KUBERNETES_ROLE", None),
|
||||||
|
("RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY", Some("true")),
|
||||||
|
];
|
||||||
|
|
||||||
|
temp_env::with_vars(vars, || {
|
||||||
|
let error = build_vault_transit_kms_config(&vault_kms_test_config("vault-transit"))
|
||||||
|
.expect_err("skipping TLS verification must not start the server");
|
||||||
|
assert!(error.to_string().contains("TLS"), "unexpected error: {error}");
|
||||||
|
});
|
||||||
|
|
||||||
|
temp_env::with_vars(vars, || {
|
||||||
|
let mut cfg = vault_kms_test_config("vault-transit");
|
||||||
|
cfg.kms_allow_insecure_dev_defaults = true;
|
||||||
|
let config = build_vault_transit_kms_config(&cfg).expect("the development opt-in should accept skip-verify");
|
||||||
|
let vault = config.vault_transit_config().expect("vault transit backend config");
|
||||||
|
assert!(vault.tls.as_ref().is_some_and(|tls| tls.skip_verify));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
fn aws_kms_test_config() -> crate::config::Config {
|
fn aws_kms_test_config() -> crate::config::Config {
|
||||||
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]);
|
let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]);
|
||||||
config.kms_enable = true;
|
config.kms_enable = true;
|
||||||
|
|||||||
Reference in New Issue
Block a user