Merge main into validation runner

This commit is contained in:
overtrue
2026-10-28 20:16:48 +08:00
33 changed files with 4743 additions and 109 deletions
+2 -2
View File
@@ -128,7 +128,7 @@ setup = 'ecstore-large-stack'
# path on a real store; measured with RUST_MIN_STACK on 2026-10-15 they need
# 2016-2040 KiB of the 2 MiB libtest default and overflowed on Linux CI.
[[profile.default.scripts]]
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
setup = 'ecstore-base-stack'
[[profile.default.scripts]]
@@ -419,7 +419,7 @@ setup = 'ecstore-large-stack'
# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
[[profile.ci.scripts]]
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
+9
View File
@@ -371,6 +371,15 @@ pub mod compression {
pub mod config {
pub mod com {
/// Preserve empty existing objects so callers can distinguish corruption from absence.
pub async fn read_config_preserve_empty_with_metadata_opts(
api: std::sync::Arc<crate::store::ECStore>,
file: &str,
opts: &crate::object_api::ObjectOptions,
) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
}
pub use crate::config::com::{
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
+12
View File
@@ -488,6 +488,18 @@ where
Ok(data)
}
/// Read config bytes and metadata while preserving an existing empty payload.
pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
api: Arc<S>,
file: &str,
opts: &ObjectOptions,
) -> Result<(Vec<u8>, ObjectInfo)>
where
S: EcstoreObjectIO,
{
read_config_with_metadata_inner(api, file, opts, true, None).await
}
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
@@ -40,9 +40,35 @@ pub(crate) mod test_hooks {
use std::sync::{Arc, Mutex};
type Hook = Arc<dyn Fn() -> BoxFuture<'static, ()> + Send + Sync>;
type TransactionLockHook = Arc<dyn Fn(&rustfs_lock::NamespaceLockGuard) + Send + Sync>;
static BEFORE_GC_FINALIZE: Mutex<Vec<(String, String, Hook)>> = Mutex::new(Vec::new());
static CLEANUP_ATTEMPTS: Mutex<Vec<(String, String)>> = Mutex::new(Vec::new());
static AFTER_TRANSACTION_LOCK: Mutex<Vec<(String, String, TransactionLockHook)>> = Mutex::new(Vec::new());
pub(crate) fn after_transaction_lock_acquired(bucket: &str, object: &str, hook: TransactionLockHook) {
AFTER_TRANSACTION_LOCK
.lock()
.expect("transaction lock hooks")
.push((bucket.to_owned(), object.to_owned(), hook));
}
pub(in crate::set_disk) fn run_after_transaction_lock_acquired(
bucket: &str,
object: &str,
guard: &rustfs_lock::NamespaceLockGuard,
) {
let hook = {
let mut hooks = AFTER_TRANSACTION_LOCK.lock().expect("transaction lock hooks");
hooks
.iter()
.position(|(hook_bucket, hook_object, _)| hook_bucket == bucket && hook_object == object)
.map(|index| hooks.remove(index).2)
};
if let Some(hook) = hook {
hook(guard);
}
}
pub(super) fn record_cleanup_attempt(bucket: &str, object: &str) {
CLEANUP_ATTEMPTS
@@ -112,6 +138,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
/// Only the locked heal path may repair a live marker-creation source.
pub repair_marker_source: bool,
pub namespace_owner: Option<Arc<dyn Send + Sync>>,
pub transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
pub can_commit: &'a (dyn Fn() -> bool + Sync),
@@ -125,6 +152,7 @@ impl<'a> DeleteRecoveryContext<'a> {
explicit_delete: false,
repair_marker_source: false,
namespace_owner: None,
transaction_guard: None,
recovery_ticket: None,
published_hint: None,
can_commit,
@@ -373,9 +401,22 @@ impl SetDisks {
write_quorum: usize,
mut context: DeleteRecoveryContext<'_>,
) -> Result<()> {
if context.namespace_owner.is_none() {
context.namespace_owner = Some(self.ctx.begin_namespace_commit());
}
let transaction_guard = match context.transaction_guard.take() {
Some(guard) => guard,
None => self
.acquire_delete_transaction_lock(bucket, object)
.await
.map_err(DiskError::other)?,
};
let outer_can_commit = context.can_commit;
let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
context.can_commit = &can_commit;
context.transaction_guard = Some(Arc::clone(&transaction_guard));
let owner = context
.namespace_owner
.take()
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
let can_commit = context.can_commit;
let states = inspect(disks, bucket, object, transaction).await;
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
@@ -547,8 +588,25 @@ impl SetDisks {
&self,
bucket: &str,
object: &str,
context: DeleteRecoveryContext<'_>,
mut context: DeleteRecoveryContext<'_>,
) -> Result<DeleteRecoverySummary> {
let transaction_guard = match context.transaction_guard.take() {
Some(guard) => guard,
None => self
.acquire_delete_transaction_lock(bucket, object)
.await
.map_err(DiskError::other)?,
};
let outer_can_commit = context.can_commit;
let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
context.can_commit = &can_commit;
if !context.dry_run {
let owner = context
.namespace_owner
.take()
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
}
let DeleteRecoveryContext {
incarnation,
dry_run,
@@ -558,6 +616,7 @@ impl SetDisks {
published_hint,
can_commit,
recovery_ticket: _,
transaction_guard: _,
} = context;
let disks = self.disk_inventory().await;
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
@@ -1257,6 +1316,135 @@ mod tests {
assert_eq!(body(&store, bucket, None).await, original);
}
#[tokio::test]
async fn delete_recovery_deferred_batch_discards_released_alias_fence_and_preserves_caller_fence() {
use crate::bucket::metadata_sys::ObjectLockConfigState;
use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence, ObjectLockConfigSnapshot};
use crate::storage_api_contracts::object::ObjectToDelete;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
for lose_caller in [false, true] {
let bucket = if lose_caller {
"deferred-batch-lost-caller"
} else {
"deferred-batch-released-alias"
};
let (_dirs, store, disks) = fixture(bucket).await;
put(&store, bucket, &[27; 4096], false).await;
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
let before = join_all(
disks
.iter()
.map(|disk| disk.inspect_delete_transaction(bucket, "object", transaction)),
)
.await
.into_iter()
.collect::<Result<Vec<_>>>()
.expect("capture pending recovery evidence");
let mut set = (*store.pools[0].disk_set[0]).clone();
set.lockers = vec![Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(
rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::FastObjectLockManager::new())),
)))];
set.ctx
.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
.await;
let (caller_fence, caller_lost) = NamespaceLockFence::loss_handle_for_test();
let released_alias_lost = Arc::new(AtomicBool::new(false));
let signal_fence = Arc::new(Mutex::new(None));
let captured_signal_fence = Arc::clone(&signal_fence);
let captured_alias_lost = Arc::clone(&released_alias_lost);
test_hooks::after_transaction_lock_acquired(
bucket,
"object",
Arc::new(move |guard| {
let signal = guard.lock_lost_signal().expect("distributed alias signal");
let fence = NamespaceLockSignalTestFence::install_with_loss_handle(&signal, Arc::clone(&captured_alias_lost));
// Retain the signal so a later allocation cannot reuse its
// registry identity after the original guard is dropped.
*captured_signal_fence.lock().expect("captured alias fence") = Some((fence, signal));
}),
);
let replay_started = Arc::new(AtomicBool::new(false));
let captured_replay_started = Arc::clone(&replay_started);
let expired_alias = Arc::clone(&released_alias_lost);
test_hooks::after_transaction_lock_acquired(
bucket,
"object",
Arc::new(move |_| {
// Acquiring the same transaction key again proves the first
// lease was released. Only its copied fence becomes stale.
expired_alias.store(true, Ordering::Release);
caller_lost.store(lose_caller, Ordering::Release);
captured_replay_started.store(true, Ordering::Release);
}),
);
let (deleted, errors) = set
.delete_objects(
bucket,
vec![ObjectToDelete {
object_name: "object".into(),
..Default::default()
}],
ObjectOptions {
namespace_lock_fence: Some(caller_fence),
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await;
assert!(
replay_started.load(Ordering::Acquire),
"missing metadata must take the deferred replay path"
);
assert!(
released_alias_lost.load(Ordering::Acquire),
"the completed batch alias must be marked lost"
);
if lose_caller {
assert!(errors[0].is_some(), "losing the original caller fence must reject deferred replay");
for (disk, original) in disks.iter().zip(before) {
let after = disk
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("retained proof");
assert_eq!(
after.fingerprint, original.fingerprint,
"lost caller authority must preserve every prepared journal"
);
}
} else {
assert!(
errors.iter().all(Option::is_none),
"released alias loss must not reject fresh recovery: {errors:?}"
);
assert!(
!deleted[0].found,
"fresh unversioned recovery commits absence before the idempotent batch retry"
);
for disk in &disks {
assert!(
discover(disk, bucket, Some("object"))
.await
.expect("final recovery evidence")
.is_empty()
);
assert_eq!(
disk.read_version("", bucket, "object", "", &ReadOptions::default())
.await
.expect_err("replayed delete must remove the source"),
DiskError::FileNotFound
);
}
}
drop(signal_fence);
}
}
#[tokio::test]
async fn delete_recovery_corrupt_bucket_does_not_prevent_other_bucket_rollback() {
let bad_bucket = "recover-corrupt-bucket";
+27 -8
View File
@@ -3652,6 +3652,18 @@ impl SetDisks {
return Ok((result, err.map(|e| e.into())));
}
// Keep the transaction key through both replay and the metadata heal
// based on that replay. The raw target key always precedes this key.
let transaction_guard =
if _write_lock_guard.is_some() && !opts.read_repair && !crate::bucket::utils::is_meta_bucketname(bucket) {
Some(self.acquire_delete_transaction_lock(bucket, object).await?)
} else {
None
};
let locks_lost = || {
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
};
let mut recovery_removed_version = false;
if !opts.read_repair
&& _write_lock_guard.is_some()
@@ -3673,12 +3685,17 @@ impl SetDisks {
dry_run: opts.dry_run,
explicit_delete,
repair_marker_source: true,
transaction_guard: transaction_guard.clone(),
namespace_owner: (!opts.dry_run).then(|| {
Arc::new((scope.clone(), self.ctx.begin_namespace_commit(), _write_lock_guard.clone()))
as Arc<dyn Send + Sync>
Arc::new((
scope.clone(),
self.ctx.begin_namespace_commit(),
_write_lock_guard.clone(),
transaction_guard.clone(),
)) as Arc<dyn Send + Sync>
}),
..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &|| {
scope.check().is_ok() && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
scope.check().is_ok() && !locks_lost()
})
},
)
@@ -3770,7 +3787,7 @@ impl SetDisks {
&& errs
.iter()
.all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)))
&& !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
&& !locks_lost()
{
*absence = Some(HealedObjectAbsence {
pool_index: self.pool_index,
@@ -3785,8 +3802,10 @@ impl SetDisks {
// Pass no_lock=true since we already obtained write lock (or are already called with no_lock=true)
let mut inner_opts = *opts;
inner_opts.no_lock = true;
let commit_fence = || !_write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
let namespace_owner = _write_lock_guard.as_ref().map(|guard| guard.clone() as Arc<dyn Send + Sync>);
let commit_fence = || !locks_lost();
let namespace_owner = _write_lock_guard
.as_ref()
.map(|guard| Arc::new((Arc::clone(guard), transaction_guard.clone())) as Arc<dyn Send + Sync>);
let (result, err) = pin_object_heal_future(|| {
self.heal_object_with_explicit_version_regen(
bucket,
@@ -3833,7 +3852,7 @@ impl SetDisks {
result,
err.map(Into::into),
absence,
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()),
locks_lost(),
bucket,
object,
));
@@ -3845,7 +3864,7 @@ impl SetDisks {
result,
err.map(Into::into),
absence,
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()),
locks_lost(),
bucket,
object,
))
+40 -12
View File
@@ -44,6 +44,21 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
#[tracing::instrument(level = "trace", skip(self))]
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
// Lock the file the disk resolves, not the spelling: on Windows `a\b`
// and `a/b` share one xl.meta, so they must share one lock. The key is
// folded by the requesting node only; lock servers compare it verbatim.
let resource = ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object_name_disk_identity(object).as_ref()),
version: None,
};
Ok(self.namespace_lock_for_key(resource).await)
}
}
impl SetDisks {
async fn namespace_lock_for_key(&self, resource: ObjectKey) -> NamespaceLockWrapper {
// Resolved from this set's own instance context (backlog#1052), not the
// ambient facade: the facade tracks whichever context is currently
// published process-wide, so a second instance (or, in tests, another
@@ -69,20 +84,33 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
};
// Lock the file the disk resolves, not the spelling: on Windows `a\b`
// and `a/b` share one xl.meta, so they must share one lock. The key is
// folded by the requesting node only; lock servers compare it verbatim.
let resource = ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object_name_disk_identity(object).as_ref()),
version: None,
};
Ok(NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone()))
NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone())
}
/// Raw object locks precede this set-local transaction lock. Its versioned
/// resource cannot collide with normal namespace keys, which use None.
/// Keeping the target set's clients avoids nesting overlapping raw quorums.
pub(in crate::set_disk) async fn acquire_delete_transaction_lock(
&self,
bucket: &str,
object: &str,
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
let resource = ObjectKey::with_version(
bucket,
object_name_disk_identity(object).as_ref(),
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
);
let guard = self
.namespace_lock_for_key(resource)
.await
.get_write_lock(super::super::get_lock_acquire_timeout())
.await
.map_err(|error| self.map_namespace_lock_error(bucket, object, "delete_transaction", error))?;
#[cfg(test)]
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
Ok(Arc::new(guard))
}
}
impl SetDisks {
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
match err {
+172 -24
View File
@@ -73,6 +73,7 @@ struct DeleteCommitContext<'a> {
incarnation: Option<Uuid>,
options: Option<&'a ObjectOptions>,
guard: Option<Arc<super::super::ObjectLockDiagGuard>>,
transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
}
impl<'a> DeleteCommitContext<'a> {
@@ -81,17 +82,37 @@ impl<'a> DeleteCommitContext<'a> {
incarnation: options.expected_bucket_incarnation_id,
options: Some(options),
guard: guard.cloned(),
transaction_guard: None,
}
}
fn with_transaction_guard(mut self, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) -> Self {
self.transaction_guard = guard.cloned();
self
}
fn ensure_transaction_lock_held(&self, bucket: &str, object: &str) -> Result<()> {
if self.transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "delete_transaction",
bucket: bucket.to_owned(),
object: object.to_owned(),
required: 1,
achieved: 0,
});
}
Ok(())
}
fn can_commit(&self, bucket: &str, object: &str) -> bool {
self.options.is_none_or(|options| {
ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok()
&& options
.scanner_publication_commit_scope
.as_ref()
.is_none_or(|scope| scope.can_commit())
})
!self.transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
&& self.options.is_none_or(|options| {
ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok()
&& options
.scanner_publication_commit_scope
.as_ref()
.is_none_or(|scope| scope.can_commit())
})
}
}
@@ -3852,6 +3873,7 @@ impl SetDisks {
object: &str,
opts: &ObjectOptions,
lock_guard: Option<&Arc<super::super::ObjectLockDiagGuard>>,
transaction_guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
) -> Result<bool> {
if is_meta_bucketname(bucket)
|| opts.version_id.is_some()
@@ -3888,6 +3910,7 @@ impl SetDisks {
object,
DeleteRecoveryContext {
explicit_delete: true,
transaction_guard: transaction_guard.cloned(),
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
lock_guard.cloned(),
@@ -8947,8 +8970,11 @@ impl SetDisks {
fi: &FileInfo,
force_del_marker: bool,
delete_marker_purge: Option<MrfDeleteMarkerPurge>,
commit_context: DeleteCommitContext<'_>,
mut commit_context: DeleteCommitContext<'_>,
) -> Result<()> {
if !is_meta_bucketname(bucket) && commit_context.transaction_guard.is_none() {
commit_context.transaction_guard = Some(self.acquire_delete_transaction_lock(bucket, object).await?);
}
let source_bucket_incarnation_id = commit_context.incarnation;
let transported = delete_file_info_with_replication_transport_metadata(fi);
let fi = &transported;
@@ -8957,6 +8983,7 @@ impl SetDisks {
Arc::new((
self.ctx.begin_namespace_commit(),
commit_context.guard.clone(),
commit_context.transaction_guard.clone(),
commit_context
.options
.and_then(|options| options.object_lock_config_snapshot.clone()),
@@ -8996,7 +9023,11 @@ impl SetDisks {
for disk in disks.iter() {
let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
let transaction_guard = commit_context.transaction_guard.as_ref();
futures.push(async move {
if transaction_guard.is_some_and(|guard| guard.is_lock_lost()) {
return Err(DiskError::DiskOngoingReq);
}
if let Some(disk) = disk {
let options = DeleteOptions {
old_data_dir: Some(rollback_dir),
@@ -9046,6 +9077,7 @@ impl SetDisks {
}
}
commit_context.ensure_transaction_lock_held(bucket, object)?;
let mut quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object);
if !is_meta_bucketname(bucket) && errs.iter().flatten().any(super::delete_recovery::unsupported_recovery) {
quorum_result = Err(DiskError::UnsupportedDisk.into());
@@ -9078,6 +9110,7 @@ impl SetDisks {
write_quorum,
DeleteRecoveryContext {
recovery_ticket,
transaction_guard: commit_context.transaction_guard.clone(),
namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
..DeleteRecoveryContext::new(source_bucket_incarnation_id, &|| commit_context.can_commit(bucket, object))
},
@@ -9087,6 +9120,7 @@ impl SetDisks {
}
#[cfg(test)]
pause_delete_before_cleanup(bucket).await;
commit_context.ensure_transaction_lock_held(bucket, object)?;
let mut rollback_futures = Vec::new();
for (index, err) in errs.iter().enumerate() {
if structured_cleanup {
@@ -9112,7 +9146,11 @@ impl SetDisks {
let object = object.to_string();
let fi = fi.clone();
let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
let transaction_guard = commit_context.transaction_guard.as_ref();
rollback_futures.push(async move {
if transaction_guard.is_some_and(|guard| guard.is_lock_lost()) {
return;
}
if should_rollback {
// The dedicated undo path never forwards the marker-only creation flag.
if let Err(err) = disk
@@ -9169,6 +9207,7 @@ impl SetDisks {
}
join_all(rollback_futures).await;
commit_context.ensure_transaction_lock_held(bucket, object)?;
drop(namespace_owner);
if quorum_result.is_ok()
&& errs.iter().any(Option::is_some)
@@ -9628,9 +9667,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
mut opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
let current_generations = opts.conditional_delete_generations.as_deref();
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut accounting = vec![None; objects.len()];
let delete_config_snapshot = opts
@@ -9725,6 +9763,30 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
let outer_namespace_lock_fence = opts.namespace_lock_fence.clone();
let mut transaction_guards = std::collections::BTreeMap::new();
if !is_meta_bucketname(bucket) {
// Every raw batch lock is acquired first. Canonical sorting also
// gives aliases of one disk key exactly one transaction owner.
let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
for key in keys {
match self.acquire_delete_transaction_lock(bucket, &key).await {
Ok(guard) => {
opts.add_namespace_lock_guard(&guard);
transaction_guards.insert(key, guard);
}
Err(error) => {
for (index, lock_name) in lock_names.iter().enumerate() {
if lock_name.as_ref() == key {
del_errs[index] = Some(error.clone());
}
}
}
}
}
}
let transaction_guards = Arc::new(transaction_guards);
let current_generations = opts.conditional_delete_generations.as_deref();
let object_lock_config = if is_meta_bucketname(bucket) {
Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))
} else {
@@ -9822,6 +9884,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&dobj.object_name,
DeleteRecoveryContext {
dry_run: !opts.no_lock,
transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
namespace_owner: opts.delete_mutation_owner.clone(),
..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| {
ensure_delete_commit_locks_held(None, bucket, &dobj.object_name, &opts).is_ok()
@@ -10103,6 +10166,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
Arc::new((
self.ctx.begin_namespace_commit(),
Arc::clone(&transaction_guards),
opts.delete_mutation_owner.clone(),
opts.object_lock_config_snapshot.clone(),
opts.scanner_publication_commit_scope.clone(),
@@ -10278,6 +10342,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
let results = join_all(futures).await;
if transaction_guards.values().any(|guard| guard.is_lock_lost()) {
for (index, object) in objects.iter().enumerate() {
if del_errs[index].is_none() {
accounting[index] = None;
del_errs[index] = Some(StorageError::NamespaceLockQuorumUnavailable {
mode: "delete_transaction",
bucket: bucket.to_owned(),
object: object.object_name.clone(),
required: 1,
achieved: 0,
});
}
}
if dist_erasure {
self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
}
return (del_objects, del_errs, accounting);
}
let mut del_obj_errs: Vec<Vec<Option<DiskError>>> = vec![vec![None; objects.len()]; disks.len()];
@@ -10373,6 +10455,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let group = &vers[index];
let recovery_ticket = recovery_tickets[index].clone();
let namespace_owner = namespace_owner.clone();
let transaction_guard = transaction_guards
.get(object_name_disk_identity(&group.name).as_ref())
.cloned();
let disks = &disks;
let opts = &opts;
async move {
@@ -10393,6 +10478,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
DeleteRecoveryContext {
recovery_ticket,
namespace_owner,
transaction_guard,
..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &can_commit)
},
)
@@ -10439,7 +10525,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let object = fi_vers.name.clone();
let versions = fi_vers.clone();
let disk_namespace_owner = namespace_owner.clone();
let transaction_guard = transaction_guards.get(object_name_disk_identity(&object).as_ref()).cloned();
rollback_futures.push(async move {
if transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return;
}
if should_rollback {
let errs = disk
.delete_versions_with_namespace_owner(
@@ -10515,7 +10605,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if del_errs[index].is_none() {
let object = &objects[index].object_name;
let result = match ensure_delete_commit_locks_held(None, bucket, object, &opts) {
Ok(()) => self.reclaim_missing_delete_residue(bucket, object, &opts, None).await,
Ok(()) => {
self.reclaim_missing_delete_residue(
bucket,
object,
&opts,
None,
transaction_guards.get(object_name_disk_identity(object).as_ref()),
)
.await
}
Err(err) => Err(err),
};
if let Err(err) = result {
@@ -10524,6 +10623,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
// Deferred recovery takes fresh raw and transaction locks. Discard the
// completed leases' deadlines while retaining every caller-owned fence.
opts.namespace_lock_fence = outer_namespace_lock_fence;
// Release inner owners before reacquiring any raw key lock.
drop(namespace_owner);
drop(transaction_guards);
// TODO(backlog): support partial object deletion for multi-part objects
if dist_erasure {
@@ -10646,6 +10752,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
} else {
None
};
// Lock order: the caller's raw object lock, then this target set's
// transaction key. The inner key remains distinct even when pool
// client quorums overlap, and no_lock never bypasses journal ownership.
let transaction_guard = if !is_meta_bucketname(bucket) && (!opts.delete_prefix || opts.delete_prefix_object) {
let guard = self.acquire_delete_transaction_lock(bucket, object).await?;
opts.add_namespace_lock_guard(&guard);
Some(guard)
} else {
None
};
if opts.delete_prefix {
let destructive_dispatch_phase = opts.lifecycle_delete_all.as_ref().is_none_or(|request| {
matches!(
@@ -10746,7 +10862,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&delete_request,
false,
None,
DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
DeleteCommitContext::request(&opts, _lock_guard.as_ref())
.with_transaction_guard(transaction_guard.as_ref()),
)
.await?;
if let Some((_, deleted_object)) = replication_delete {
@@ -10780,7 +10897,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&delete_request,
false,
None,
DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
DeleteCommitContext::request(&opts, _lock_guard.as_ref())
.with_transaction_guard(transaction_guard.as_ref()),
)
.await?;
}
@@ -10800,7 +10918,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&delete_request,
false,
None,
DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
DeleteCommitContext::request(&opts, _lock_guard.as_ref())
.with_transaction_guard(transaction_guard.as_ref()),
)
.await?;
}
@@ -10889,6 +11008,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
bucket,
object,
DeleteRecoveryContext {
transaction_guard: transaction_guard.clone(),
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
_lock_guard.clone(),
@@ -10929,7 +11049,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if is_err_object_not_found(err) {
ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?;
if self
.reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref())
.reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref(), transaction_guard.as_ref())
.await?
{
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
@@ -10979,7 +11099,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&dfi,
false,
None,
DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -11081,7 +11201,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&fi,
should_force_delete_marker_for_missing_version(&opts),
delete_marker_purge.clone(),
DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -11146,7 +11266,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
&dfi,
opts.delete_marker,
delete_marker_purge,
DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -25689,6 +25809,7 @@ mod delete_objects_lock_gating_tests {
}
#[tokio::test]
#[serial_test::serial]
async fn delete_object_aborts_when_outer_lock_is_lost_at_commit() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "single-lost-outer-lock";
@@ -25909,6 +26030,13 @@ mod delete_objects_lock_gating_tests {
.as_fast_lock_manager()
.expect("isolated fixture should use the local lock manager");
let lock_metrics = lock_manager.get_metrics().shard_metrics;
let raw_key = ObjectKey::new(bucket, "obj-a");
let raw_holder = lock_manager.get_lock_info(&raw_key).expect("outer holder must be visible");
let transaction_key = ObjectKey::with_version(
bucket,
"obj-a",
format!("rustfs:delete-transaction:{}:{}", set_disks.pool_index, set_disks.set_index),
);
assert_eq!(lock_manager.current_lock_counts(), (1, 0));
let mut delete = Box::pin(set_disks.delete_objects(
bucket,
@@ -25925,23 +26053,43 @@ mod delete_objects_lock_gating_tests {
// than a second on CI without waiting for the namespace lock at all.
let deletion = futures::future::poll_fn(|cx| {
let result = std::future::Future::poll(delete.as_mut(), cx);
let current_holder = lock_manager.get_lock_info(&raw_key).expect("outer holder must remain held");
assert_eq!(
(current_holder.mode, &current_holder.owner, current_holder.acquired_at),
(raw_holder.mode, &raw_holder.owner, raw_holder.acquired_at),
"no_lock must preserve the original raw object holder"
);
let transaction_holder = lock_manager.get_lock_info(&transaction_key);
if let Some(holder) = &transaction_holder {
assert_eq!(holder.mode, rustfs_lock::LockMode::Exclusive);
}
assert_eq!(
lock_manager.current_lock_counts(),
(1, 0),
"no_lock batch delete must preserve the outer holder without adding lock waiters"
(1 + u64::from(transaction_holder.is_some()), 0),
"only the distinct transaction owner may be added, without raw-key waiters"
);
let current = lock_manager.get_metrics().shard_metrics;
assert_eq!(
(current.total_acquisitions(), current.timeouts),
(lock_metrics.total_acquisitions(), lock_metrics.timeouts),
"no_lock batch delete must not acquire or time out on another namespace lock"
assert!(
current.total_acquisitions() <= lock_metrics.total_acquisitions() + 1,
"the transaction owner must be reused by inline recovery"
);
assert_eq!(current.timeouts, lock_metrics.timeouts, "the raw key must never be reacquired");
result
});
let (deleted, errs) = tokio::time::timeout(Duration::from_secs(30), deletion)
.await
.expect("no_lock batch delete disk work should complete");
assert_eq!(lock_manager.current_lock_counts(), (1, 0));
assert!(
lock_manager.get_lock_info(&transaction_key).is_none(),
"the transaction owner must be released"
);
assert_eq!(
lock_manager.get_metrics().shard_metrics.total_acquisitions(),
lock_metrics.total_acquisitions() + 1,
"no_lock still requires exactly one distinct transaction owner"
);
assert!(errs[0].is_none(), "no_lock batch delete should not fail with a lock error: {:?}", errs[0]);
assert!(deleted[0].found, "existing object must still be deleted");
for disk in &disk_stores {
+196
View File
@@ -13167,6 +13167,202 @@ mod tests {
})
.await;
}
async fn assert_delete_excludes_recovery_across_pool_lock_domains(topology: &str, batch: bool) {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
let mut store = new_prepared_reader_test_store_with_ctx(&[first_set, second_set], Arc::clone(&ctx)).await;
let clients: Vec<Arc<dyn rustfs_lock::LockClient>> = (0..4)
.map(|_| {
Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(
Arc::new(rustfs_lock::FastObjectLockManager::new()),
)))) as Arc<dyn rustfs_lock::LockClient>
})
.collect();
let domains = match topology {
"disjoint" => [vec![clients[0].clone()], vec![clients[1].clone()]],
"identical" => [clients[..3].to_vec(), clients[..3].to_vec()],
"overlapping" => [clients[..3].to_vec(), clients[1..].to_vec()],
_ => panic!("unknown lock topology"),
};
for (pool, lockers) in store.pools.iter_mut().zip(domains) {
let mut set = (*pool.disk_set[0]).clone();
set.lockers = lockers;
Arc::make_mut(pool).disk_set[0] = Arc::new(set);
}
let store = Arc::new(store);
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "delete-nonzero-pool-lock";
let object = "object.bin";
store
.handle_make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create the bucket in both pools");
if batch {
// Latest-marker routing sends this batch only to the nonzero pool,
// so the cleanup barrier observes its prepared journal directly.
store
.update_bucket_metadata_config(
bucket,
crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
)
.await
.expect("enable versioning for the routed batch");
}
let target = store.pools[1].get_disks_by_key(object);
let source = target
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(b"delete lock regression".to_vec()),
&ObjectOptions {
versioned: batch,
..Default::default()
},
)
.await
.expect("seed an object only in the nonzero pool");
ctx.update_erasure_type(SetupType::DistErasure).await;
let object_barrier = (!batch).then(|| crate::set_disk::DeleteObjectCommitBarrier::install(bucket, object));
let batch_barrier = batch.then(|| crate::set_disk::DeleteCleanupBarrier::install(bucket));
let delete_store = Arc::clone(&store);
let delete = tokio::spawn(async move {
if batch {
let (_, errors) = delete_store
.delete_objects(
bucket,
vec![ObjectToDelete {
object_name: object.to_owned(),
..Default::default()
}],
ObjectOptions::default(),
)
.await;
match errors.into_iter().flatten().next() {
Some(error) => Err(error),
None => Ok(()),
}
} else {
delete_store
.delete_object(bucket, object, ObjectOptions::default())
.await
.map(|_| ())
}
});
if let Some(barrier) = &object_barrier {
barrier.wait_until_paused().await;
}
if let Some(barrier) = &batch_barrier {
barrier.wait_until_paused().await;
}
let recovery_store = Arc::clone(&store);
let mut recovery = tokio::spawn(async move {
recovery_store
.recover_delete_transactions_at_incarnation(
bucket,
Some(object),
None,
&HealOpts {
pool: Some(1),
set: Some(0),
..Default::default()
},
)
.await
});
let early_recovery = tokio::time::timeout(Duration::from_millis(100), &mut recovery).await;
let recovery_was_blocked = early_recovery.is_err();
if let Some(barrier) = &object_barrier {
barrier.release();
}
if let Some(barrier) = &batch_barrier {
barrier.release();
}
tokio::time::timeout(Duration::from_secs(10), delete)
.await
.expect("the delete should finish after releasing its commit barrier")
.expect("the delete task should join")
.expect("the delete should succeed");
let recovered = match early_recovery {
Ok(result) => result,
Err(_) => tokio::time::timeout(Duration::from_secs(10), recovery)
.await
.expect("recovery should finish after the frontend delete"),
};
recovered
.expect("the recovery task should join")
.expect("recovery should succeed");
assert!(
recovery_was_blocked,
"recovery must not inspect an in-flight frontend delete with {topology} pool lock clients"
);
if batch {
let marker = target
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("the versioned batch must publish a current delete marker");
assert!(marker.delete_marker && marker.is_latest);
assert_eq!(marker.num_versions, 2, "the source version must remain beside the new marker");
assert_ne!(marker.version_id, source.version_id);
let mut reader = target
.get_object_reader(
bucket,
object,
None,
HeaderMap::new(),
&ObjectOptions {
version_id: Some(source.version_id.expect("versioned source identity").to_string()),
..Default::default()
},
)
.await
.expect("the original source version must remain readable");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("read the retained source version");
assert_eq!(body, b"delete lock regression");
} else {
let error = target
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect_err("the deleted object must be absent");
assert!(is_err_object_not_found(&error));
}
}
#[tokio::test]
#[serial_test::serial]
#[serial_test::serial(storage_class_env)]
async fn batch_delete_excludes_recovery_on_nonzero_pool() {
assert_delete_excludes_recovery_across_pool_lock_domains("disjoint", true).await;
}
#[tokio::test]
#[serial_test::serial]
#[serial_test::serial(storage_class_env)]
async fn delete_excludes_recovery_with_disjoint_pool_lock_clients() {
assert_delete_excludes_recovery_across_pool_lock_domains("disjoint", false).await;
}
#[tokio::test]
#[serial_test::serial]
#[serial_test::serial(storage_class_env)]
async fn delete_excludes_recovery_with_identical_pool_lock_clients() {
assert_delete_excludes_recovery_across_pool_lock_domains("identical", false).await;
}
#[tokio::test]
#[serial_test::serial]
#[serial_test::serial(storage_class_env)]
async fn delete_excludes_recovery_with_overlapping_pool_lock_clients() {
assert_delete_excludes_recovery_across_pool_lock_domains("overlapping", false).await;
}
}
#[cfg(test)]
+37 -1
View File
@@ -16,7 +16,7 @@ use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
use thiserror::Error;
use super::heal::{DiskError, EcstoreError};
use super::heal::{DiskError, EcstoreError, storage_api::owner::EcstorePoolMetadataFailure};
const HEAL_DANGLING_DELETE_GRACE_MESSAGE: &str = "dangling object deletion deferred by heal grace window";
pub(crate) const MRF_MISSING_PROOF_PREFIX: &str = "Missing exact storage proof for durable MRF repair ";
@@ -236,6 +236,14 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
/// with object absence, the conflation that once let an offline drive be
/// recorded as absent and skipped (backlog#856 / #799 B7).
pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
if let Some(failure) = error.pool_metadata_failure() {
return match failure.kind {
EcstorePoolMetadataFailure::ReadUnavailable | EcstorePoolMetadataFailure::TransactionUnknown => {
HealErrorClass::Transient
}
EcstorePoolMetadataFailure::RecoveryRequired | EcstorePoolMetadataFailure::FenceLost => HealErrorClass::Permanent,
};
}
if let EcstoreError::Lock(lock_error) = error {
return if lock_error.is_fatal() {
HealErrorClass::Permanent
@@ -435,6 +443,34 @@ mod tests {
use super::Error;
use crate::heal::{DiskError, EcstoreError};
#[test]
fn pool_metadata_failures_keep_their_typed_recovery_classification() {
use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
for (kind, retryable) in [
(EcstorePoolMetadataFailure::ReadUnavailable, true),
(EcstorePoolMetadataFailure::TransactionUnknown, true),
(EcstorePoolMetadataFailure::RecoveryRequired, false),
(EcstorePoolMetadataFailure::FenceLost, false),
] {
let error = EcstoreError::other(EcstorePoolMetadataError {
kind,
operation: "bucket heal".to_owned(),
phase: "native_heal",
since: std::time::UNIX_EPOCH.into(),
source: Some(std::sync::Arc::new(EcstoreError::SlowDown)),
});
let text = error.to_string();
for error in [error.clone(), error] {
assert_eq!(Error::Storage(error).is_recoverable_heal(), retryable, "{kind:?}");
}
assert!(
!Error::Storage(EcstoreError::other(text)).is_recoverable_heal(),
"display text must not grant recovery authority"
);
}
}
#[test]
fn namespace_lock_quorum_is_recoverable_across_io_error_conversions() {
for (required, achieved) in [(2, 1), (1, 0)] {
+8 -1
View File
@@ -26,7 +26,11 @@ pub(crate) use rustfs_ecstore::api::disk::{
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, ReplacementExecutionLease as EcstoreReplacementExecutionLease,
};
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::error::PoolMetadataError as EcstorePoolMetadataError;
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, PoolMetadataFailure as EcstorePoolMetadataFailure, StorageError as EcstoreStorageError,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
@@ -53,6 +57,9 @@ pub(crate) use rustfs_ecstore::api::storage::{
use rustfs_storage_api as storage_contracts;
pub(crate) mod owner {
#[cfg(test)]
pub(crate) use super::EcstorePoolMetadataError;
pub(crate) use super::EcstorePoolMetadataFailure;
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
pub(crate) use super::{
+2 -6
View File
@@ -520,9 +520,7 @@ impl HealTask {
error = %err,
"Heal cluster pool metadata failed"
);
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal cluster pool metadata: {err}"),
})
Err(err)
}
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
@@ -539,9 +537,7 @@ impl HealTask {
error = %err,
"Heal cluster pool metadata failed"
);
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal cluster pool metadata: {err}"),
})
Err(err)
}
}
}
+143 -8
View File
@@ -16,6 +16,7 @@ use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
use super::*;
use crate::heal::EcstoreError;
use crate::heal::storage::HealStorageObjectResult;
use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
mod absent_verdict;
mod concurrent_delete;
@@ -831,7 +832,10 @@ mod canonical_outcome {
async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
let storage = Arc::new(MockStorage {
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
bucket_heal_errors: Mutex::new(HashMap::from([(
"bucket-a".to_string(),
VecDeque::from([Error::other("metadata unavailable")]),
)])),
..Default::default()
});
let task = HealTask::from_request(
@@ -1692,7 +1696,7 @@ struct MockStorage {
include_object_dir_candidate: Mutex<bool>,
listed_buckets: Mutex<Option<Vec<String>>>,
bucket_discovery_incomplete: bool,
bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
bucket_heal_calls: Mutex<Vec<String>>,
delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
@@ -2238,6 +2242,7 @@ fn replacement_identity(
#[derive(Clone)]
enum MockHealObjectOutcome {
PoolMetadata { kind: EcstorePoolMetadataFailure, outer: bool },
LegacyAmbiguous { outer: bool },
MissingObject { outer: bool },
MissingVersion,
@@ -2257,6 +2262,16 @@ enum MockHealObjectOutcome {
PermanentOther(&'static str),
}
fn pool_metadata_error(kind: EcstorePoolMetadataFailure) -> Error {
Error::Storage(EcstoreError::other(EcstorePoolMetadataError {
kind,
operation: "bucket heal".to_owned(),
phase: "native_heal",
since: std::time::UNIX_EPOCH.into(),
source: None,
}))
}
fn unavailable_drive_heal_result(state: DriveState) -> (HealResultItem, Option<Error>) {
(
HealResultItem {
@@ -2428,6 +2443,14 @@ impl HealStorageAPI for MockStorage {
.and_then(VecDeque::pop_front)
{
return match outcome {
MockHealObjectOutcome::PoolMetadata { kind, outer } => {
let error = pool_metadata_error(kind);
if outer {
Err(error)
} else {
Ok((HealResultItem::default(), Some(error)))
}
}
MockHealObjectOutcome::LegacyAmbiguous { outer } => {
let error = Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous);
if outer {
@@ -2504,6 +2527,14 @@ impl HealStorageAPI for MockStorage {
}
if let Some(outcome) = self.heal_object_outcome.lock().unwrap().take() {
return match outcome {
MockHealObjectOutcome::PoolMetadata { kind, outer } => {
let error = pool_metadata_error(kind);
if outer {
Err(error)
} else {
Ok((HealResultItem::default(), Some(error)))
}
}
MockHealObjectOutcome::LegacyAmbiguous { outer } => {
let error = Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous);
if outer {
@@ -2644,14 +2675,14 @@ impl HealStorageAPI for MockStorage {
{
return Err(error);
}
if let Some(message) = self
if let Some(error) = self
.bucket_heal_errors
.lock()
.unwrap()
.get_mut(bucket)
.and_then(VecDeque::pop_front)
{
return Err(Error::other(message));
return Err(error);
}
Ok(HealResultItem::default())
}
@@ -3686,13 +3717,70 @@ async fn cluster_recreate_fails_when_pool_metadata_heal_fails() {
.await
.expect_err("cluster recreate heal must not hide pool metadata failure");
assert!(matches!(err, Error::TaskExecutionFailed { .. }));
assert!(matches!(err, Error::Other(ref message) if message == "pool metadata missing"));
assert_eq!(
storage.heal_object_calls.lock().expect("object calls").as_slice(),
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
);
}
#[tokio::test]
async fn cluster_pool_metadata_failures_preserve_recovery_classification_in_both_response_shapes() {
for outer in [false, true] {
for (kind, retryable) in [
(EcstorePoolMetadataFailure::ReadUnavailable, true),
(EcstorePoolMetadataFailure::TransactionUnknown, true),
(EcstorePoolMetadataFailure::RecoveryRequired, false),
(EcstorePoolMetadataFailure::FenceLost, false),
] {
let storage = Arc::new(MockStorage {
listed_buckets: Mutex::new(Some(Vec::new())),
..Default::default()
});
storage.heal_object_outcomes.lock().expect("object outcomes").insert(
POOL_META_NAME.to_string(),
VecDeque::from([MockHealObjectOutcome::PoolMetadata { kind, outer }]),
);
let mut request = HealRequest::new(
HealType::Cluster,
HealOptions {
recreate_missing: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
);
request.source = HealRequestSource::Admin;
let task = HealTask::from_request(request, storage.clone());
let error = task
.execute()
.await
.expect_err("pool metadata failure must reach the scheduler");
assert!(
matches!(&error, Error::Storage(error) if error.pool_metadata_failure().is_some_and(|failure| failure.kind == kind)),
"the original typed pool metadata state must survive either response shape: {error}"
);
assert_eq!(error.is_recoverable_heal(), retryable, "{kind:?}, outer={outer}");
assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), [POOL_META_NAME]);
if retryable {
let retry = task
.retry_request_with_remaining_timeout()
.await
.expect("existing retry request");
assert_eq!(retry.id, task.id, "recovery retains the accepted task token");
HealTask::from_request(retry, storage.clone())
.execute()
.await
.expect("the same task should complete after pool metadata recovery");
assert_eq!(
storage.heal_object_calls.lock().expect("object calls").as_slice(),
[POOL_META_NAME, POOL_META_NAME]
);
}
}
}
}
#[tokio::test]
async fn cluster_dry_run_does_not_heal_pool_metadata() {
let storage = Arc::new(MockStorage::default());
@@ -4056,7 +4144,10 @@ async fn test_recursive_bucket_heal_reports_typed_exhausted_and_permanent_failur
async fn test_cluster_heal_continues_after_bucket_failure() {
let storage = Arc::new(MockStorage {
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
bucket_heal_errors: Mutex::new(HashMap::from([(
"bucket-a".to_string(),
VecDeque::from([Error::other("metadata unavailable")]),
)])),
..Default::default()
});
let request = HealRequest::new(
@@ -4088,7 +4179,10 @@ async fn test_cluster_heal_continues_after_bucket_failure() {
async fn test_cluster_heal_retries_only_recoverable_bucket() {
let storage = Arc::new(MockStorage {
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["lock acquisition timeout"]))])),
bucket_heal_errors: Mutex::new(HashMap::from([(
"bucket-a".to_string(),
VecDeque::from([Error::other("lock acquisition timeout")]),
)])),
..Default::default()
});
let request = HealRequest::new(
@@ -4112,6 +4206,47 @@ async fn test_cluster_heal_retries_only_recoverable_bucket() {
);
}
#[tokio::test(start_paused = true)]
async fn cluster_bucket_heal_retries_only_recoverable_pool_metadata_failures() {
for (kind, retryable) in [
(EcstorePoolMetadataFailure::ReadUnavailable, true),
(EcstorePoolMetadataFailure::TransactionUnknown, true),
(EcstorePoolMetadataFailure::RecoveryRequired, false),
(EcstorePoolMetadataFailure::FenceLost, false),
] {
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_owned(), VecDeque::from([pool_metadata_error(kind)]))])),
..Default::default()
});
let mut request = HealRequest::new(
HealType::Cluster,
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
);
request.source = HealRequestSource::Admin;
let task = HealTask::from_request(request, storage.clone());
let result = task.execute().await;
if retryable {
result.expect("a recoverable pool metadata state must not terminate the root heal");
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
assert_eq!(
storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
["bucket-a", "bucket-a"]
);
} else {
result.expect_err("operator recovery and lost fences must remain terminal");
let failure = task.take_batch_failure().await.expect("permanent cluster failure");
assert_eq!((failure.retryable, failure.permanent), (0, 1));
assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
}
}
}
#[tokio::test]
async fn test_recursive_bucket_heal_does_not_remove_bucket_metadata() {
let storage = Arc::new(MockStorage::default());
@@ -4874,7 +5009,7 @@ async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
let storage = Arc::new(MockStorage {
bucket_heal_errors: Mutex::new(HashMap::from([(
"bucket-a".to_string(),
VecDeque::from(["injected bucket prepass failure"]),
VecDeque::from([Error::other("injected bucket prepass failure")]),
)])),
resume_disk: Mutex::new(Some(disk)),
..Default::default()
+313
View File
@@ -0,0 +1,313 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Durable restrictions owned by an authenticated user. A record is not an IAM
//! grant or credential; issuing credentials still requires current IAM checks.
use crate::{error::is_err_config_not_found, store::Store};
use serde::{Deserialize, Serialize};
const PREFIX: &str = "config/iam/plugin-delegations/v1";
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct Restrictions {
#[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
pub version: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
pub output_bucket: Option<String>,
pub deployment: String,
pub bucket: String,
pub input_prefix: String,
pub output_prefix: String,
pub expires_at_unix_seconds: u64,
}
// An absent v1 field is compatible; an explicitly null v2 field is invalid.
fn present_value<'de, T: Deserialize<'de>, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Option<T>, D::Error> {
T::deserialize(deserializer).map(Some)
}
impl Restrictions {
pub fn output_bucket(&self) -> Result<&str, DelegationError> {
match (self.version, self.output_bucket.as_deref()) {
(None, None) => Ok(&self.bucket),
(Some(2), Some(output)) => Ok(output),
_ => Err(DelegationError::Invalid),
}
}
pub fn validate(&self, now: u64) -> Result<(), DelegationError> {
let output_bucket = self.output_bucket()?;
let names: [&str; 5] = [
&self.deployment,
&self.bucket,
&self.input_prefix,
&self.output_prefix,
output_bucket,
];
if names
.iter()
.any(|s| s.is_empty() || s.len() > 1024 || s.contains("${") || s.chars().any(char::is_control))
|| self.bucket.contains('/')
|| self.bucket.contains('*')
|| self.bucket.contains('?')
|| output_bucket.contains('/')
|| output_bucket.contains('*')
|| output_bucket.contains('?')
|| self.input_prefix.contains('*')
|| self.input_prefix.contains('?')
|| self.output_prefix.contains('*')
|| self.output_prefix.contains('?')
|| self.expires_at_unix_seconds <= now
|| self.expires_at_unix_seconds.saturating_sub(now) > 86400
{
return Err(DelegationError::Invalid);
}
Ok(())
}
}
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct Delegation {
pub version: u32,
pub id: String,
pub revision: String,
pub owner_access_key: String,
pub created_at_unix_seconds: u64,
pub restrictions: Restrictions,
}
#[derive(Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
struct Tombstone {
version: u32,
id: String,
revision: String,
owner_access_key: String,
}
#[derive(Debug, thiserror::Error)]
pub enum DelegationError {
#[error("invalid delegation restriction")]
Invalid,
#[error("delegation not found")]
NotFound,
#[error("delegation owner rejected")]
OwnerRejected,
#[error("delegation unavailable")]
Unavailable,
#[error("delegation revoked")]
Revoked,
#[error("delegation expired")]
Expired,
}
fn record_path(id: &str) -> Result<String, DelegationError> {
if id.len() != 32 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
return Err(DelegationError::Invalid);
}
Ok(format!("{PREFIX}/{id}/record.json"))
}
async fn load<T: Store>(store: &T, id: &str) -> Result<Delegation, DelegationError> {
let record: Delegation = store.load_iam_config_strict(record_path(id)?).await.map_err(|error| {
if is_err_config_not_found(&error) {
DelegationError::NotFound
} else {
DelegationError::Unavailable
}
})?;
if !matches!(record.version, 1 | 2)
|| record.version != record.restrictions.version.unwrap_or(1)
|| record.id != id
|| record.revision != id
|| record.owner_access_key.is_empty()
{
return Err(DelegationError::Unavailable);
}
record
.restrictions
.validate(record.created_at_unix_seconds)
.map_err(|_| DelegationError::Unavailable)?;
Ok(record)
}
pub(crate) async fn create<T: Store>(
store: &T,
owner: &str,
restrictions: Restrictions,
now: u64,
) -> Result<Delegation, DelegationError> {
restrictions.validate(now)?;
if owner.is_empty() {
return Err(DelegationError::OwnerRejected);
}
// IDs are generated exclusively here; no update operation accepts an ID.
let id = format!("{:032x}", rand::random::<u128>());
let path = record_path(&id)?;
match store.load_iam_config_strict::<Delegation>(&path).await {
Err(error) if is_err_config_not_found(&error) => {}
_ => return Err(DelegationError::Unavailable),
}
let record = Delegation {
version: restrictions.version.unwrap_or(1),
revision: id.clone(),
id,
owner_access_key: owner.to_owned(),
created_at_unix_seconds: now,
restrictions,
};
store
.save_iam_config(&record, path)
.await
.map_err(|_| DelegationError::Unavailable)?;
if load(store, &record.id).await? != record {
return Err(DelegationError::Unavailable);
}
Ok(record)
}
pub(crate) async fn revoke<T: Store>(store: &T, owner: &str, id: &str) -> Result<(), DelegationError> {
let record = load(store, id).await?;
if record.owner_access_key != owner {
return Err(DelegationError::OwnerRejected);
}
let tombstone = Tombstone {
version: 1,
id: record.id,
revision: record.revision,
owner_access_key: record.owner_access_key,
};
let path = format!("{PREFIX}/{id}/revoked.json");
// Every retry writes exactly the same revocation; active records never change.
store
.save_iam_config(&tombstone, &path)
.await
.map_err(|_| DelegationError::Unavailable)?;
let saved: Tombstone = store
.load_iam_config_strict(path)
.await
.map_err(|_| DelegationError::Unavailable)?;
if saved != tombstone {
return Err(DelegationError::Unavailable);
}
Ok(())
}
pub(crate) async fn read_active<T: Store>(store: &T, owner: &str, id: &str, now: u64) -> Result<Delegation, DelegationError> {
let record = load(store, id).await?;
if record.owner_access_key != owner {
return Err(DelegationError::OwnerRejected);
}
check_active(store, record, now).await
}
pub(crate) async fn read_owned<T: Store>(store: &T, owner: &str, id: &str) -> Result<Delegation, DelegationError> {
let record = load(store, id).await?;
if record.owner_access_key != owner {
return Err(DelegationError::OwnerRejected);
}
Ok(record)
}
pub(crate) async fn read_active_for_broker<T: Store>(store: &T, id: &str, now: u64) -> Result<Delegation, DelegationError> {
let record = load(store, id).await?;
check_active(store, record, now).await
}
async fn check_active<T: Store>(store: &T, record: Delegation, now: u64) -> Result<Delegation, DelegationError> {
let id = &record.id;
match store
.load_iam_config_strict::<Tombstone>(format!("{PREFIX}/{id}/revoked.json"))
.await
{
Ok(_) => return Err(DelegationError::Revoked),
Err(error) if is_err_config_not_found(&error) => {}
Err(_) => return Err(DelegationError::Unavailable),
}
if now >= record.restrictions.expires_at_unix_seconds {
return Err(DelegationError::Expired);
}
Ok(record)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn delegation_v1_json_remains_compatible() {
let legacy = serde_json::json!({"deployment":"runner-a","bucket":"photos",
"input_prefix":"inputs/","output_prefix":"outputs/","expires_at_unix_seconds":200});
let restriction: Restrictions = serde_json::from_value(legacy.clone()).expect("legacy restriction");
restriction.validate(100).expect("legacy validation");
assert_eq!(restriction.output_bucket().expect("v1 output"), "photos");
assert_eq!(serde_json::to_value(restriction).expect("legacy serialization"), legacy);
let record = serde_json::json!({"version":1,"id":"a".repeat(32),"revision":"a".repeat(32),
"owner_access_key":"owner","created_at_unix_seconds":100,"restrictions":legacy});
let decoded: Delegation = serde_json::from_value(record.clone()).expect("legacy record");
assert_eq!(serde_json::to_value(decoded).expect("legacy record serialization"), record);
}
#[test]
fn delegation_v2_requires_explicit_output_bucket() {
let valid = serde_json::json!({"version":2,"deployment":"runner-a","bucket":"photos",
"input_prefix":"inputs/","output_bucket":"artifacts","output_prefix":"outputs/",
"expires_at_unix_seconds":200});
let restriction: Restrictions = serde_json::from_value(valid.clone()).expect("v2 restriction");
restriction.validate(100).expect("explicit output bucket");
assert_eq!(restriction.output_bucket().expect("v2 output"), "artifacts");
assert_eq!(serde_json::to_value(restriction).expect("v2 serialization"), valid);
for key in ["version", "output_bucket"] {
let mut bad = valid.clone();
bad.as_object_mut().expect("object").remove(key);
assert!(
serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
"v2 required {key}"
);
}
for (key, value) in [
("version", serde_json::json!(1)),
("version", serde_json::json!(3)),
("version", serde_json::Value::Null),
("output_bucket", serde_json::Value::Null),
("output_bucket", serde_json::json!("")),
("output_bucket", serde_json::json!("other/*")),
("output_bucket", serde_json::json!("${aws:username}")),
("output_bucket", serde_json::json!("other?")),
("output_bucket", serde_json::json!("other\n")),
("unknown", serde_json::json!(true)),
] {
let mut bad = valid.clone();
bad[key] = value;
assert!(
serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
"invalid v2 {key}"
);
}
}
#[test]
fn restrictions_are_literal_bounded_and_expiring() {
let mut restriction = Restrictions {
version: None,
output_bucket: None,
deployment: "runner-a".into(),
bucket: "photos".into(),
input_prefix: "inputs/".into(),
output_prefix: "outputs/".into(),
expires_at_unix_seconds: 200,
};
assert!(restriction.validate(100).is_ok());
restriction.output_prefix = "outputs/*".into();
assert!(matches!(restriction.validate(100), Err(DelegationError::Invalid)));
restriction.output_prefix = "outputs/".into();
assert!(matches!(restriction.validate(200), Err(DelegationError::Invalid)));
restriction.expires_at_unix_seconds = 200;
restriction.input_prefix = "inputs/${aws:username}/".into();
assert!(
matches!(restriction.validate(100), Err(DelegationError::Invalid)),
"C07_LITERAL_SCOPE_BOUND: policy variables are not literal restrictions"
);
assert!(record_path("../../users").is_err());
}
}
+1
View File
@@ -28,6 +28,7 @@ pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
const EVENT_OIDC_STATE: &str = "oidc_state";
pub mod cache;
pub mod delegation;
pub mod error;
pub mod federation;
pub mod keyring;
+1
View File
@@ -91,6 +91,7 @@ pub(crate) mod crate_boundary {
pub(crate) mod object_store {
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
}
pub(crate) mod runtime {
+5
View File
@@ -26,6 +26,11 @@ pub trait Store: Clone + Send + Sync + 'static {
fn has_watcher(&self) -> bool;
async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
/// Only confirmed storage absence may return ConfigNotFound. Corrupt or
/// undecryptable objects must remain errors, especially for revocations.
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, _path: impl AsRef<str> + Send) -> Result<Item> {
Err(crate::error::Error::IamSysNotInitialized)
}
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
+46
View File
@@ -42,6 +42,7 @@ use tracing::{debug, error, warn};
use crate::storage_api::object_store::{
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
read_config_preserve_empty_with_metadata_opts,
};
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
@@ -611,6 +612,11 @@ impl ObjectStore {
Ok(serde_json::from_slice(&outcome.plain)?)
}
fn decode_iam_config_strict<Item: DeserializeOwned>(data: &[u8]) -> Result<Item> {
let outcome = Self::decrypt_data_with_source(data)?;
Ok(serde_json::from_slice(&outcome.plain)?)
}
/// Parameterized core of [`Store::load_user_identity`].
async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
let mut u: UserIdentity = self
@@ -824,6 +830,12 @@ impl Store for ObjectStore {
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
self.load_iam_config_with(path, LoadMode::Locked).await
}
async fn load_iam_config_strict<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
let (data, _) =
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path.as_ref(), &LoadMode::Locked.read_opts())
.await?;
Self::decode_iam_config_strict(&data)
}
/// Saves IAM configuration with a retry mechanism on failure.
///
/// Attempts to save the IAM configuration up to 5 times if the storage layer is not ready,
@@ -1458,6 +1470,40 @@ mod tests {
assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
}
#[test]
fn strict_revocation_load_keeps_real_crypto_failure_distinct_from_absence() {
let cred = test_cred();
let key = format!("{}:{}", cred.access_key, cred.secret_key);
let plain = br#"{"version":1,"id":"revoked"}"#;
let mut encrypted = rustfs_crypto::encrypt_stream_io(key.as_bytes(), plain).expect("real stored ciphertext");
// Exercise the exact decoder used after the actual object read.
assert!(ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted).is_ok());
encrypted[50] ^= 0xFF;
let error = ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted)
.expect_err("corrupt tombstone must not become missing");
assert!(
matches!(error, Error::CryptoError(_)),
"C07_REVOCATION_FAIL_CLOSED: preserve actual crypto error"
);
assert!(
!crate::error::is_err_config_not_found(&error),
"C07_REVOCATION_FAIL_CLOSED: corrupted revocation is not absence"
);
// The preserve-empty reader keeps an existing zero-byte object as
// Ok(empty); ecstore's read_config_preserve_empty_distinguishes_empty_object
// covers that storage boundary. Its payload must fail here, not become
// ConfigNotFound and reactivate a revoked delegation.
let empty =
ObjectStore::decode_iam_config_strict::<serde_json::Value>(&[]).expect_err("an existing empty tombstone is corrupt");
assert!(
!crate::error::is_err_config_not_found(&empty),
"C07_REVOCATION_FAIL_CLOSED: empty existing revocation is not absence"
);
let invalid_json =
ObjectStore::decode_iam_config_strict::<serde_json::Value>(br#"{"version": }"#).expect_err("invalid stored JSON");
assert!(!crate::error::is_err_config_not_found(&invalid_json));
}
#[test]
fn test_decrypt_data_short_data_fails() {
let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON
+257 -7
View File
@@ -75,6 +75,7 @@ pub const STATUS_ENABLED: &str = "enabled";
pub const STATUS_DISABLED: &str = "disabled";
pub const POLICYNAME: &str = "policy";
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-policy";
@@ -168,6 +169,12 @@ enum PreparedServicePolicyMode {
#[derive(Clone)]
enum PreparedIamMode {
TerminalDeny,
PluginAttemptSts {
is_owner: bool,
combined_policy: Policy,
session_policy: PreparedSessionPolicy,
},
Opa,
Owner,
Deny,
@@ -202,7 +209,7 @@ impl PreparedIamAuth {
pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool {
match &self.mode {
PreparedIamMode::Opa => true,
PreparedIamMode::Owner | PreparedIamMode::Deny => false,
PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => false,
PreparedIamMode::Regular { combined_policy } => {
policy_needs_existing_object_tag_for_args(combined_policy, args).await
}
@@ -210,6 +217,11 @@ impl PreparedIamAuth {
combined_policy,
session_policy,
..
}
| PreparedIamMode::PluginAttemptSts {
combined_policy,
session_policy,
..
} => {
policy_needs_existing_object_tag_for_args(combined_policy, args).await
|| prepared_session_policy_needs_existing_object_tag_for_args(session_policy, args).await
@@ -239,14 +251,56 @@ impl PreparedIamAuth {
pub fn combined_policy_for_view(&self) -> Option<&Policy> {
match &self.mode {
PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
PreparedIamMode::Sts { combined_policy, .. } | PreparedIamMode::PluginAttemptSts { combined_policy, .. } => {
Some(combined_policy)
}
PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => None,
}
}
}
impl<T: Store> IamSys<T> {
/// For the authenticated, explicitly configured broker handler only.
pub async fn read_active_plugin_delegation_for_broker(
&self,
id: &str,
now: u64,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::read_active_for_broker(&self.store.api, id, now).await
}
pub async fn create_plugin_delegation(
&self,
owner: &str,
restrictions: crate::delegation::Restrictions,
now: u64,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::create(&self.store.api, owner, restrictions, now).await
}
pub async fn revoke_plugin_delegation(
&self,
owner: &str,
id: &str,
) -> std::result::Result<(), crate::delegation::DelegationError> {
crate::delegation::revoke(&self.store.api, owner, id).await
}
pub async fn read_active_plugin_delegation(
&self,
owner: &str,
id: &str,
now: u64,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::read_active(&self.store.api, owner, id, now).await
}
/// Reads an owned delegation record for historical lookups, including expired or revoked records.
pub async fn read_owned_plugin_delegation(
&self,
owner: &str,
id: &str,
) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
crate::delegation::read_owned(&self.store.api, owner, id).await
}
/// Create a new IamSys instance with the given IamCache store
///
/// # Arguments
@@ -281,6 +335,16 @@ impl<T: Store> IamSys<T> {
*guard = PolicyPluginState::Ready(client);
}
/// Restricted credentials require the built-in parent/session intersection.
/// External authorization bypasses that path, so only Disabled is supported.
pub async fn ensure_restricted_credential_vending() -> Result<()> {
if matches!(Self::policy_plugin_state().await, PolicyPluginState::Disabled) {
Ok(())
} else {
Err(Error::IAMActionNotAllowed)
}
}
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
let policy_plugin_state = get_policy_plugin_state();
let guard = policy_plugin_state.read().await;
@@ -1383,17 +1447,50 @@ impl<T: Store> IamSys<T> {
/// 1) know whether policy evaluation may need `s3:ExistingObjectTag`, and
/// 2) evaluate with final conditions without re-merging identity policies.
pub async fn prepare_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
if args.is_owner {
let marked = args.claims.contains_key(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
if args.is_owner && !marked {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Owner,
};
}
if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
// One observed state decides this request's path. Marked credentials
// never enter OPA, including after a restart or on another node.
let state = Self::policy_plugin_state().await;
if marked {
if args.is_owner
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|| !matches!(state, PolicyPluginState::Disabled)
{
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
};
}
let prepared = self.prepare_builtin_auth(args).await;
let mode = match prepared.mode {
PreparedIamMode::Sts {
is_owner,
combined_policy,
session_policy,
} => PreparedIamMode::PluginAttemptSts {
is_owner,
combined_policy,
session_policy,
},
_ => PreparedIamMode::TerminalDeny,
};
return PreparedIamAuth {
needs_existing_object_tag: prepared.needs_existing_object_tag,
mode,
};
}
if let Some(prepared) = state.prepared_iam_auth() {
return prepared;
}
self.prepare_builtin_auth(args).await
}
async fn prepare_builtin_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
return PreparedIamAuth {
needs_existing_object_tag: false,
@@ -1425,6 +1522,20 @@ impl<T: Store> IamSys<T> {
/// bucket policies on an IAM denial. An error must not become an implicit deny.
pub async fn try_eval_prepared(&self, prepared: &PreparedIamAuth, args: &Args<'_>) -> Result<bool> {
Ok(match &prepared.mode {
PreparedIamMode::TerminalDeny => return Err(Error::IAMActionNotAllowed),
PreparedIamMode::PluginAttemptSts {
is_owner,
combined_policy,
session_policy,
} => {
// A false IAM result permits bucket-policy fallback. Restricted
// plugin credentials must instead terminate authorization.
let session_allowed = evaluate_prepared_session_policy(session_policy, args).await == Some(true);
if !session_allowed || !(*is_owner || combined_policy.is_allowed(args).await) {
return Err(Error::IAMActionNotAllowed);
}
true
}
PreparedIamMode::Opa => {
let Some(opa_enable) = Self::get_policy_plugin_client().await else {
tracing::warn!("eval_prepared: OPA mode requested but plugin is unavailable");
@@ -2127,6 +2238,145 @@ mod tests {
assert!(prepared.combined_policy_for_view().is_none());
}
#[tokio::test]
#[serial]
async fn marked_signed_sts_denial_is_terminal_before_public_bucket_fallback() {
let iam = IamSys::new(IamCache::new(StsTestMockStore::new(false)).await.unwrap());
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
let signing_key = "c07-test-token-signing-key";
let mut metadata=HashMap::from([
("parent".into(),Value::String("sts-fallback-test-parent".into())),
("exp".into(),serde_json::json!((OffsetDateTime::now_utc()+time::Duration::hours(1)).unix_timestamp())),
(SESSION_POLICY_NAME.into(),Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/inside"]}]}"#))),
]);
metadata.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), Value::Bool(true));
let mut marked = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
marked.parent_user = "sts-fallback-test-parent".into();
let marked_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&marked.session_token, signing_key)
.expect("signed test credentials must decode")
.claims;
let marked_user = UserIdentity::from(marked.clone());
iam.store
.cache
.add_or_update_sts_account(&marked.access_key, &marked_user, OffsetDateTime::now_utc());
metadata.remove(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
let mut ordinary = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
ordinary.parent_user = marked.parent_user.clone();
let ordinary_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&ordinary.session_token, signing_key)
.expect("signed test credentials must decode")
.claims;
iam.store.cache.add_or_update_sts_account(
&ordinary.access_key,
&UserIdentity::from(ordinary.clone()),
OffsetDateTime::now_utc(),
);
let groups = None;
let conditions = HashMap::new();
let args = Args {
account: &marked.access_key,
groups: &groups,
action: Action::S3Action(S3Action::GetObjectAction),
bucket: "mybucket",
object: "outside",
conditions: &conditions,
is_owner: false,
claims: &marked_claims,
deny_only: false,
};
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
let denied = iam.prepare_auth(&args).await;
let disabled_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
let inside = Args {
object: "inside",
..args.clone()
};
let allowed = iam.prepare_auth(&inside).await;
let inside_allowed = matches!(iam.try_eval_prepared(&allowed, &inside).await, Ok(true));
let unmarked_args = Args {
account: &ordinary.access_key,
claims: &ordinary_claims,
..args.clone()
};
let unmarked = iam.prepare_auth(&unmarked_args).await;
let ordinary_false = matches!(iam.try_eval_prepared(&unmarked, &unmarked_args).await, Ok(false));
let public:rustfs_policy::policy::BucketPolicy=serde_json::from_value(serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]})).unwrap();
let public_allows = public
.is_allowed(&rustfs_policy::policy::BucketPolicyArgs {
account: &marked.access_key,
groups: &groups,
action: args.action,
bucket: args.bucket,
object: args.object,
conditions: &conditions,
is_owner: false,
})
.await;
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
url: "http://127.0.0.1:8181/allow".into(),
auth_token: String::new(),
}))
.await;
let opa_marked = iam.prepare_auth(&args).await;
let ready_terminal = matches!(iam.try_eval_prepared(&opa_marked, &args).await, Err(Error::IAMActionNotAllowed));
let opa_ordinary = iam.prepare_auth(&unmarked_args).await;
let ordinary_keeps_opa = matches!(opa_ordinary.mode, PreparedIamMode::Opa);
// The captured Disabled preparation stays on built-in intersection even
// if an external policy client becomes ready before evaluation.
let captured_still_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
let pending = iam.prepare_auth(&args).await;
let initializing_terminal = matches!(iam.try_eval_prepared(&pending, &args).await, Err(Error::IAMActionNotAllowed));
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
let failed = iam.prepare_auth(&args).await;
let failed_terminal = matches!(iam.try_eval_prepared(&failed, &args).await, Err(Error::IAMActionNotAllowed));
*get_policy_plugin_state().write().await = previous;
assert!(public_allows, "C07_PUBLIC_BUCKET_CONTROL: real public policy permits outside object");
assert!(
disabled_terminal && captured_still_terminal,
"C07_SCOPE_TERMINAL_BOUND: bucket policy must not rescue restricted scope denial"
);
assert!(
ready_terminal && initializing_terminal && failed_terminal,
"C07_EXTERNAL_POLICY_BOUND: marked persisted STS cannot enter OPA"
);
assert!(
inside_allowed && ordinary_false && ordinary_keeps_opa,
"C07_UNMARKED_COMPAT: preserve ordinary STS and valid marked scope"
);
}
#[tokio::test]
#[serial]
async fn restricted_vending_rejects_actual_external_policy_plugin_state() {
let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".into(),
auth_token: String::new(),
}))
.await;
let active_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_err();
*get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
let initializing_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_err();
*get_policy_plugin_state().write().await = PolicyPluginState::Failed;
let failed_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_err();
*get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
let disabled_allowed = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
.await
.is_ok();
*get_policy_plugin_state().write().await = previous;
assert!(
active_rejected && initializing_rejected && failed_rejected,
"C07_EXTERNAL_POLICY_BOUND: external authorization cannot bypass minted session scope"
);
assert!(disabled_allowed);
}
#[tokio::test]
#[serial]
async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() {
+56
View File
@@ -568,6 +568,24 @@ pub enum AdminAction {
GetTableLifecycleAction,
#[strum(serialize = "admin:SetTableLifecycle")]
SetTableLifecycleAction,
#[strum(serialize = "admin:CreatePluginDelegation")]
CreatePluginDelegationAction,
#[strum(serialize = "admin:IssuePluginAttempt")]
IssuePluginAttemptAction,
#[strum(serialize = "admin:SubmitPluginJob")]
SubmitPluginJobAction,
#[strum(serialize = "admin:GetPluginRun")]
GetPluginRunAction,
#[strum(serialize = "admin:CancelPluginRun")]
CancelPluginRunAction,
#[strum(serialize = "admin:ListPluginRuns")]
ListPluginRunsAction,
#[strum(serialize = "admin:ReplayPluginRun")]
ReplayPluginRunAction,
#[strum(serialize = "admin:GetPluginDelegation")]
GetPluginDelegationAction,
#[strum(serialize = "admin:RevokePluginDelegation")]
RevokePluginDelegationAction,
#[strum(serialize = "admin:GetTableCredentials")]
GetTableCredentialsAction,
#[strum(serialize = "admin:RunTableMaintenance")]
@@ -713,6 +731,15 @@ impl AdminAction {
| AdminAction::GetTableLifecycleAction
| AdminAction::SetTableLifecycleAction
| AdminAction::GetTableCredentialsAction
| AdminAction::IssuePluginAttemptAction
| AdminAction::SubmitPluginJobAction
| AdminAction::GetPluginRunAction
| AdminAction::CancelPluginRunAction
| AdminAction::ListPluginRunsAction
| AdminAction::ReplayPluginRunAction
| AdminAction::GetPluginDelegationAction
| AdminAction::CreatePluginDelegationAction
| AdminAction::RevokePluginDelegationAction
| AdminAction::RunTableMaintenanceAction
| AdminAction::GetTableMetadataLocationAction
| AdminAction::SetTableMetadataLocationAction
@@ -913,6 +940,35 @@ mod tests {
assert_eq!(arr[0].as_str().unwrap(), "s3:*");
}
#[test]
fn plugin_delegation_actions_are_explicit_and_unscoped() {
for (name, expected) in [
("admin:GetPluginDelegation", AdminAction::GetPluginDelegationAction),
("admin:CreatePluginDelegation", AdminAction::CreatePluginDelegationAction),
("admin:RevokePluginDelegation", AdminAction::RevokePluginDelegationAction),
("admin:SubmitPluginJob", AdminAction::SubmitPluginJobAction),
("admin:GetPluginRun", AdminAction::GetPluginRunAction),
("admin:CancelPluginRun", AdminAction::CancelPluginRunAction),
("admin:ListPluginRuns", AdminAction::ListPluginRunsAction),
("admin:ReplayPluginRun", AdminAction::ReplayPluginRunAction),
] {
assert_eq!(AdminAction::try_from(name).unwrap(), expected);
assert!(expected.is_valid());
assert!(!expected.is_table_resource_scoped());
}
assert_eq!(
AdminAction::try_from("admin:IssuePluginAttempt").unwrap(),
AdminAction::IssuePluginAttemptAction
);
assert!(AdminAction::IssuePluginAttemptAction.is_valid());
assert!(AdminAction::SubmitPluginJobAction.is_valid());
assert!(AdminAction::GetPluginRunAction.is_valid());
assert!(AdminAction::CancelPluginRunAction.is_valid());
assert!(AdminAction::ListPluginRunsAction.is_valid());
assert!(AdminAction::ReplayPluginRunAction.is_valid());
assert!(AdminAction::try_from("admin:IssuePluginCredentials").is_err());
}
#[test]
fn test_get_replication_metrics_admin_action_is_valid() {
assert!(AdminAction::GetReplicationMetricsAction.is_valid());
+184 -2
View File
@@ -238,6 +238,71 @@ pub enum ScannerDurableDirtyUsageReplayScope {
TopLevelEntries { entries: BTreeSet<String> },
}
impl ScannerDurableDirtyUsageReplayRecord {
/// Replays decoded producer hints with the canonical byte limit and semantic checks.
///
/// Callers must retain their framing and owner checks. This performs the same
/// state update as [`replay_durable_dirty_usage_producer_record`]; it does not
/// establish storage-verified source authority.
pub fn replay(self) -> std::result::Result<ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayError> {
check_durable_dirty_usage_replay_bytes(&self)?;
replay_decoded_dirty_usage_record(&self)
}
}
struct ReplayByteLimit {
remaining: usize,
}
impl std::io::Write for ReplayByteLimit {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.remaining = self
.remaining
.checked_sub(bytes.len())
.ok_or_else(|| std::io::Error::other("dirty replay byte limit"))?;
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
fn check_durable_dirty_usage_replay_bytes(
record: &ScannerDurableDirtyUsageReplayRecord,
) -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> {
// Bound structure traversal and string scanning before counting escaped JSON.
// Each entry or prefix contributes at least one additional serialized byte.
let mut remaining = SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES;
let mut charge = |size: usize| -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> {
remaining = remaining
.checked_sub(size)
.ok_or(ScannerDurableDirtyUsageReplayError::ByteLimit)?;
Ok(())
};
charge(record.writer_epoch.len())?;
charge(record.entries.len())?;
for entry in &record.entries {
charge(entry.bucket.len())?;
if let ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } = &entry.scope {
charge(entries.len())?;
for entry in entries {
charge(entry.len())?;
}
}
}
let mut limit = ReplayByteLimit {
remaining: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES,
};
serde_json::to_writer(&mut limit, record).map_err(|error| {
if error.is_io() {
ScannerDurableDirtyUsageReplayError::ByteLimit
} else {
ScannerDurableDirtyUsageReplayError::InvalidJson
}
})
}
pub fn encode_durable_dirty_usage_producer_replay_record(
entries: Vec<ScannerDurableDirtyUsageReplayEntry>,
) -> std::result::Result<Vec<u8>, ScannerDurableDirtyUsageReplayError> {
@@ -263,7 +328,13 @@ pub fn replay_durable_dirty_usage_producer_record(
}
let record: ScannerDurableDirtyUsageReplayRecord =
serde_json::from_slice(bytes).map_err(|_| ScannerDurableDirtyUsageReplayError::InvalidJson)?;
let replay = validate_durable_dirty_usage_replay_record(&record)?;
replay_decoded_dirty_usage_record(&record)
}
fn replay_decoded_dirty_usage_record(
record: &ScannerDurableDirtyUsageReplayRecord,
) -> std::result::Result<ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayError> {
let replay = validate_durable_dirty_usage_replay_record(record)?;
let max_generation = replay.iter().map(|entry| entry.generation).max().unwrap_or(0);
set_dirty_usage_generation_floor(max_generation)?;
let pending = {
@@ -772,6 +843,109 @@ mod scoped_dirty_usage_tests {
);
}
fn decoded_replay_fixture() -> ScannerDurableDirtyUsageReplayRecord {
ScannerDurableDirtyUsageReplayRecord {
schema: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA,
cache_key_format: crate::DATA_USAGE_CACHE_KEY_FORMAT,
writer_epoch: "writer".to_string(),
entries: vec![ScannerDurableDirtyUsageReplayEntry {
bucket: "photos".to_string(),
generation: 7,
scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
entries: BTreeSet::from(["2026".to_string(), "archive".to_string()]),
},
producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(),
}],
}
}
#[test]
#[serial]
fn durable_dirty_usage_decoded_replay_matches_byte_replay_at_size_boundary() {
let record = decoded_replay_fixture();
let fixed_bytes = serde_json::to_vec(&record).expect("fixture encoding").len() - record.writer_epoch.len();
for size in [
SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES - 1,
SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES,
SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1,
] {
clear_dirty_usage_buckets_for_tests();
let mut record = record.clone();
record.writer_epoch = "x".repeat(size - fixed_bytes);
let bytes = serde_json::to_vec(&record).expect("bounded fixture encoding");
assert_eq!(bytes.len(), size);
let expected = replay_durable_dirty_usage_producer_record(&bytes);
let buckets = dirty_usage_buckets_for_tests();
let scopes = dirty_usage_bucket_scopes_for_tests();
let identities = dirty_usage_producer_identities().clone();
let coverage = DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire);
clear_dirty_usage_buckets_for_tests();
assert_eq!(record.replay(), expected, "canonical size {size}");
assert_eq!(dirty_usage_buckets_for_tests(), buckets);
assert_eq!(dirty_usage_bucket_scopes_for_tests(), scopes);
assert_eq!(*dirty_usage_producer_identities(), identities);
assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), coverage);
}
clear_dirty_usage_buckets_for_tests();
}
#[test]
fn durable_dirty_usage_decoded_size_counts_escaped_utf8() {
for repeats in [6000, 8000] {
let mut record = decoded_replay_fixture();
record.writer_epoch = "\"\n雪\\".repeat(repeats);
let bytes = serde_json::to_vec(&record).expect("escaped fixture encoding");
let expected = if bytes.len() > SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
Err(ScannerDurableDirtyUsageReplayError::ByteLimit)
} else {
Ok(())
};
assert_eq!(check_durable_dirty_usage_replay_bytes(&record), expected);
}
}
#[test]
#[serial]
fn durable_dirty_usage_decoded_replay_rejects_large_values_before_state_changes() {
for field in ["epoch", "prefix", "entries"] {
clear_dirty_usage_buckets_for_tests();
let before = scanner_dirty_usage_state();
let mut record = decoded_replay_fixture();
match field {
"epoch" => record.writer_epoch = "x".repeat(1024 * 1024),
"prefix" => {
record.entries[0].scope = ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
entries: BTreeSet::from(["x".repeat(1024 * 1024)]),
};
}
_ => record.entries = vec![record.entries[0].clone(); SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1],
}
assert_eq!(record.replay(), Err(ScannerDurableDirtyUsageReplayError::ByteLimit), "{field}");
assert_eq!(scanner_dirty_usage_state(), before);
assert!(dirty_usage_buckets_for_tests().is_empty());
assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
assert!(dirty_usage_producer_identities().is_empty());
assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
}
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn durable_dirty_usage_decoded_replay_preserves_generation_exhaustion() {
clear_dirty_usage_buckets_for_tests();
let generation = DIRTY_USAGE_BUCKET_GENERATION.swap(u64::MAX, Ordering::AcqRel);
let result = decoded_replay_fixture().replay();
DIRTY_USAGE_BUCKET_GENERATION.store(generation, Ordering::Release);
assert_eq!(result, Err(ScannerDurableDirtyUsageReplayError::InvalidGeneration));
assert!(dirty_usage_buckets_for_tests().is_empty());
assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
assert!(dirty_usage_producer_identities().is_empty());
assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn durable_dirty_usage_replay_rejects_invalid_records_without_partial_state() {
@@ -843,9 +1017,17 @@ mod scoped_dirty_usage_tests {
for (bytes, expected_error) in invalid_cases {
clear_dirty_usage_buckets_for_tests();
assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error));
let before = scanner_dirty_usage_state();
assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error.clone()));
assert!(dirty_usage_buckets_for_tests().is_empty());
assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
let record: ScannerDurableDirtyUsageReplayRecord = serde_json::from_slice(&bytes).expect("typed invalid record");
assert_eq!(record.replay(), Err(expected_error));
assert_eq!(scanner_dirty_usage_state(), before);
assert!(dirty_usage_buckets_for_tests().is_empty());
assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
assert!(dirty_usage_producer_identities().is_empty());
assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
}
clear_dirty_usage_buckets_for_tests();
+20 -8
View File
@@ -1471,6 +1471,7 @@ mod tests {
inner: QueueStore<QueuedPayload>,
fail_put_failed_raw: Arc<std::sync::atomic::AtomicBool>,
fail_del: Arc<std::sync::atomic::AtomicBool>,
delete_attempts: Arc<Semaphore>,
}
impl Store<QueuedPayload> for FailingFailedStore {
@@ -1499,10 +1500,13 @@ mod tests {
self.inner.get_raw(key)
}
fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
if self.fail_del.load(Ordering::SeqCst) {
return Err(StoreError::Internal("injected del failure".to_string()));
}
self.inner.del(key)
let result = if self.fail_del.load(Ordering::SeqCst) {
Err(StoreError::Internal("injected del failure".to_string()))
} else {
self.inner.del(key)
};
self.delete_attempts.add_permits(1);
result
}
fn delete(&self) -> Result<(), Self::Error> {
self.inner.delete()
@@ -1986,6 +1990,7 @@ mod tests {
inner: inner.clone(),
fail_put_failed_raw: fail_flag.clone(),
fail_del: Arc::new(std::sync::atomic::AtomicBool::new(false)),
delete_attempts: Arc::new(Semaphore::new(0)),
};
// The target's terminal move writes through the rejecting failed handle, so the injected
// failure is what the move hits.
@@ -2042,6 +2047,7 @@ mod tests {
inner: inner.clone(),
fail_put_failed_raw: Arc::new(std::sync::atomic::AtomicBool::new(false)),
fail_del: Arc::new(std::sync::atomic::AtomicBool::new(true)),
delete_attempts: Arc::new(Semaphore::new(0)),
};
let target = Arc::new(ProgrammedTarget::new(
Some(TargetError::JetStreamPublish {
@@ -2053,8 +2059,15 @@ mod tests {
let tally = Arc::new(ReplayTally::default());
let (cancel_tx, worker) = spawn_failing_store_worker(&store, target, tally.clone());
// Many scans run while the live delete keeps failing.
tokio::time::sleep(Duration::from_secs(300)).await;
// Observe repeated durable overwrites without auto-advancing through thousands of
// real filesystem writes. Keep the original virtual observation bound on progress.
tokio::time::timeout(Duration::from_secs(300), store.delete_attempts.acquire_many(8))
.await
.expect("replay should retry the failing delete eight times")
.expect("delete progress remains open")
.forget();
cancel_tx.send(()).await.expect("replay worker should remain cancellable");
worker.await.expect("replay worker should drain after cancellation");
assert_eq!(
tally.permanent_failure.load(Ordering::SeqCst),
0,
@@ -2063,8 +2076,6 @@ mod tests {
assert_eq!(inner.len(), 1, "the live entry stays for the next scan");
assert_eq!(inner.failed_len(), 1, "the durable failed copy landed exactly once");
let _ = cancel_tx.send(()).await;
let _ = worker.await;
let _ = inner.delete();
}
@@ -2084,6 +2095,7 @@ mod tests {
inner: store.clone(),
fail_put_failed_raw: Arc::new(std::sync::atomic::AtomicBool::new(false)),
fail_del: Arc::new(std::sync::atomic::AtomicBool::new(false)),
delete_attempts: Arc::new(Semaphore::new(0)),
};
let target = Arc::new(ProgrammedTarget::new(
Some(TargetError::JetStreamPublish {
+1
View File
@@ -50,6 +50,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
| [scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md) | changing what makes a scanner cycle `Complete`, `Superseded`, or converged, adding or re-classifying a dirty-usage producer, deciding whether a background writeback may invalidate a cycle, changing scanner worker concurrency or throttling, or activating scoped dirty acknowledgements |
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
| [plugin-delegation.md](plugin-delegation.md) | changing original-user plugin delegation, attempt credentials, or Runner task APIs |
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |
+142
View File
@@ -0,0 +1,142 @@
# Plugin delegation
**Use this when:** changing plugin delegation registration, revocation, broker identity, or the scope of attempt credentials.
## Integration boundary
Runner owns scheduling, leases, container execution and result publication. RustFS provides original-user delegation, scoped temporary credentials and authenticated task API forwarding. Inputs and outputs use ordinary versioned S3 buckets; event-driven jobs use the existing bucket notification/Webhook path. Managed artifact sealing, inventory, garbage collection and backup retention require separate storage contracts and are outside this integration.
Set `RUSTFS_PLUGIN_RUNNER_URL` to the Controller HTTPS URL ending in `/v1/ingress/manual/jobs`, and `RUSTFS_PLUGIN_RUNNER_DEPLOYMENT` to the delegation deployment. Forwarding requires the published outbound mTLS client identity and uses the published trust roots when configured. Connections are reused within a TLS generation; publishing new TLS material replaces the pool, and removing the client identity rejects new forwarding requests.
## Authority
A delegation is an immutable restriction record owned by an ordinary IAM user. It is neither a permission grant nor a credential. Creating it requires that user's real signed request and `admin:CreatePluginDelegation`; reading and revoking require the corresponding `admin:GetPluginDelegation` and `admin:RevokePluginDelegation` actions. Root, temporary credentials and service accounts cannot own records.
The record binds a deployment, an input bucket, literal input and output prefixes, and an expiry of at most 24 hours. The v1 profile writes to that same bucket; v2 explicitly names the output bucket. Wildcards, control characters and IAM variable expressions are rejected. Its random ID is also its immutable revision. There is no update operation.
The Controller is responsible for validating accepted task identity, current lease, epoch, node session and control generation before requesting credentials. RustFS authenticates the configured broker and enforces the delegation and current parent IAM. RustFS does not independently verify a remote Controller lease.
```mermaid
sequenceDiagram
participant U as Original IAM user
participant C as Controller broker
participant R as RustFS
participant P as Plugin
U->>R: Signed delegation registration
R-->>U: Immutable ID and revision
C->>C: Validate task and current lease
C->>R: Signed issue-attempt request
R->>R: Broker mapping + active record + current IAM
R->>R: Persist scoped STS and recheck authority
R-->>C: Short-lived credentials
C-->>P: Attempt credentials
P->>R: Exact version GET / attempt-prefix PUT
R->>R: Current parent IAM intersected with session policy
```
## Persistence and revocation
`crates/iam/src/delegation.rs` owns record validation and immutable revocation tombstones. Records use the existing IAM store under `config/iam/plugin-delegations/v1`. Writes require a successful readback. Revocation is idempotent and durable; it does not delete or overwrite the restriction record.
`ObjectStore::load_iam_config_strict` distinguishes a missing object from an empty object, invalid JSON or failed decryption. Only a genuinely absent tombstone permits an active record. Corrupt or unavailable storage rejects access rather than treating revocation as absent. Existing non-delegation readers keep their compatibility behavior.
Revocation prevents future issuance. It does not promise immediate invalidation of credentials already delivered; those remain subject to their expiration and current parent IAM. Registration, IAM changes and issuance are not one atomic transaction. Issuance rechecks after persistence and deletes the generated STS account if authority changed. Failed persistence or invalidation acknowledgment is reported as unknown issuance, never as safe retry permission.
Manual job submission reads the active delegation before calling the Runner. A submission that passed this check is in flight and may be admitted if revocation races with the outbound request; revocation prevents submissions whose authority check occurs afterward, but does not cancel an admitted job. Callers must treat a successful submit receipt as accepted work even if the delegation is revoked later. After an ambiguous timeout or service error, retry the same idempotency key while the delegation remains active; after revocation or expiry, query status instead of retrying submission.
`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/query` accepts `{version:1,run_id}` and requires an ordinary IAM user with `admin:GetPluginRun` who owns the delegation in `{id}`. It reads the immutable owner record without checking expiry or revocation, then queries Runner using the record's delegation ID and revision and the authenticated access key. The result can contain only that caller's run status and committed manifest; it does not perform object reads or reauthorize object access. The Runner response must match the requested run ID and the strict status schema before RustFS returns it.
`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/cancel` accepts the same request and requires `admin:CancelPluginRun` plus ownership of `{id}`. Expiry and revocation do not block cancelling an existing run. Runner scopes the run to that owner; missing and other-owner runs share the same not-found response, while successful and failed runs return conflict. Success returns `{version:1,run_id,state:"CANCELLED",cleanup_pending}` and does not imply that asynchronous cleanup has finished.
`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/page` accepts `{version:1,limit,cursor?}` with a limit from 1 to 100 and requires `admin:ListPluginRuns` plus ownership of `{id}`. Expiry and revocation do not hide historical runs. RustFS supplies signer and delegation identity from authenticated IAM state; the Runner page is scoped to the exact caller and deployment. The strict response contains run summaries and an opaque-to-clients stable cursor; callers pass the returned cursor unchanged until it is null. RustFS bounds request/response sizes and rejects malformed, unordered, over-limit, or identity-invalid summaries.
`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/history` accepts `{version:1,run_id}` and requires `admin:ListPluginRuns` plus ownership of `{id}`. RustFS derives signer and delegation identity from the authenticated user and IAM record, then proxies Runner's bounded history response. The response contains at most the latest 32 committed state events in Runner's history-ID order and a `has_older` flag; timestamps are display data and are not used to reorder events. Missing and foreign runs share the same not-found response. RustFS rejects malformed, empty, over-limit, unexpected-state, or mismatched-run responses and bounds both request and response bodies.
`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/replay` accepts `{version:1,origin_run_id,idempotency_key,deadline_unix_micros}` and requires `admin:ReplayPluginRun`, current IAM authorization, ownership, and an active unexpired delegation. Caller identity fields are not accepted; RustFS derives signer and delegation revision from the authenticated request and IAM record. Runner accepts only an owned failed origin, preserves its fixed plugin/configuration/input version, and requires a new deadline no later than its configured run timeout. RustFS requires a positive Unix-microsecond deadline and verifies the strict receipt echoes it. Missing or foreign origins share not-found behavior; conflicts map to an operation conflict. Requests and receipts are bounded, and responses use `Cache-Control: no-store`.
## Broker profile
`rustfs/src/admin/handlers/plugin_attempt.rs` implements `POST /rustfs/admin/v3/plugin-delegations/{id}/issue-attempt`. The signer must be an ordinary IAM user with `admin:IssuePluginAttempt`, independently listed in `RUSTFS_PLUGIN_BROKERS`:
```json
{"broker-access-key":"deployment-name"}
```
This configuration contains public access keys, not secrets. It is immutable per process. Missing or invalid configuration disables broker issuance. The broker is a trusted Controller identity; ordinary users cannot substitute their own request fields for Controller validation.
The request carries `tenant`, `run_id`, `attempt_id`, `revision`, `deployment`, `input_bucket`, `input_key`, `input_version`, `output_bucket`, `output_prefix` and `ttl_seconds`. IDs must be non-nil. Both profiles accept one exact input version and an output prefix equal to:
```text
<delegation-output-prefix><lowercase-hex-tenant>/<run-uuid>/<attempt-uuid>/
```
The session policy permits only `s3:GetObjectVersion` for that input with an exact `s3:VersionId`, and `s3:PutObject` beneath that attempt prefix. Credential lifetime is at most 900 seconds and cannot exceed delegation expiry. The Controller must additionally cap it to its validated lease and hard deadline.
The original user's current IAM remains the parent permission boundary on every S3 request. The server's signing key only seals the token; it is not the plugin's authorization identity. Broker identity and original user identity are separate.
External policy plugins must be disabled: their authorization path does not enforce this built-in session-policy boundary. Initializing, failed and ready plugin states all reject issuance. Attempt tokens carry a signed, persisted marker: after restart or on another node, marked credentials are accepted only by built-in STS authorization. A denied marked request returns a terminal error, so public bucket policies cannot bypass its session scope. The first profile also rejects parent policies containing conditions, because a broker request cannot reconstruct the original user's IP, MFA or other request context. It does not silently replace that context with the broker's.
## Response and failures
Successful issuance returns `version`, `delegation_id`, `delegation_revision`, `owner_access_key`, `access_key_id`, `secret_access_key`, `session_token` and `expires_at_unix_seconds`, with `Cache-Control: no-store`. The Controller verifies the returned delegation and owner against the accepted caller before delivery. Credentials must never enter logs, metrics or task result manifests.
Missing records and records owned by a different user have the same read response. Revoked or expired records reject access. Storage failures remain errors. Post-persistence failures carry static `error_code`, `issued` and `issuance_unknown` facts without credentials; callers must reconcile an unknown outcome rather than automatically mint another credential.
The route/action inventory is maintained in `rustfs/src/admin/route_policy.rs`. This contract does not imply event capture, physical container cleanup, restart reconciliation or upstream release acceptance.
## Cross-bucket restriction profile v2
The existing registration endpoint accepts this explicit v2 restriction:
```json
{
"version": 2,
"deployment": "runner-production",
"bucket": "source-images",
"input_prefix": "inputs/",
"output_bucket": "plugin-artifacts",
"output_prefix": "outputs/",
"expires_at_unix_seconds": 1893456000
}
```
The example expiry must be replaced by a future value within the existing
24-hour limit. `bucket` still identifies the input bucket. The existing
`issue-attempt` request and credential response do not change: its
`input_bucket` must equal `bucket`, and `output_bucket` must equal the explicit
v2 output bucket. Neither a different bucket nor a sibling attempt prefix can
be supplied by the broker. The exact input version and deterministic output
prefix checks remain in force.
A v1 restriction has neither `version` nor `output_bucket`; its JSON fields
and same-bucket behavior are unchanged. V2 requires both `version: 2` and a
nonempty, bounded, literal `output_bucket`. Missing pairs, explicit nulls,
unknown versions and unknown fields are rejected rather than downgraded to v1.
The immutable record envelope carries version 1 or 2 to match its restriction
profile. Both remain in the existing IAM record namespace; no data migration
or mutable record update is introduced. Readers reject mismatched envelope
and restriction versions. Older readers reject v2, so deploy compatible
readers before creating v2 records.
V2 does not grant the original user additional permissions. Issuance retains
its current-parent checks and post-persistence recheck/invalidation, and each
S3 request still requires both the original user's current IAM policy and the
attempt session policy to allow it. An explicit parent Deny or removal of
output permission prevents writes even with an already issued token. No
administrator credential substitutes for missing parent permission.
Focused tests are `delegation_v1_json_remains_compatible`,
`delegation_v2_requires_explicit_output_bucket`,
`attempt_v2_policy_keeps_input_version_and_output_bucket` and
`delegation_v2_persists_and_vends_with_real_parent_intersection`.
The last uses the actual issuance handler, persistent IAM and four-disk
storage, including a real input version and a third bucket allowed by the
parent but forbidden by the delegation. Removing only the output-bucket
comparison must fail `A03_CROSS_BUCKET_OUTPUT_BOUND`; parent intersection is
checked independently with issued temporary credentials. These are internal
handler/IAM tests, not signed HTTP or complete cross-bucket S3 execution.
This profile adds no bucket-creation API, Runner binding support, credential
endpoint or GC permission. Coordinating Runner's pinned binding with its
result storage bucket remains a separate integration requirement.
+2
View File
@@ -52,6 +52,8 @@ pub mod object_data_cache;
pub mod object_zip_download;
pub mod oidc;
pub mod on_demand_migration;
pub mod plugin_attempt;
pub mod plugin_delegation;
pub mod plugins_catalog;
pub mod plugins_instances;
pub mod policies;
+568
View File
@@ -0,0 +1,568 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Restricted vending for explicitly configured brokers, never AssumeRole chaining.
use super::plugin_delegation::{owner, response, storage_error};
use crate::{
admin::{
router::{AdminOperation, Operation, S3Router},
runtime_sources,
storage_api::s3::{self, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result},
},
auth::check_key_valid_with_context,
server::ADMIN_PREFIX,
};
use http::{Method, StatusCode};
use matchit::Params;
use rustfs_iam::{
delegation::Delegation,
sys::{PLUGIN_ATTEMPT_CREDENTIAL_CLAIM, SESSION_POLICY_NAME},
};
use rustfs_policy::{
auth::get_new_credentials_with_metadata,
policy::{
Args, Policy,
action::{Action, AdminAction, S3Action},
},
};
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
sync::OnceLock,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use time::OffsetDateTime;
// Operator configuration contains public signer access keys, not credentials.
// Empty/invalid configuration disables the endpoint. It is immutable per process.
static BROKERS: OnceLock<Option<HashMap<String, String>>> = OnceLock::new();
fn configured_deployment(key: &str) -> Option<&'static str> {
BROKERS
.get_or_init(|| {
let raw = std::env::var("RUSTFS_PLUGIN_BROKERS").ok()?;
let map: HashMap<String, String> = serde_json::from_str(&raw).ok()?;
if map.is_empty()
|| map
.iter()
.any(|(k, v)| k.is_empty() || v.is_empty() || k.len() > 1024 || v.len() > 1024)
{
return None;
}
Some(map)
})
.as_ref()?
.get(key)
.map(String::as_str)
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Attempt {
tenant: String,
run_id: uuid::Uuid,
attempt_id: uuid::Uuid,
revision: String,
deployment: String,
input_bucket: String,
input_key: String,
input_version: String,
output_bucket: String,
output_prefix: String,
ttl_seconds: u64,
}
#[derive(Serialize)]
struct Issued {
version: u32,
delegation_id: String,
delegation_revision: String,
owner_access_key: String,
access_key_id: String,
secret_access_key: String,
session_token: String,
expires_at_unix_seconds: i64,
}
pub fn register_plugin_attempt_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/issue-attempt"),
AdminOperation(&Issue),
)?;
Ok(())
}
fn now() -> S3Result<u64> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))
}
fn literal(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 1024
&& !value.contains("${")
&& !value.chars().any(|c| c.is_control() || matches!(c, '*' | '?'))
}
fn session_policy(record: &Delegation, attempt: &Attempt) -> S3Result<Policy> {
let restriction = &record.restrictions;
// Stored records from earlier versions must obey the same literal rules.
restriction.validate(record.created_at_unix_seconds).map_err(storage_error)?;
let exact_output = format!(
"{}{}/{}/{}/",
restriction.output_prefix,
hex_simd::encode_to_string(attempt.tenant.as_bytes(), hex_simd::AsciiCase::Lower),
attempt.run_id,
attempt.attempt_id
);
if attempt.tenant.is_empty()
|| attempt.tenant.len() > 1024
|| attempt.run_id.is_nil()
|| attempt.attempt_id.is_nil()
|| attempt.output_prefix != exact_output
|| attempt.revision != record.revision
|| attempt.deployment != restriction.deployment
|| attempt.input_bucket != restriction.bucket
|| attempt.output_bucket != restriction.output_bucket().map_err(storage_error)?
|| !literal(&attempt.input_key)
|| !attempt.input_key.starts_with(&restriction.input_prefix)
|| !literal(&attempt.input_version)
|| attempt.input_version == "null"
|| !literal(&attempt.output_prefix)
|| !attempt.output_prefix.starts_with(&restriction.output_prefix)
|| attempt.output_prefix == restriction.output_prefix
|| !attempt.output_prefix.ends_with('/')
|| !(1..=900).contains(&attempt.ttl_seconds)
{
return Err(s3::error(S3ErrorCode::AccessDenied, "attempt restriction rejected"));
}
let value = serde_json::json!({"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":["s3:GetObjectVersion"],"Resource":[format!("arn:aws:s3:::{}/{}",attempt.input_bucket,attempt.input_key)],"Condition":{"StringEquals":{"s3:VersionId":attempt.input_version}}},
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":[format!("arn:aws:s3:::{}/{}*",attempt.output_bucket,attempt.output_prefix)]}
]});
// Condition operators borrow their keys while the policy is decoded.
let bytes = serde_json::to_vec(&value).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))?;
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))
}
fn issuance_error(code: &'static str, issued: bool, unknown: bool) -> S3Result<S3Response<(StatusCode, Body)>> {
let mut answer = response(&serde_json::json!({"error_code":code,"issued":issued,"issuance_unknown":unknown}))?;
answer.output.0 = StatusCode::SERVICE_UNAVAILABLE;
Ok(answer)
}
fn same_parent_authority(before_names: &[String], before_policy: &Policy, after_names: &[String], after_policy: &Policy) -> bool {
fn names(names: &[String]) -> Vec<&str> {
let mut names: Vec<_> = names.iter().map(String::as_str).collect();
names.sort_unstable();
names.dedup();
names
}
fn statements(policy: &Policy) -> Option<Vec<Vec<u8>>> {
let mut statements = policy
.statements
.iter()
.map(serde_json::to_vec)
.collect::<Result<Vec<_>, _>>()
.ok()?;
statements.sort_unstable();
Some(statements)
}
names(before_names) == names(after_names)
&& matches!(
(statements(before_policy), statements(after_policy)),
(Some(before), Some(after)) if before == after
)
}
async fn require_builtin_authorization() -> S3Result<()> {
rustfs_iam::sys::IamSys::<rustfs_iam::store::object::ObjectStore>::ensure_restricted_credential_vending()
.await
.map_err(|_| s3::error(S3ErrorCode::AccessDenied, "external policy plugin unsupported for attempt credentials"))
}
struct Issue;
#[async_trait::async_trait]
impl Operation for Issue {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
require_builtin_authorization().await?;
let (iam, broker) = owner(&req, AdminAction::IssuePluginAttemptAction).await?;
let deployment =
configured_deployment(&broker).ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "broker deployment rejected"))?;
let context = runtime_sources::app_context_from_req(&req)
.ok_or_else(|| s3::error(S3ErrorCode::InternalError, "application context unavailable"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "attempt body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "attempt body unavailable"))?;
let attempt: Attempt =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid attempt request"))?;
if attempt.deployment != deployment {
return Err(s3::error(S3ErrorCode::AccessDenied, "broker deployment rejected"));
}
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let record = iam
.read_active_plugin_delegation_for_broker(id, now()?)
.await
.map_err(storage_error)?;
let policy = session_policy(&record, &attempt)?;
let (principal, root) = check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
if root || principal.is_temp() || principal.is_service_account() {
return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
}
// A broker request cannot reconstruct the original user's IP/MFA/tag
// conditions. This first profile rejects conditional parent policies.
let policies = iam
.policy_db_get(&principal.access_key, &principal.groups)
.await
.map_err(|_| s3::error(S3ErrorCode::InternalError, "parent policy unavailable"))?;
let parent_policy = iam.get_combined_policy(&policies).await;
if policies.is_empty() || parent_policy.statements.iter().any(|s| !s.conditions.is_empty()) {
return Err(s3::error(S3ErrorCode::AccessDenied, "parent condition context unsupported"));
}
let mut conditions = HashMap::new();
conditions.insert("s3:VersionId".to_owned(), vec![attempt.input_version.clone()]);
if !iam
.is_allowed(&Args {
account: &principal.access_key,
groups: &principal.groups,
action: Action::S3Action(S3Action::GetObjectVersionAction),
bucket: &attempt.input_bucket,
object: &attempt.input_key,
conditions: &conditions,
is_owner: false,
claims: principal.claims_or_empty(),
deny_only: false,
})
.await
{
return Err(s3::error(S3ErrorCode::AccessDenied, "parent input rejected"));
}
let expiration = now()?
.checked_add(attempt.ttl_seconds)
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?
.min(record.restrictions.expires_at_unix_seconds);
let expires_at = OffsetDateTime::from_unix_timestamp(
i64::try_from(expiration).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?,
)
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?;
let mut claims = principal.claims.clone().unwrap_or_default();
claims.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), serde_json::Value::Bool(true));
claims.insert("exp".into(), serde_json::json!(expires_at.unix_timestamp()));
claims.insert("parent".into(), serde_json::json!(principal.access_key));
let bytes =
serde_json::to_vec(&policy).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))?;
claims.insert(
SESSION_POLICY_NAME.into(),
serde_json::json!(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&bytes)),
);
let fresh = iam
.read_active_plugin_delegation_for_broker(id, now()?)
.await
.map_err(storage_error)?;
if fresh != record {
return Err(s3::error(S3ErrorCode::AccessDenied, "attempt authority changed"));
}
owner(&req, AdminAction::IssuePluginAttemptAction).await?;
require_builtin_authorization().await?;
let signing = context
.action_credentials()
.get()
.ok_or_else(|| s3::error(S3ErrorCode::InternalError, "token signer unavailable"))?;
// The server key seals the token only. Authorization above uses the
// original user's current IAM, and S3 continues its parent intersection.
let mut credential = get_new_credentials_with_metadata(&claims, &signing.secret_key)
.map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt credential unavailable"))?;
credential.parent_user = principal.access_key.clone();
if iam.set_temp_user(&credential.access_key, &credential, None).await.is_err() {
return issuance_error("PLUGIN_ATTEMPT_ISSUANCE_UNKNOWN", false, true);
}
// No atomic IAM/delegation transaction spans persistence. Recheck and
// invalidate the generated credential if any authority changed.
let valid = async {
require_builtin_authorization().await?;
let after = iam
.read_active_plugin_delegation_for_broker(id, now()?)
.await
.map_err(storage_error)?;
let (current, current_root) =
check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
let names = iam
.policy_db_get(&current.access_key, &current.groups)
.await
.map_err(|_| s3::error(S3ErrorCode::InternalError, "parent policy unavailable"))?;
let current_policy = iam.get_combined_policy(&names).await;
if after != record
|| current_root
|| current.is_temp()
|| current.is_service_account()
|| current.secret_key != principal.secret_key
|| !same_parent_authority(&policies, &parent_policy, &names, &current_policy)
{
return Err(s3::error(S3ErrorCode::AccessDenied, "attempt authority changed"));
}
// Re-evaluate the broker against its real signed request context.
owner(&req, AdminAction::IssuePluginAttemptAction).await?;
Ok::<(), S3Error>(())
}
.await;
if valid.is_err() {
if iam.delete_temp_account(&credential.access_key, true).await.is_err() {
return issuance_error("PLUGIN_ATTEMPT_INVALIDATION_UNKNOWN", true, true);
}
return issuance_error("PLUGIN_ATTEMPT_POST_ISSUANCE_REJECTED", true, false);
}
response(&Issued {
version: 1,
delegation_id: record.id,
delegation_revision: record.revision,
owner_access_key: record.owner_access_key,
access_key_id: credential.access_key,
secret_access_key: credential.secret_key,
session_token: credential.session_token,
expires_at_unix_seconds: expires_at.unix_timestamp(),
})
}
}
#[cfg(test)]
#[path = "plugin_attempt_v2_tests.rs"]
mod v2_tests;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parent_authority_recheck_ignores_policy_order_but_detects_changes() {
let first: Policy = serde_json::from_value(serde_json::json!({
"Version":"2012-10-17",
"Statement":[
{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"}
]
}))
.unwrap();
let reordered: Policy = serde_json::from_value(serde_json::json!({
"Version":"2012-10-17",
"Statement":[
{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"},
{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"}
]
}))
.unwrap();
let before_names = vec!["owner-policy".to_owned(), "bucket-policy".to_owned()];
let after_names = vec!["bucket-policy".to_owned(), "owner-policy".to_owned()];
assert!(same_parent_authority(&before_names, &first, &after_names, &reordered));
let changed: Policy = serde_json::from_value(serde_json::json!({
"Version":"2012-10-17",
"Statement":[
{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
{"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::photos/output/*"}
]
}))
.unwrap();
assert!(!same_parent_authority(&before_names, &first, &after_names, &changed));
let changed_resource: Policy = serde_json::from_value(serde_json::json!({
"Version":"2012-10-17",
"Statement":[
{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/private/*"}
]
}))
.unwrap();
assert!(!same_parent_authority(&before_names, &first, &after_names, &changed_resource));
let added_deny: Policy = serde_json::from_value(serde_json::json!({
"Version":"2012-10-17",
"Statement":[
{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/private/*"},
{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"}
]
}))
.unwrap();
assert!(!same_parent_authority(&before_names, &first, &after_names, &added_deny));
assert!(!same_parent_authority(&before_names, &first, &["other-policy".to_owned()], &first));
}
#[tokio::test]
async fn attempt_policy_keeps_exact_version_and_narrow_output() {
let record = Delegation {
version: 1,
id: "a".repeat(32),
revision: "a".repeat(32),
owner_access_key: "user".into(),
created_at_unix_seconds: 1,
restrictions: rustfs_iam::delegation::Restrictions {
version: None,
output_bucket: None,
deployment: "runner".into(),
bucket: "photos".into(),
input_prefix: "input/".into(),
output_prefix: "output/".into(),
expires_at_unix_seconds: 1000,
},
};
let run_id = uuid::Uuid::new_v4();
let attempt_id = uuid::Uuid::new_v4();
let output_prefix = format!(
"output/{}/{}/{}/",
hex_simd::encode_to_string(b"tenant", hex_simd::AsciiCase::Lower),
run_id,
attempt_id
);
let mut attempt = Attempt {
tenant: "tenant".into(),
run_id,
attempt_id,
revision: record.revision.clone(),
deployment: "runner".into(),
input_bucket: "photos".into(),
input_key: "input/photo.png".into(),
input_version: "version-1".into(),
output_bucket: "photos".into(),
output_prefix: output_prefix.clone(),
ttl_seconds: 60,
};
let policy = session_policy(&record, &attempt).unwrap();
let scoped_output = format!("{output_prefix}result");
let groups = None;
let claims = HashMap::new();
for (action, object, version, expected) in [
(S3Action::GetObjectVersionAction, "input/photo.png", Some("version-1"), true),
(S3Action::GetObjectVersionAction, "input/photo.png", Some("version-2"), false),
(S3Action::GetObjectVersionAction, "input/photo.png", None, false),
(S3Action::GetObjectAction, "input/photo.png", None, false),
(S3Action::PutObjectAction, scoped_output.as_str(), None, true),
(S3Action::PutObjectAction, "outside/result", None, false),
(S3Action::PutObjectAction, "output/result", None, false),
] {
let conditions = version
.map(|value| HashMap::from([("versionid".into(), vec![value.into()])]))
.unwrap_or_default();
assert_eq!(
policy
.is_allowed(&Args {
account: "user",
groups: &groups,
action: Action::S3Action(action),
bucket: "photos",
object,
conditions: &conditions,
is_owner: false,
claims: &claims,
deny_only: false,
})
.await,
expected,
"attempt policy action={action:?} object={object} version={version:?}"
);
}
attempt.output_bucket = "artifacts".into();
assert!(
session_policy(&record, &attempt).is_err(),
"v1 still requires the input bucket for output"
);
attempt.output_bucket = "photos".into();
attempt.output_prefix = "output/".into();
assert!(session_policy(&record, &attempt).is_err());
attempt.output_prefix = output_prefix;
attempt.input_key = "outside/photo.png".into();
assert!(session_policy(&record, &attempt).is_err());
}
#[tokio::test]
async fn attempt_v2_policy_keeps_input_version_and_output_bucket() {
let record: Delegation = serde_json::from_value(serde_json::json!({
"version":2,"id":"a".repeat(32),"revision":"a".repeat(32),"owner_access_key":"user",
"created_at_unix_seconds":1,"restrictions":{"version":2,"deployment":"runner","bucket":"photos",
"input_prefix":"input/","output_bucket":"artifacts","output_prefix":"output/","expires_at_unix_seconds":1000}
}))
.expect("explicit v2 delegation");
let run_id = uuid::Uuid::new_v4();
let attempt_id = uuid::Uuid::new_v4();
let output = format!("output/74656e616e74/{run_id}/{attempt_id}/");
let mut attempt = Attempt {
tenant: "tenant".into(),
run_id,
attempt_id,
revision: record.revision.clone(),
deployment: "runner".into(),
input_bucket: "photos".into(),
input_key: "input/photo.png".into(),
input_version: "version-1".into(),
output_bucket: "artifacts".into(),
output_prefix: output.clone(),
ttl_seconds: 60,
};
let policy = session_policy(&record, &attempt).expect("v2 scope");
let result = format!("{output}result");
for (action, bucket, object, version, expected) in [
(S3Action::GetObjectVersionAction, "photos", "input/photo.png", Some("version-1"), true),
(S3Action::GetObjectVersionAction, "photos", "input/photo.png", Some("version-2"), false),
(S3Action::GetObjectVersionAction, "artifacts", "input/photo.png", Some("version-1"), false),
(S3Action::PutObjectAction, "artifacts", result.as_str(), None, true),
(S3Action::PutObjectAction, "photos", result.as_str(), None, false),
(S3Action::PutObjectAction, "third", result.as_str(), None, false),
(S3Action::PutObjectAction, "artifacts", "output/other/result", None, false),
] {
let conditions = version
.map(|v| HashMap::from([("versionid".into(), vec![v.into()])]))
.unwrap_or_default();
assert_eq!(
policy
.is_allowed(&Args {
account: "user",
groups: &None,
action: Action::S3Action(action),
bucket,
object,
conditions: &conditions,
claims: &HashMap::new(),
is_owner: false,
deny_only: false
})
.await,
expected,
"A03_CROSS_BUCKET_SCOPE: bucket={bucket} object={object}"
);
}
attempt.output_bucket = "third".into();
assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_OUTPUT_BOUND");
attempt.output_bucket = "photos".into();
assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_OUTPUT_BOUND");
attempt.output_bucket = "artifacts".into();
attempt.input_bucket = "artifacts".into();
assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_INPUT_BOUND");
}
#[tokio::test]
async fn actual_policy_variable_resolution_is_rejected_by_literal_scope() {
let resource = rustfs_policy::policy::resource::Resource::S3("photos/input/${aws:username}.png".into());
let conditions = HashMap::from([("username".into(), vec!["alice".into()])]);
let resolver =
rustfs_policy::policy::variables::VariableResolver::new(rustfs_policy::policy::variables::VariableContext {
username: Some("alice".into()),
..Default::default()
});
assert!(
resource
.is_match_with_resolver("photos/input/alice.png", &conditions, Some(&resolver))
.await,
"C07_LITERAL_SCOPE_BOUND: actual policy engine substitutes resource variables"
);
assert!(
!literal("input/${aws:username}.png"),
"C07_LITERAL_SCOPE_BOUND: requested resources must stay literal"
);
let restriction = rustfs_iam::delegation::Restrictions {
version: None,
output_bucket: None,
deployment: "runner".into(),
bucket: "photos".into(),
input_prefix: "input/${aws:username}/".into(),
output_prefix: "output/".into(),
expires_at_unix_seconds: 200,
};
assert!(
restriction.validate(100).is_err(),
"C07_LITERAL_SCOPE_BOUND: legacy stored variable restrictions are rejected"
);
}
}
@@ -0,0 +1,365 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Real handler, persisted IAM and four-disk storage; this is not a signed HTTP/S3 end-to-end test.
use super::*;
use crate::admin::{
runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot},
storage_api::{
ECStore,
contract::bucket::{BucketOperations as _, MakeBucketOptions},
contract::object::ObjectIO as _,
},
};
use crate::storage::storage_api::{StorageObjectOptions, StoragePutObjReader};
use http_body_util::BodyExt as _;
use rustfs_iam::{
store::{
Store as _, UserType,
object::{IAM_CONFIG_PREFIX, ObjectStore},
},
sys::IamSys,
};
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
use serde_json::{Value, json};
use std::sync::Arc;
const PARENT: &str = "cross-parent";
const BROKER: &str = "cross-broker";
const SECRET: &str = "cross-bucket-test-secret";
const ROOT: &str = "cross-root";
const ROOT_SECRET: &str = "cross-bucket-root-test-secret";
struct RequestIam(Arc<IamSys<ObjectStore>>);
impl IamInterface for RequestIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
self.0.clone()
}
fn is_ready(&self) -> bool {
true
}
}
struct RequestKms;
impl KmsInterface for RequestKms {
fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
Arc::new(rustfs_kms::KmsServiceManager::new())
}
}
struct Fixture {
_root: tempfile::TempDir,
store: Arc<ECStore>,
iam: Arc<IamSys<ObjectStore>>,
context: Arc<AppContext>,
input_version: String,
}
impl Fixture {
async fn new() -> Self {
let root = tempfile::tempdir().expect("cross-bucket storage root");
let env = rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(root.path())
.disk_count(4)
.build()
.await;
assert_eq!(env.disk_paths.len(), 4);
let store = env.ecstore;
for bucket in ["photos", "artifacts", "third"] {
store
.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("real versioned bucket");
}
let object = store
.put_object(
"photos",
"input/photo.png",
&mut StoragePutObjReader::from_vec(vec![0xA5; 273]),
&StorageObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("real input version");
let input_version = object.version_id.expect("exact input version").to_string();
ObjectStore::new(store.clone())
.save_iam_config(json!({"version":1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
.await
.expect("persistent IAM format");
let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("real IAM");
let context = Arc::new(AppContext::new(store.clone(), Arc::new(RequestIam(iam.clone())), Arc::new(RequestKms)));
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
access_key: ROOT.into(),
secret_key: ROOT_SECRET.into(),
status: "on".into(),
..Default::default()
}));
let fixture = Self {
_root: root,
store,
iam,
context,
input_version,
};
fixture.user(PARENT, Self::parent_policy(true, false)).await;
fixture
.user(
BROKER,
json!({"Version":"2012-10-17","Statement":[{
"Effect":"Allow","Action":"admin:IssuePluginAttempt"}]}),
)
.await;
fixture
}
fn parent_policy(output: bool, deny: bool) -> Value {
let mut resources = vec!["arn:aws:s3:::photos/output/*", "arn:aws:s3:::third/output/*"];
if output {
resources.push("arn:aws:s3:::artifacts/output/*");
}
let mut statements = vec![
json!({"Effect":"Allow","Action":"s3:GetObjectVersion","Resource":"arn:aws:s3:::photos/input/*"}),
json!({"Effect":"Allow","Action":"s3:PutObject","Resource":resources}),
];
if deny {
statements.push(json!({"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::artifacts/*"}));
}
json!({"Version":"2012-10-17","Statement":statements})
}
async fn policy(&self, name: &str, value: Value) {
let policy = Policy::parse_config(&serde_json::to_vec(&value).expect("policy JSON")).expect("real IAM policy");
self.iam.set_policy(name, policy).await.expect("persist policy");
}
async fn user(&self, name: &str, policy: Value) {
self.iam
.create_user(
name,
&AddOrUpdateUserReq {
secret_key: SECRET.into(),
policy: None,
status: AccountStatus::Enabled,
},
)
.await
.expect("create real user");
self.policy(name, policy).await;
self.iam
.policy_db_set(name, UserType::Reg, false, name)
.await
.expect("attach actual IAM policy");
}
fn request(&self, body: &Value) -> S3Request<Body> {
let slot = ServerContextSlot::new();
assert!(slot.install(self.context.clone()));
let mut req = S3Request {
input: Body::from(serde_json::to_vec(body).expect("attempt JSON")),
method: Method::POST,
uri: http::Uri::default(),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
req.extensions.insert(slot);
req.method = Method::POST;
req.credentials = Some(s3::auth::Credentials {
access_key: BROKER.into(),
secret_key: s3::auth::SecretKey::from(SECRET.to_owned()),
});
req
}
async fn issue(&self, record: &Delegation, body: &Value) -> S3Result<S3Response<(StatusCode, Body)>> {
let mut router = matchit::Router::new();
router.insert("/{id}", ()).expect("parameter route");
let path = format!("/{}", record.id);
Issue
.call(self.request(body), router.at(&path).expect("actual delegation ID").params)
.await
}
async fn allowed(
&self,
credential: &rustfs_credentials::Credentials,
action: S3Action,
bucket: &str,
object: &str,
version: Option<&str>,
) -> bool {
let conditions = version
.map(|v| HashMap::from([("versionid".into(), vec![v.into()])]))
.unwrap_or_default();
self.iam
.is_allowed(&Args {
account: &credential.access_key,
groups: &credential.groups,
action: Action::S3Action(action),
bucket,
object,
conditions: &conditions,
claims: credential.claims_or_empty(),
is_owner: false,
deny_only: false,
})
.await
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial]
async fn delegation_v2_persists_and_vends_with_real_parent_intersection() {
temp_env::async_with_vars(
[
("RUSTFS_PLUGIN_BROKERS", Some(r#"{"cross-broker":"runner"}"#)),
("RUSTFS_SKIP_BACKGROUND_TASK", Some("1")),
],
async {
let fixture = Fixture::new().await;
let now = now().expect("clock");
let legacy = json!({"deployment":"runner","bucket":"photos","input_prefix":"input/",
"output_prefix":"output/","expires_at_unix_seconds":now+3600});
let v1 = fixture
.iam
.create_plugin_delegation(PARENT, serde_json::from_value(legacy.clone()).expect("v1"), now)
.await
.expect("persist legacy delegation");
assert_eq!(v1.version, 1);
assert_eq!(serde_json::to_value(&v1.restrictions).expect("v1 wire"), legacy);
let mut v2 = legacy;
v2["version"] = json!(2);
v2["output_bucket"] = json!("artifacts");
let record = fixture
.iam
.create_plugin_delegation(PARENT, serde_json::from_value(v2.clone()).expect("v2"), now)
.await
.expect("persist v2 delegation");
assert_eq!(record.version, 2);
let loaded = fixture
.iam
.read_active_plugin_delegation(PARENT, &record.id, now)
.await
.expect("strict stored v2");
assert!(loaded == record, "same persisted v2 identity");
assert_eq!(serde_json::to_value(&loaded.restrictions).expect("v2 wire"), v2);
// The record envelope and restriction profile cannot disagree on a persisted read.
let path = format!("config/iam/plugin-delegations/v1/{}/record.json", record.id);
let mut corrupt = serde_json::to_value(&record).expect("record JSON");
corrupt["version"] = json!(1);
ObjectStore::new(fixture.store.clone())
.save_iam_config(corrupt, &path)
.await
.expect("inject incompatible stored version");
assert!(
fixture
.iam
.read_active_plugin_delegation(PARENT, &record.id, now)
.await
.is_err(),
"mixed v1/v2 stored profiles fail closed"
);
ObjectStore::new(fixture.store.clone())
.save_iam_config(&record, &path)
.await
.expect("restore exact stored record");
let run = uuid::Uuid::new_v4();
let attempt = uuid::Uuid::new_v4();
let prefix = format!("output/74656e616e74/{run}/{attempt}/");
let body = json!({"tenant":"tenant","run_id":run,"attempt_id":attempt,"revision":record.revision,
"deployment":"runner","input_bucket":"photos","input_key":"input/photo.png","input_version":fixture.input_version,
"output_bucket":"artifacts","output_prefix":prefix,"ttl_seconds":60});
let loaded_v1 = fixture
.iam
.read_active_plugin_delegation(PARENT, &v1.id, now)
.await
.expect("read legacy record");
assert!(loaded_v1 == v1, "legacy persistent identity");
let mut legacy_body = body.clone();
legacy_body["revision"] = json!(v1.revision);
legacy_body["output_bucket"] = json!("photos");
let legacy_response = fixture.issue(&v1, &legacy_body).await.expect("legacy same-bucket issuance");
assert_eq!(legacy_response.output.0, StatusCode::OK);
let mut wrong = body.clone();
wrong["output_bucket"] = json!("third");
match fixture.issue(&record, &wrong).await {
Err(error) => assert_eq!(error.code(), &S3ErrorCode::AccessDenied),
Ok(_) => panic!("A03_CROSS_BUCKET_OUTPUT_BOUND"),
}
let response = fixture.issue(&record, &body).await.expect("actual v2 issuance");
assert_eq!(response.output.0, StatusCode::OK);
assert_eq!(response.headers[http::header::CACHE_CONTROL], "no-store");
let issued: Value =
serde_json::from_slice(&response.output.1.collect().await.expect("issued body").to_bytes()).expect("issued JSON");
let (credential, root) = check_key_valid_with_context(
issued["session_token"].as_str().expect("token"),
issued["access_key_id"].as_str().expect("access key"),
Some(fixture.context.as_ref()),
)
.await
.expect("persisted actual temporary identity");
assert!(!root && credential.is_temp());
let output = format!("{prefix}image.png");
for (action, bucket, object, version, expected) in [
(
S3Action::GetObjectVersionAction,
"photos",
"input/photo.png",
Some(fixture.input_version.as_str()),
true,
),
(
S3Action::GetObjectVersionAction,
"photos",
"input/photo.png",
Some("wrong-version"),
false,
),
(S3Action::GetObjectAction, "photos", "input/photo.png", None, false),
(S3Action::PutObjectAction, "artifacts", output.as_str(), None, true),
(S3Action::PutObjectAction, "photos", output.as_str(), None, false),
(S3Action::PutObjectAction, "third", output.as_str(), None, false),
(S3Action::PutObjectAction, "artifacts", "output/other/image.png", None, false),
] {
assert_eq!(
fixture.allowed(&credential, action, bucket, object, version).await,
expected,
"A03_CROSS_BUCKET_PARENT_INTERSECTION: bucket={bucket} object={object}"
);
}
fixture.policy(PARENT, Fixture::parent_policy(true, true)).await;
assert!(
!fixture
.allowed(&credential, S3Action::PutObjectAction, "artifacts", &output, None)
.await,
"explicit parent Deny remains effective"
);
fixture.policy(PARENT, Fixture::parent_policy(false, false)).await;
assert!(
!fixture
.allowed(&credential, S3Action::PutObjectAction, "artifacts", &output, None)
.await,
"removing parent output permission remains effective"
);
assert!(
fixture
.allowed(
&credential,
S3Action::GetObjectVersionAction,
"photos",
"input/photo.png",
Some(&fixture.input_version)
)
.await
);
fixture
.iam
.revoke_plugin_delegation(PARENT, &record.id)
.await
.expect("revoke v2");
assert!(fixture.issue(&record, &body).await.is_err(), "revoked v2 cannot issue again");
},
)
.await;
}
@@ -0,0 +1,624 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Restriction registration only. These endpoints do not vend credentials.
use crate::admin::{
auth::{AdminResourceScope, validate_admin_action_with_bucket_object_for_iam},
plugin_runner::{self, ObjectReference, SubmitRequest},
router::{AdminOperation, Operation, S3Router},
runtime_sources,
storage_api::s3::{self, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result},
};
use crate::{
auth::{check_key_valid_with_context, get_session_token},
server::{ADMIN_PREFIX, RemoteAddr},
};
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use matchit::Params;
use rustfs_iam::delegation::{DelegationError, Restrictions};
use rustfs_policy::policy::action::{Action, AdminAction};
use serde::Deserialize;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
pub fn register_plugin_delegation_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
router.insert(Method::POST, &format!("{ADMIN_PREFIX}/v3/plugin-delegations"), AdminOperation(&Create))?;
router.insert(Method::GET, &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}"), AdminOperation(&Get))?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/revoke"),
AdminOperation(&Revoke),
)?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/submit"),
AdminOperation(&Submit),
)?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/query"),
AdminOperation(&QueryRun),
)?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/history"),
AdminOperation(&HistoryRun),
)?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/cancel"),
AdminOperation(&CancelRun),
)?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/page"),
AdminOperation(&PageRuns),
)?;
router.insert(
Method::POST,
&format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/replay"),
AdminOperation(&ReplayRun),
)?;
Ok(())
}
pub(super) fn storage_error(error: DelegationError) -> S3Error {
match error {
DelegationError::Invalid => s3::error(S3ErrorCode::InvalidRequest, "invalid delegation restriction"),
DelegationError::NotFound | DelegationError::OwnerRejected => s3::error(S3ErrorCode::NoSuchKey, "delegation not found"),
DelegationError::Unavailable => s3::error(S3ErrorCode::InternalError, "delegation unavailable"),
_ => s3::error(S3ErrorCode::AccessDenied, "delegation rejected"),
}
}
pub(super) async fn owner(
req: &S3Request<Body>,
action: AdminAction,
) -> S3Result<(std::sync::Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>, String)> {
let signed = req
.credentials
.as_ref()
.ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?;
if get_session_token(&req.uri, &req.headers).is_some() {
return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
}
let context = runtime_sources::app_context_from_req(req)
.ok_or_else(|| s3::error(S3ErrorCode::InternalError, "application context unavailable"))?;
let (credentials, root) = check_key_valid_with_context("", &signed.access_key, Some(context.as_ref())).await?;
if root || credentials.is_temp() || credentials.is_service_account() {
return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
}
if !context.iam().is_ready() {
return Err(s3::error(S3ErrorCode::InternalError, "iam unavailable"));
}
let iam = context.iam().handle();
validate_admin_action_with_bucket_object_for_iam(
iam.clone(),
&req.headers,
&credentials,
false,
Action::AdminAction(action),
req.extensions.get::<Option<RemoteAddr>>().and_then(|a| a.map(|a| a.0)),
AdminResourceScope::bucket_object("", ""),
)
.await?;
Ok((iam, credentials.access_key))
}
pub(super) fn response<T: serde::Serialize>(value: &T) -> S3Result<S3Response<(StatusCode, Body)>> {
let bytes =
serde_json::to_vec(value).map_err(|_| s3::error(S3ErrorCode::InternalError, "delegation response unavailable"))?;
let mut headers = HeaderMap::new();
headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers))
}
struct Create;
#[async_trait::async_trait]
impl Operation for Create {
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, owner) = owner(&req, AdminAction::CreatePluginDelegationAction).await?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "delegation body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "delegation body unavailable"))?;
let restriction: Restrictions = serde_json::from_slice(&bytes)
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid delegation restriction"))?;
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
.as_secs();
let record = iam
.create_plugin_delegation(&owner, restriction, now)
.await
.map_err(storage_error)?;
response(&record)
}
}
struct Get;
#[async_trait::async_trait]
impl Operation for Get {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, owner) = owner(&req, AdminAction::GetPluginDelegationAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
.as_secs();
let record = iam
.read_active_plugin_delegation(&owner, id, now)
.await
.map_err(storage_error)?;
response(&record)
}
}
struct Revoke;
#[async_trait::async_trait]
impl Operation for Revoke {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, owner) = owner(&req, AdminAction::RevokePluginDelegationAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
iam.revoke_plugin_delegation(&owner, id).await.map_err(storage_error)?;
response(&serde_json::json!({"revoked":true,"id":id}))
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct SubmitInput {
version: u32,
instance_id: String,
idempotency_key: String,
object: SubmitObject,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct SubmitObject {
bucket: String,
key: String,
version_id: String,
}
struct Submit;
#[async_trait::async_trait]
impl Operation for Submit {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, signer) = owner(&req, AdminAction::SubmitPluginJobAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "submit body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "submit body unavailable"))?;
let input: SubmitInput =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid submit request"))?;
if input.version != 1
|| !plugin_runner::literal(&input.instance_id)
|| !plugin_runner::valid_idempotency_key(&input.idempotency_key)
{
return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid submit request"));
}
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
.as_secs();
let record = iam
.read_active_plugin_delegation(&signer, id, now)
.await
.map_err(storage_error)?;
let deployment = plugin_runner::configured_deployment()?;
if !plugin_runner::validate_delegation_scope(
&record,
&signer,
id,
&deployment,
ObjectReference {
bucket: &input.object.bucket,
key: &input.object.key,
version_id: &input.object.version_id,
},
now,
) {
return Err(s3::error(S3ErrorCode::AccessDenied, "submit restriction rejected"));
}
let receipt = plugin_runner::submit(&SubmitRequest {
version: input.version,
instance_id: &input.instance_id,
idempotency_key: &input.idempotency_key,
object: ObjectReference {
bucket: &input.object.bucket,
key: &input.object.key,
version_id: &input.object.version_id,
},
delegation_id: &record.id,
delegation_revision: &record.revision,
actual_signer_access_key: &signer,
})
.await?;
response(&receipt)
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct QueryRunInput {
version: u32,
run_id: uuid::Uuid,
}
struct QueryRun;
#[async_trait::async_trait]
impl Operation for QueryRun {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, signer) = owner(&req, AdminAction::GetPluginRunAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "query body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "query body unavailable"))?;
let input: QueryRunInput =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid query request"))?;
if input.version != 1 || input.run_id.is_nil() {
return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid query request"));
}
let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
let result = plugin_runner::query(&plugin_runner::QueryRequest {
version: input.version,
run_id: input.run_id,
delegation_id: &record.id,
delegation_revision: &record.revision,
actual_signer_access_key: &signer,
})
.await?;
response(&result)
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct HistoryRunInput {
version: u32,
run_id: uuid::Uuid,
}
fn valid_history_run_input(input: &HistoryRunInput) -> bool {
input.version == 1 && !input.run_id.is_nil()
}
struct HistoryRun;
#[async_trait::async_trait]
impl Operation for HistoryRun {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, signer) = owner(&req, AdminAction::ListPluginRunsAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "history body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "history body unavailable"))?;
let input: HistoryRunInput =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid history request"))?;
if !valid_history_run_input(&input) {
return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid history request"));
}
let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
let result = plugin_runner::history(&plugin_runner::HistoryRequest {
version: input.version,
run_id: input.run_id,
delegation_id: &record.id,
delegation_revision: &record.revision,
actual_signer_access_key: &signer,
})
.await?;
response(&result)
}
}
struct CancelRun;
#[async_trait::async_trait]
impl Operation for CancelRun {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, signer) = owner(&req, AdminAction::CancelPluginRunAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "cancel body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "cancel body unavailable"))?;
let input: QueryRunInput =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid cancel request"))?;
if input.version != 1 || input.run_id.is_nil() {
return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid cancel request"));
}
let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
let result = plugin_runner::cancel(&plugin_runner::CancelRequest {
version: input.version,
run_id: input.run_id,
delegation_id: &record.id,
delegation_revision: &record.revision,
actual_signer_access_key: &signer,
})
.await?;
response(&result)
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct PageRunsInput {
version: u32,
limit: u32,
#[serde(default)]
cursor: Option<plugin_runner::RunPageCursor>,
}
fn valid_page_runs_input(input: &PageRunsInput) -> bool {
input.version == 1 && (1..=100).contains(&input.limit) && !input.cursor.is_some_and(|cursor| cursor.run_id.is_nil())
}
struct PageRuns;
#[async_trait::async_trait]
impl Operation for PageRuns {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, signer) = owner(&req, AdminAction::ListPluginRunsAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "page body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "page body unavailable"))?;
let input: PageRunsInput =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid page request"))?;
if !valid_page_runs_input(&input) {
return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid page request"));
}
let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
let result = plugin_runner::page(&plugin_runner::PageRequest {
version: input.version,
actual_signer_access_key: &signer,
delegation_id: &record.id,
delegation_revision: &record.revision,
limit: input.limit,
cursor: input.cursor,
})
.await?;
response(&result)
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct ReplayRunInput {
version: u32,
origin_run_id: uuid::Uuid,
idempotency_key: String,
deadline_unix_micros: i64,
}
fn valid_replay_run_input(input: &ReplayRunInput) -> bool {
input.version == 1
&& !input.origin_run_id.is_nil()
&& plugin_runner::valid_idempotency_key(&input.idempotency_key)
&& input.deadline_unix_micros > 0
}
struct ReplayRun;
#[async_trait::async_trait]
impl Operation for ReplayRun {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let (iam, signer) = owner(&req, AdminAction::ReplayPluginRunAction).await?;
let id = params
.get("id")
.ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
.await
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "replay body unavailable"))?
.map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "replay body unavailable"))?;
let input: ReplayRunInput =
serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid replay request"))?;
if !valid_replay_run_input(&input) {
return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid replay request"));
}
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
.as_secs();
let record = iam
.read_active_plugin_delegation(&signer, id, now)
.await
.map_err(storage_error)?;
let result = plugin_runner::replay(&plugin_runner::ReplayRequest {
version: input.version,
origin_run_id: input.origin_run_id,
idempotency_key: &input.idempotency_key,
deadline_unix_micros: input.deadline_unix_micros,
actual_signer_access_key: &signer,
delegation_id: &record.id,
delegation_revision: &record.revision,
})
.await?;
response(&result)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn delegation_mutations_require_authenticated_original_user() {
for operation in 0..8 {
let req = S3Request {
input: Body::from("{}".to_owned()),
method: Method::POST,
uri: http::Uri::from_static("/rustfs/admin/v3/plugin-delegations"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let result = if operation == 0 {
Get.call(req, Params::new()).await
} else if operation == 1 {
Revoke.call(req, Params::new()).await
} else if operation == 2 {
Submit.call(req, Params::new()).await
} else if operation == 3 {
QueryRun.call(req, Params::new()).await
} else if operation == 4 {
HistoryRun.call(req, Params::new()).await
} else if operation == 5 {
CancelRun.call(req, Params::new()).await
} else if operation == 6 {
PageRuns.call(req, Params::new()).await
} else if operation == 7 {
ReplayRun.call(req, Params::new()).await
} else {
Create.call(req, Params::new()).await
};
let error = result.expect_err("unsigned mutations must not reach persistence");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
}
}
#[test]
fn submit_dto_rejects_client_authority_fields_and_null_versions() {
assert!(serde_json::from_str::<SubmitInput>(
r#"{"version":1,"instance_id":"img","idempotency_key":"k","object":{"bucket":"b","key":"k","version_id":"v"},"actual_signer_access_key":"forged"}"#
)
.is_err());
let input: SubmitInput = serde_json::from_str(
r#"{"version":1,"instance_id":"img","idempotency_key":"k","object":{"bucket":"b","key":"a/x","version_id":"null"}}"#,
)
.unwrap();
let record = rustfs_iam::delegation::Delegation {
version: 1,
id: "a".repeat(32),
revision: "a".repeat(32),
owner_access_key: "owner".into(),
created_at_unix_seconds: 1,
restrictions: Restrictions {
version: None,
output_bucket: None,
deployment: "prod".into(),
bucket: "b".into(),
input_prefix: "a/".into(),
output_prefix: "out/".into(),
expires_at_unix_seconds: 100,
},
};
assert!(!plugin_runner::validate_delegation_scope(
&record,
"owner",
&record.id,
"prod",
ObjectReference {
bucket: &input.object.bucket,
key: &input.object.key,
version_id: &input.object.version_id,
},
2,
));
}
#[test]
fn query_run_dto_rejects_client_authority_fields() {
let run_id = uuid::Uuid::from_u128(1);
let input: QueryRunInput =
serde_json::from_str(&format!(r#"{{"version":1,"run_id":"{run_id}"}}"#)).expect("valid run query");
assert_eq!(input.run_id, run_id);
assert!(
serde_json::from_str::<QueryRunInput>(&format!(
r#"{{"version":1,"run_id":"{run_id}","actual_signer_access_key":"forged"}}"#
))
.is_err()
);
}
#[test]
fn history_run_dto_is_strict_and_rejects_nil_run() {
let run_id = uuid::Uuid::from_u128(1);
let input: HistoryRunInput =
serde_json::from_str(&format!(r#"{{"version":1,"run_id":"{run_id}"}}"#)).expect("valid history request");
assert!(valid_history_run_input(&input));
for raw in [
r#"{"version":1,"run_id":"00000000-0000-0000-0000-000000000000"}"#,
r#"{"version":2,"run_id":"00000000-0000-0000-0000-000000000001"}"#,
r#"{"version":1,"run_id":"00000000-0000-0000-0000-000000000001","actual_signer_access_key":"forged"}"#,
] {
let rejected = match serde_json::from_str::<HistoryRunInput>(raw) {
Ok(input) => !valid_history_run_input(&input),
Err(_) => true,
};
assert!(rejected, "accepted invalid history request: {raw}");
}
}
#[test]
fn page_runs_dto_is_strict_and_bounded() {
let cursor = plugin_runner::RunPageCursor {
created_at_unix_micros: 10,
run_id: uuid::Uuid::from_u128(1),
};
let input: PageRunsInput = serde_json::from_str(
r#"{"version":1,"limit":100,"cursor":{"created_at_unix_micros":10,"run_id":"00000000-0000-0000-0000-000000000001"}}"#,
)
.expect("valid page request");
assert_eq!(input.cursor, Some(cursor));
for raw in [
r#"{"version":1,"limit":0}"#,
r#"{"version":1,"limit":101}"#,
r#"{"version":1,"limit":1,"unknown":true}"#,
r#"{"version":1,"limit":1,"actual_signer_access_key":"forged"}"#,
r#"{"version":1,"limit":1,"cursor":{"created_at_unix_micros":10,"run_id":"00000000-0000-0000-0000-000000000000"}}"#,
] {
let rejected = match serde_json::from_str::<PageRunsInput>(raw) {
Ok(input) => !valid_page_runs_input(&input),
Err(_) => true,
};
assert!(rejected, "accepted invalid page request: {raw}");
}
}
#[test]
fn replay_run_dto_is_strict_and_bounded() {
let run_id = uuid::Uuid::from_u128(1);
let input: ReplayRunInput = serde_json::from_str(&format!(
r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"retry-1","deadline_unix_micros":123}}"#
))
.expect("valid replay request");
assert!(valid_replay_run_input(&input));
for raw in [
r#"{"version":1,"origin_run_id":"00000000-0000-0000-0000-000000000000","idempotency_key":"k","deadline_unix_micros":123}"#.to_string(),
format!(r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"","deadline_unix_micros":123}}"#),
format!(r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"k","deadline_unix_micros":0}}"#),
format!(
r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"k","deadline_unix_micros":123,"actual_signer_access_key":"forged"}}"#
),
format!(
r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"{}","deadline_unix_micros":123}}"#,
"x".repeat(1025)
),
] {
let rejected = match serde_json::from_str::<ReplayRunInput>(&raw) {
Ok(input) => !valid_replay_run_input(&input),
Err(_) => true,
};
assert!(rejected, "accepted invalid replay request: {raw}");
}
}
}
+3
View File
@@ -17,6 +17,7 @@ mod auth;
pub mod console;
pub mod handlers;
mod plugin_contract;
pub(crate) mod plugin_runner;
pub(crate) mod replication_metrics_wire;
// Contract inventory is validated by tests before later runtime integration.
pub(crate) mod route_policy;
@@ -64,6 +65,8 @@ pub fn make_admin_route(
fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
health::register_health_route(r)?;
sts::register_admin_auth_route(r)?;
handlers::plugin_delegation::register_plugin_delegation_route(r)?;
handlers::plugin_attempt::register_plugin_attempt_route(r)?;
account::register_account_route(r)?;
mfa::register_mfa_route(r)?;
File diff suppressed because it is too large Load Diff
+135
View File
@@ -25,6 +25,15 @@ const TLS_CERTIFICATE_PUBLISH: AdminActionRef = AdminActionRef::new("TLSCertific
const TLS_CERTIFICATE_STATUS: AdminActionRef = AdminActionRef::new("TLSCertificateStatusAdminAction");
const CONSOLE_LOG: AdminActionRef = AdminActionRef::new("ConsoleLogAdminAction");
const COMMIT_TABLE: AdminActionRef = AdminActionRef::new("CommitTableAction");
const ISSUE_PLUGIN_ATTEMPT: AdminActionRef = AdminActionRef::new("IssuePluginAttemptAction");
const SUBMIT_PLUGIN_JOB: AdminActionRef = AdminActionRef::new("SubmitPluginJobAction");
const GET_PLUGIN_RUN: AdminActionRef = AdminActionRef::new("GetPluginRunAction");
const CANCEL_PLUGIN_RUN: AdminActionRef = AdminActionRef::new("CancelPluginRunAction");
const LIST_PLUGIN_RUNS: AdminActionRef = AdminActionRef::new("ListPluginRunsAction");
const REPLAY_PLUGIN_RUN: AdminActionRef = AdminActionRef::new("ReplayPluginRunAction");
const GET_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("GetPluginDelegationAction");
const CREATE_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("CreatePluginDelegationAction");
const REVOKE_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("RevokePluginDelegationAction");
const CREATE_POLICY: AdminActionRef = AdminActionRef::new("CreatePolicyAdminAction");
const CREATE_USER: AdminActionRef = AdminActionRef::new("CreateUserAdminAction");
const CREATE_SERVICE_ACCOUNT: AdminActionRef = AdminActionRef::new("CreateServiceAccountAdminAction");
@@ -656,6 +665,66 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SERVER_INFO,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/issue-attempt",
ISSUE_PLUGIN_ATTEMPT,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/submit",
SUBMIT_PLUGIN_JOB,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/runs/query",
GET_PLUGIN_RUN,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/runs/history",
LIST_PLUGIN_RUNS,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/runs/cancel",
CANCEL_PLUGIN_RUN,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/runs/page",
LIST_PLUGIN_RUNS,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/runs/replay",
REPLAY_PLUGIN_RUN,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/plugin-delegations/{id}",
GET_PLUGIN_DELEGATION,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations",
CREATE_PLUGIN_DELEGATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/plugin-delegations/{id}/revoke",
REVOKE_PLUGIN_DELEGATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v4/plugins/catalog",
@@ -1933,6 +2002,72 @@ mod tests {
assert!(validate_admin_route_policy_specs().is_ok());
}
#[test]
fn plugin_submit_route_is_registered_as_high_risk_authenticated_action() {
let path = "/rustfs/admin/v3/plugin-delegations/{id}/submit";
assert_action(HttpMethod::Post, path, SUBMIT_PLUGIN_JOB);
let spec = ADMIN_ROUTE_POLICY_SPECS
.iter()
.find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
.expect("submit route policy");
assert_eq!(spec.risk_level(), RouteRiskLevel::High);
}
#[test]
fn plugin_run_query_route_uses_dedicated_authenticated_action() {
let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/query";
assert_action(HttpMethod::Post, path, GET_PLUGIN_RUN);
let spec = ADMIN_ROUTE_POLICY_SPECS
.iter()
.find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
.expect("plugin run query route policy");
assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
}
#[test]
fn plugin_run_history_route_uses_list_action() {
let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/history";
assert_action(HttpMethod::Post, path, LIST_PLUGIN_RUNS);
let spec = ADMIN_ROUTE_POLICY_SPECS
.iter()
.find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
.expect("plugin run history route policy");
assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
}
#[test]
fn plugin_run_cancel_route_uses_dedicated_authenticated_action() {
let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/cancel";
assert_action(HttpMethod::Post, path, CANCEL_PLUGIN_RUN);
let spec = ADMIN_ROUTE_POLICY_SPECS
.iter()
.find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
.expect("plugin run cancel route policy");
assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
}
#[test]
fn plugin_run_page_route_uses_dedicated_authenticated_action() {
let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/page";
assert_action(HttpMethod::Post, path, LIST_PLUGIN_RUNS);
let spec = ADMIN_ROUTE_POLICY_SPECS
.iter()
.find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
.expect("plugin run page route policy");
assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
}
#[test]
fn plugin_run_replay_route_uses_dedicated_authenticated_action() {
let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/replay";
assert_action(HttpMethod::Post, path, REPLAY_PLUGIN_RUN);
let spec = ADMIN_ROUTE_POLICY_SPECS
.iter()
.find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
.expect("plugin run replay route policy");
assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
}
#[test]
fn route_policy_inventory_has_no_internal_overlap() {
let mut seen = BTreeSet::new();
@@ -116,6 +116,48 @@ fn registered_admin_router() -> S3Router<AdminOperation> {
fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
vec![
admin_route(Method::POST, "/v3/plugin-delegations"),
admin_route_sample(Method::GET, "/v3/plugin-delegations/{id}", "/v3/plugin-delegations/test-delegation"),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/revoke",
"/v3/plugin-delegations/test-delegation/revoke",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/submit",
"/v3/plugin-delegations/test-delegation/submit",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/runs/query",
"/v3/plugin-delegations/test-delegation/runs/query",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/runs/history",
"/v3/plugin-delegations/test-delegation/runs/history",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/runs/cancel",
"/v3/plugin-delegations/test-delegation/runs/cancel",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/runs/page",
"/v3/plugin-delegations/test-delegation/runs/page",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/runs/replay",
"/v3/plugin-delegations/test-delegation/runs/replay",
),
admin_route_sample(
Method::POST,
"/v3/plugin-delegations/{id}/issue-attempt",
"/v3/plugin-delegations/test-delegation/issue-attempt",
),
route(Method::GET, HEALTH_PREFIX),
route(Method::HEAD, HEALTH_PREFIX),
route(Method::GET, HEALTH_READY_PATH),
+17 -2
View File
@@ -64,6 +64,13 @@ use std::sync::OnceLock;
use time::OffsetDateTime;
use url::{Url, form_urlencoded};
fn iam_authorization_error(err: IamError) -> ApiError {
match err {
IamError::IAMActionNotAllowed => ApiError::access_denied(),
other => ApiError::from(other),
}
}
const EVENT_OBJECT_TAG_AUTHORIZATION: &str = "object_tag_authorization";
const LOG_COMPONENT_ACCESS: &str = "storage_access";
const LOG_SUBSYSTEM_AUTHORIZATION: &str = "authorization";
@@ -1483,7 +1490,7 @@ async fn authorize_request_with_location_constraint<T>(
let allowed = iam_store
.try_eval_prepared(&prepared, &final_args)
.await
.map_err(ApiError::from)?;
.map_err(iam_authorization_error)?;
if !allowed
&& matches!(
action,
@@ -1505,7 +1512,7 @@ async fn authorize_request_with_location_constraint<T>(
if !iam_store
.try_eval_prepared(&prepared, &final_args)
.await
.map_err(ApiError::from)?
.map_err(iam_authorization_error)?
{
return Err(denial.deny("iam_explicit_deny", action));
}
@@ -3618,6 +3625,14 @@ mod tests {
use std::{collections::HashMap, sync::Arc};
use time::OffsetDateTime;
#[test]
fn terminal_iam_denial_is_access_denied_and_backend_failure_stays_internal() {
let denied = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IAMActionNotAllowed));
assert_eq!(denied.code(), &S3ErrorCode::AccessDenied);
let unavailable = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IamSysNotInitialized));
assert_eq!(unavailable.code(), &S3ErrorCode::InternalError);
}
struct UnreadyIam;
impl IamInterface for UnreadyIam {
+104 -24
View File
@@ -222,12 +222,12 @@ impl JournalState {
Some(entries)
}
fn from_replay_record(record: ScannerDurableDirtyUsageReplayRecord) -> Option<Self> {
fn from_replay_record(record: &ScannerDurableDirtyUsageReplayRecord) -> Option<Self> {
if record.entries.is_empty() || record.entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES {
return None;
}
let mut state = JournalState::default();
for entry in record.entries {
for entry in &record.entries {
if entry.bucket.is_empty()
|| entry.bucket.contains(['/', '\\', '\0'])
|| entry.bucket == "."
@@ -242,7 +242,7 @@ impl JournalState {
{
return None;
}
let scope = match entry.scope {
let scope = match &entry.scope {
ScannerDurableDirtyUsageReplayScope::WholeBucket => JournalScope::WholeBucket,
ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } => {
if entries.is_empty() || entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES {
@@ -254,18 +254,18 @@ impl JournalState {
{
return None;
}
JournalScope::TopLevelEntries(entries)
JournalScope::TopLevelEntries(entries.clone())
}
};
state.scope_bytes += journal_scope_bytes(&scope);
if state
.buckets
.insert(
entry.bucket,
entry.bucket.clone(),
JournalBucketState {
generation: entry.generation,
scope,
producers: entry.producers,
producers: entry.producers.clone(),
},
)
.is_some()
@@ -338,7 +338,7 @@ async fn replay_durable_dirty_usage_journal(store: Arc<ECStore>, owner: &str) ->
..Default::default()
};
};
match rustfs_scanner::replay_durable_dirty_usage_producer_record(&replay) {
match replay.replay() {
Ok(replayed) => {
debug!(
event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL,
@@ -368,7 +368,7 @@ async fn replay_durable_dirty_usage_journal(store: Arc<ECStore>, owner: &str) ->
}
}
fn decode_owned_replay_record(owner: &str, bytes: &[u8]) -> Option<(JournalState, Vec<u8>)> {
fn decode_owned_replay_record(owner: &str, bytes: &[u8]) -> Option<(JournalState, ScannerDurableDirtyUsageReplayRecord)> {
if bytes.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
return None;
}
@@ -377,13 +377,16 @@ fn decode_owned_replay_record(owner: &str, bytes: &[u8]) -> Option<(JournalState
return None;
}
let replay = record.replay?;
let bytes = serde_json::to_vec(&replay).ok()?;
let state = JournalState::from_replay_record(replay)?;
Some((state, bytes))
let state = JournalState::from_replay_record(&replay)?;
Some((state, replay))
}
fn encode_owned_replay_record(owner: &str, state: &JournalState) -> Result<Vec<u8>, serde_json::Error> {
let mut entries = state.replay_entries();
let mut prefix = Vec::with_capacity(owner.len() + 32);
prefix.extend_from_slice(b"{\"owner\":");
serde_json::to_writer(&mut prefix, owner)?;
prefix.extend_from_slice(b",\"replay\":");
for compact in [false, true] {
if let Some(entries) = &mut entries {
if compact {
@@ -391,13 +394,12 @@ fn encode_owned_replay_record(owner: &str, state: &JournalState) -> Result<Vec<u
entry.scope = ScannerDurableDirtyUsageReplayScope::WholeBucket;
}
}
if let Ok(replay) = rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries.clone())
&& let Ok(replay) = serde_json::from_slice(&replay)
{
let bytes = serde_json::to_vec(&OwnedReplayRecord {
owner: owner.to_string(),
replay: Some(replay),
})?;
if let Ok(replay) = rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries.clone()) {
// This producer already validated and serialized the complete frame.
let mut bytes = prefix.clone();
bytes.reserve(replay.len() + 1);
bytes.extend_from_slice(&replay);
bytes.push(b'}');
if bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
return Ok(bytes);
}
@@ -405,10 +407,8 @@ fn encode_owned_replay_record(owner: &str, state: &JournalState) -> Result<Vec<u
}
}
// Never label a truncated record as complete producer coverage.
serde_json::to_vec(&OwnedReplayRecord {
owner: owner.to_string(),
replay: None,
})
prefix.extend_from_slice(b"null}");
Ok(prefix)
}
async fn run_durable_dirty_usage_journal(store: Arc<ECStore>, owner: String, shared: Weak<SharedJournal>, changed: Arc<Notify>) {
@@ -575,6 +575,84 @@ mod tests {
ENV.get_or_init(|| rustfs_test_utils::TestECStoreEnv::builder().build()).await
}
fn legacy_owned_encoding(owner: &str, state: &JournalState) -> Vec<u8> {
let mut entries = state.replay_entries();
for compact in [false, true] {
if let Some(entries) = &mut entries {
if compact {
for entry in entries.iter_mut() {
entry.scope = ScannerDurableDirtyUsageReplayScope::WholeBucket;
}
}
if let Ok(replay) = rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries.clone())
&& let Ok(replay) = serde_json::from_slice(&replay)
{
let bytes = serde_json::to_vec(&OwnedReplayRecord {
owner: owner.to_string(),
replay: Some(replay),
})
.expect("legacy typed wrapper");
if bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
return bytes;
}
}
}
}
serde_json::to_vec(&OwnedReplayRecord {
owner: owner.to_string(),
replay: None,
})
.expect("legacy unknown marker")
}
#[test]
fn owned_encoding_reuses_the_producer_frame_without_changing_wire_bytes() {
let mut scoped = JournalState::default();
for producer in SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION {
scoped.record_mutation("bucket-a".to_string(), "quote\"雪/object", 17, producer);
}
scoped.record_mutation("bucket-b".to_string(), "object", 9, SegmentInvalidationProducerIdentity::PutObject);
let unknown = JournalState {
invalidated: true,
..Default::default()
};
for owner in ["owner", "quote\"slash\\newline\n", "所有者", "\0"] {
for state in [&scoped, &unknown, &JournalState::default()] {
let expected = legacy_owned_encoding(owner, state);
let actual = encode_owned_replay_record(owner, state).expect("complete owned encoding");
assert_eq!(actual, expected, "legacy bytes for owner {owner:?}");
}
}
}
#[test]
fn owned_unknown_marker_keeps_the_pinned_legacy_wire_fixture() {
let bytes = encode_owned_replay_record("owner", &JournalState::default()).expect("unknown marker");
assert_eq!(bytes, br#"{"owner":"owner","replay":null}"#);
assert!(decode_owned_replay_record("owner", &bytes).is_none());
}
#[test]
fn owned_replay_keeps_the_outer_json_recursion_boundary() {
// Ignored scope fields still consume the complete owned frame's depth
// budget. Parsing replay from a fresh JSON root must not bypass it.
for (depth, accepted) in [(122, true), (123, false)] {
let bytes = format!(
r#"{{"owner":"owner","replay":{{"schema":1,"cache_key_format":1,"writer_epoch":"epoch","entries":[{{"bucket":"bucket","generation":7,"scope":{{"type":"whole_bucket","ignored":{}0{}}},"producers":["put_object"]}}]}}}}"#,
"[".repeat(depth),
"]".repeat(depth),
);
assert!(bytes.len() < DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
let decoded = decode_owned_replay_record("owner", bytes.as_bytes());
assert_eq!(decoded.is_some(), accepted, "owned frame depth {depth}");
if let Some((state, _)) = decoded {
assert_eq!(state.buckets.len(), 1);
assert_eq!(state.buckets["bucket"].generation, 7);
assert_eq!(state.buckets["bucket"].scope, super::JournalScope::WholeBucket);
}
}
}
#[test]
fn journal_owner_is_stable_and_never_defaults_to_a_shared_identity() {
assert!(durable_dirty_usage_owner("").is_none());
@@ -654,6 +732,7 @@ mod tests {
}
assert!(state.scope_bytes <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
let bytes = encode_owned_replay_record(&owner, &state).expect("bounded record");
assert_eq!(bytes, legacy_owned_encoding(&owner, &state), "legacy compacted bytes");
assert!(bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
let (decoded, _) = decode_owned_replay_record(&owner, &bytes).expect("compacted replay");
assert_eq!(decoded.buckets.len(), 3);
@@ -673,6 +752,7 @@ mod tests {
assert!(!state.invalidated);
assert_eq!(state.buckets.len(), 1024);
let bytes = encode_owned_replay_record(&owner, &state).expect("unverified marker");
assert_eq!(bytes, legacy_owned_encoding(&owner, &state), "legacy unknown marker bytes");
assert!(bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
let record: OwnedReplayRecord = serde_json::from_slice(&bytes).expect("marker");
assert!(record.replay.is_none());
@@ -984,7 +1064,7 @@ mod tests {
producers: BTreeSet::from([SegmentInvalidationProducerIdentity::Replication]),
}],
};
let mut state = JournalState::from_replay_record(record).expect("valid replay record should hydrate");
let mut state = JournalState::from_replay_record(&record).expect("valid replay record should hydrate");
state.record_mutation(
"videos".to_string(),
"clips/object",
@@ -1011,7 +1091,7 @@ mod tests {
producers: BTreeSet::from([SegmentInvalidationProducerIdentity::Replication]),
}],
};
let mut state = JournalState::from_replay_record(record).expect("valid replay record should hydrate");
let mut state = JournalState::from_replay_record(&record).expect("valid replay record should hydrate");
state.record_mutation(
"photos".to_string(),
"2026/object",