fix(iam): invalidate peer STS caches on revocation (#5718)

crates/iam/src/cache.rs

@@ -16,7 +16,7 @@ use std::{
     collections::{HashMap, HashSet},
     ops::{Deref, DerefMut},
     ptr,
-    sync::{Arc, Mutex},
+    sync::{Arc, Mutex, Weak},
 };
 
 use arc_swap::{ArcSwap, Guard};
@@ -65,6 +65,29 @@ pub struct Cache {
     state: ArcSwap<CacheState>,
     write_lock: Mutex<()>,
     service_account_mutation_lock: AsyncMutex<()>,
+    sts_account_mutation_locks: Arc<StsMutationLockRegistry>,
+}
+
+struct StsMutationLockRegistry {
+    locks: Mutex<HashMap<String, Weak<AsyncMutex<StsMutationLockState>>>>,
+}
+
+pub(crate) struct StsMutationLockState {
+    access_key: String,
+    registry: Weak<StsMutationLockRegistry>,
+    lock: Weak<AsyncMutex<StsMutationLockState>>,
+}
+
+impl Drop for StsMutationLockState {
+    fn drop(&mut self) {
+        let Some(registry) = self.registry.upgrade() else {
+            return;
+        };
+        let mut locks = registry.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        if locks.get(&self.access_key).is_some_and(|current| current.ptr_eq(&self.lock)) {
+            locks.remove(&self.access_key);
+        }
+    }
 }
 
 impl Default for Cache {
@@ -73,6 +96,9 @@ impl Default for Cache {
             state: ArcSwap::new(Arc::new(CacheState::default())),
             write_lock: Mutex::new(()),
             service_account_mutation_lock: AsyncMutex::new(()),
+            sts_account_mutation_locks: Arc::new(StsMutationLockRegistry {
+                locks: Mutex::new(HashMap::new()),
+            }),
         }
     }
 }
@@ -84,6 +110,29 @@ impl Cache {
         &self.service_account_mutation_lock
     }
 
+    pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<StsMutationLockState>> {
+        let mut locks = self
+            .sts_account_mutation_locks
+            .locks
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
+            return lock;
+        }
+
+        let registry = Arc::downgrade(&self.sts_account_mutation_locks);
+        let access_key_owned = access_key.to_string();
+        let lock = Arc::new_cyclic(|lock| {
+            AsyncMutex::new(StsMutationLockState {
+                access_key: access_key_owned,
+                registry,
+                lock: lock.clone(),
+            })
+        });
+        locks.insert(access_key.to_string(), Arc::downgrade(&lock));
+        lock
+    }
+
     pub(crate) fn snapshot(&self) -> CacheSnapshot {
         self.state.load()
     }
@@ -445,6 +494,30 @@ mod tests {
     use crate::cache::Cache;
     use crate::store::MappedPolicy;
 
+    #[test]
+    fn sts_mutation_locks_are_keyed_and_prune_unused_entries() {
+        let cache = Cache::default();
+        let first = cache.sts_account_mutation_lock("first");
+        let same = cache.sts_account_mutation_lock("first");
+        let different = cache.sts_account_mutation_lock("different");
+
+        assert!(Arc::ptr_eq(&first, &same));
+        assert!(!Arc::ptr_eq(&first, &different));
+        drop(first);
+        drop(same);
+        drop(different);
+
+        let _next = cache.sts_account_mutation_lock("next");
+        let locks = cache
+            .sts_account_mutation_locks
+            .locks
+            .lock()
+            .expect("STS mutation lock registry mutex poisoned");
+        assert!(!locks.contains_key("first"));
+        assert!(!locks.contains_key("different"));
+        assert!(locks.contains_key("next"));
+    }
+
     #[tokio::test]
     async fn test_cache_entity_add() {
         let owner = Arc::new(Cache::default());

crates/iam/src/lib.rs

@@ -102,7 +102,49 @@ pub(crate) async fn notify_iam_delete_user(access_key: &str) -> Vec<IamNotificat
     }
 }
 
+#[cfg(test)]
+pub(crate) struct LoadUserNotificationProbe {
+    pub(crate) observed: std::sync::Mutex<Option<(String, bool)>>,
+    pub(crate) remaining_failures: std::sync::atomic::AtomicUsize,
+    pub(crate) attempts: std::sync::atomic::AtomicUsize,
+    pub(crate) panic: bool,
+    pub(crate) started: tokio::sync::Notify,
+    pub(crate) release: Option<tokio::sync::Notify>,
+    pub(crate) completed: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+tokio::task_local! {
+    pub(crate) static LOAD_USER_NOTIFICATION_PROBE: std::sync::Arc<LoadUserNotificationProbe>;
+}
+
 pub(crate) async fn notify_iam_load_user(access_key: &str, temp: bool) -> Vec<IamNotificationPeerErr> {
+    #[cfg(test)]
+    if let Ok(probe) = LOAD_USER_NOTIFICATION_PROBE.try_with(std::sync::Arc::clone) {
+        probe.attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        *probe.observed.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some((access_key.to_string(), temp));
+        probe.started.notify_one();
+        if let Some(release) = &probe.release {
+            release.notified().await;
+        }
+        assert!(!probe.panic, "notification probe panic");
+        let should_fail = probe
+            .remaining_failures
+            .fetch_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |remaining| {
+                remaining.checked_sub(1)
+            })
+            .is_ok();
+        let result = if should_fail {
+            vec![IamNotificationPeerErr {
+                err: Some(IamEcstoreError::other("peer notification failed")),
+            }]
+        } else {
+            Vec::new()
+        };
+        probe.completed.notify_one();
+        return result;
+    }
+
     match runtime_sources::notification_sys() {
         Some(notification_sys) => notification_sys
             .load_user(access_key, temp)

crates/iam/src/manager.rs

@@ -293,6 +293,8 @@ where
     }
 
     pub async fn load_user(&self, access_key: &str) -> Result<()> {
+        let sts_mutation_lock = self.cache.sts_account_mutation_lock(access_key);
+        let _sts_mutation_guard = sts_mutation_lock.lock().await;
         let mut users_map: HashMap<String, UserIdentity> = HashMap::new();
         let mut user_policy_map = HashMap::new();
         let mut sts_users_map = HashMap::new();
@@ -1207,6 +1209,9 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let mutation_lock = self.cache.sts_account_mutation_lock(access_key);
+        let _mutation_guard = mutation_lock.lock().await;
+
         let sts_policy_update = if let Some(policy) = policy_name {
             let mp = MappedPolicy::new(policy);
             let (_, combined_policy_stmt) = filter_policies(&self.cache, &mp.policies, "temp");
@@ -1433,6 +1438,11 @@ where
             return Err(Error::InvalidArgument);
         }
 
+        let sts_mutation_lock = (utype == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(access_key));
+        let _sts_mutation_guard = match &sts_mutation_lock {
+            Some(lock) => Some(lock.lock().await),
+            None => None,
+        };
         let _service_account_guard = if utype == UserType::Svc {
             Some(self.cache.service_account_mutation_lock().lock().await)
         } else {
@@ -1493,9 +1503,13 @@ where
             });
         }
 
-        let _ = self.api.delete_mapped_policy(access_key, utype, false).await;
+        if utype != UserType::Sts {
+            let _ = self.api.delete_mapped_policy(access_key, utype, false).await;
+        }
 
-        self.cache.delete_user_policy(access_key, OffsetDateTime::now_utc());
+        if utype != UserType::Sts {
+            self.cache.delete_user_policy(access_key, OffsetDateTime::now_utc());
+        }
 
         if let Err(err) = self.api.delete_user_identity(access_key, utype).await
             && !is_err_no_such_user(&err)
@@ -1507,8 +1521,17 @@ where
         self.cache.with_write_lock(|cache| {
             if utype == UserType::Sts {
                 cache.delete_sts_account(access_key, deleted_at);
+                if cache
+                    .state()
+                    .users
+                    .get(access_key)
+                    .is_some_and(|identity| identity.credentials.is_temp())
+                {
+                    cache.delete_user(access_key, deleted_at);
+                }
+            } else {
+                cache.delete_user(access_key, deleted_at);
             }
-            cache.delete_user(access_key, deleted_at);
         });
 
         Ok(deleted_at)
@@ -2032,6 +2055,11 @@ where
         Ok(())
     }
     pub async fn user_notification_handler(&self, name: &str, user_type: UserType) -> Result<()> {
+        let sts_mutation_lock = (user_type == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(name));
+        let _sts_mutation_guard = match &sts_mutation_lock {
+            Some(lock) => Some(lock.lock().await),
+            None => None,
+        };
         let _service_account_guard = if user_type == UserType::Svc {
             Some(self.cache.service_account_mutation_lock().lock().await)
         } else {
@@ -2077,7 +2105,9 @@ where
                     UserType::Reg | UserType::Svc => cache.delete_user(name, now),
                     UserType::None => {}
                 }
-                self.remove_user_from_cached_groups(cache, name, now);
+                if user_type != UserType::Sts {
+                    self.remove_user_from_cached_groups(cache, name, now);
+                }
                 if user_type == UserType::Reg {
                     for access_key in service_accounts_to_delete.iter() {
                         cache.delete_user(access_key, now);
@@ -2087,7 +2117,9 @@ where
                         cache.delete_user(access_key, now);
                     }
                 }
-                cache.delete_user_policy(name, now);
+                if user_type != UserType::Sts {
+                    cache.delete_user_policy(name, now);
+                }
             });
 
             return Ok(());
@@ -2446,12 +2478,12 @@ mod tests {
         saved_user: Arc<Mutex<Option<UserIdentity>>>,
         load_attempts: Arc<AtomicUsize>,
         visible_after_attempt: usize,
-        block_service_save: Arc<AtomicBool>,
-        service_save_started: Arc<Notify>,
-        release_service_save: Arc<Notify>,
-        block_service_load: Arc<AtomicBool>,
-        service_load_started: Arc<Notify>,
-        release_service_load: Arc<Notify>,
+        block_account_save: Arc<AtomicBool>,
+        account_save_started: Arc<Notify>,
+        release_account_save: Arc<Notify>,
+        block_account_load: Arc<AtomicBool>,
+        account_load_started: Arc<Notify>,
+        release_account_load: Arc<Notify>,
     }
 
     impl DelayedTempUserVisibilityStore {
@@ -2460,12 +2492,12 @@ mod tests {
                 saved_user: Arc::new(Mutex::new(None)),
                 load_attempts: Arc::new(AtomicUsize::new(0)),
                 visible_after_attempt,
-                block_service_save: Arc::new(AtomicBool::new(false)),
-                service_save_started: Arc::new(Notify::new()),
-                release_service_save: Arc::new(Notify::new()),
-                block_service_load: Arc::new(AtomicBool::new(false)),
-                service_load_started: Arc::new(Notify::new()),
-                release_service_load: Arc::new(Notify::new()),
+                block_account_save: Arc::new(AtomicBool::new(false)),
+                account_save_started: Arc::new(Notify::new()),
+                release_account_save: Arc::new(Notify::new()),
+                block_account_load: Arc::new(AtomicBool::new(false)),
+                account_load_started: Arc::new(Notify::new()),
+                release_account_load: Arc::new(Notify::new()),
             }
         }
     }
@@ -2495,9 +2527,9 @@ mod tests {
             item: UserIdentity,
             _ttl: Option<usize>,
         ) -> Result<()> {
-            if user_type == UserType::Svc && self.block_service_save.load(Ordering::SeqCst) {
-                self.service_save_started.notify_one();
-                self.release_service_save.notified().await;
+            if matches!(user_type, UserType::Svc | UserType::Sts) && self.block_account_save.load(Ordering::SeqCst) {
+                self.account_save_started.notify_one();
+                self.release_account_save.notified().await;
             }
             *self.saved_user.lock().expect("saved_user mutex poisoned") = Some(item);
             Ok(())
@@ -2528,9 +2560,18 @@ mod tests {
                 .expect("saved_user mutex poisoned")
                 .clone()
                 .ok_or_else(|| Error::NoSuchUser(name.to_string()))?;
-            if user_type == UserType::Svc && self.block_service_load.load(Ordering::SeqCst) {
-                self.service_load_started.notify_one();
-                self.release_service_load.notified().await;
+            let matches_user_type = match user_type {
+                UserType::Sts => loaded.credentials.is_temp(),
+                UserType::Svc => loaded.credentials.is_service_account(),
+                UserType::Reg => !loaded.credentials.is_temp() && !loaded.credentials.is_service_account(),
+                UserType::None => false,
+            };
+            if !matches_user_type {
+                return Err(Error::NoSuchUser(name.to_string()));
+            }
+            if self.block_account_load.load(Ordering::SeqCst) {
+                self.account_load_started.notify_one();
+                self.release_account_load.notified().await;
             }
             m.insert(name.to_string(), loaded);
             Ok(())
@@ -2746,12 +2787,12 @@ mod tests {
         };
         cache.add_service_account(credentials).await.expect("seed service account");
 
-        store.block_service_load.store(true, Ordering::SeqCst);
+        store.block_account_load.store(true, Ordering::SeqCst);
         let notification = {
             let cache = Arc::clone(&cache);
             tokio::spawn(async move { cache.user_notification_handler(access_key, UserType::Svc).await })
         };
-        store.service_load_started.notified().await;
+        store.account_load_started.notified().await;
 
         let update = {
             let cache = Arc::clone(&cache);
@@ -2776,7 +2817,7 @@ mod tests {
         tokio::task::yield_now().await;
         assert!(!update.is_finished(), "update must wait for the in-flight cache refresh");
 
-        store.release_service_load.notify_one();
+        store.release_account_load.notify_one();
         notification.await.expect("notification task").expect("notification refresh");
         update.await.expect("update task").expect("service account update");
 
@@ -2793,7 +2834,7 @@ mod tests {
     #[tokio::test]
     async fn concurrent_service_account_create_cannot_overwrite_first_writer() {
         let store = DelayedTempUserVisibilityStore::new(0);
-        store.block_service_save.store(true, Ordering::SeqCst);
+        store.block_account_save.store(true, Ordering::SeqCst);
         let cache = Arc::new(build_test_iam_cache(store.clone()));
         let access_key = "SERIALIZEDSERVICE00";
         let credentials = |secret_key: &str| Credentials {
@@ -2808,7 +2849,7 @@ mod tests {
             let cache = Arc::clone(&cache);
             tokio::spawn(async move { cache.add_service_account(credentials("firstServiceSecret123")).await })
         };
-        store.service_save_started.notified().await;
+        store.account_save_started.notified().await;
 
         let second = {
             let cache = Arc::clone(&cache);
@@ -2817,8 +2858,8 @@ mod tests {
         tokio::task::yield_now().await;
         assert!(!second.is_finished(), "second create must wait for the first writer");
 
-        store.block_service_save.store(false, Ordering::SeqCst);
-        store.release_service_save.notify_waiters();
+        store.block_account_save.store(false, Ordering::SeqCst);
+        store.release_account_save.notify_waiters();
         first.await.expect("first create task").expect("first create");
         let err = second
             .await
@@ -2862,12 +2903,12 @@ mod tests {
         };
         cache.add_service_account(credentials).await.expect("seed service account");
 
-        store.block_service_load.store(true, Ordering::SeqCst);
+        store.block_account_load.store(true, Ordering::SeqCst);
         let notification = {
             let cache = Arc::clone(&cache);
             tokio::spawn(async move { cache.user_notification_handler(access_key, UserType::Svc).await })
         };
-        store.service_load_started.notified().await;
+        store.account_load_started.notified().await;
 
         let delete = {
             let cache = Arc::clone(&cache);
@@ -2876,7 +2917,7 @@ mod tests {
         tokio::task::yield_now().await;
         assert!(!delete.is_finished(), "delete must wait for the in-flight cache refresh");
 
-        store.release_service_load.notify_one();
+        store.release_account_load.notify_one();
         notification.await.expect("notification task").expect("notification refresh");
         delete.await.expect("delete task").expect("service account delete");
 
@@ -2884,6 +2925,130 @@ mod tests {
         assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
     }
 
+    #[tokio::test]
+    async fn sts_notification_cannot_restore_concurrent_delete() {
+        let store = DelayedTempUserVisibilityStore::new(0);
+        let cache = Arc::new(build_test_iam_cache(store.clone()));
+        let credentials = build_test_temp_credentials();
+        let access_key = credentials.access_key.clone();
+        cache
+            .set_temp_user(&access_key, &credentials, None)
+            .await
+            .expect("seed temporary account");
+
+        store.block_account_load.store(true, Ordering::SeqCst);
+        let notification = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            tokio::spawn(async move { cache.user_notification_handler(&access_key, UserType::Sts).await })
+        };
+        store.account_load_started.notified().await;
+
+        let delete_started = Arc::new(Notify::new());
+        let delete = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            let delete_started = Arc::clone(&delete_started);
+            tokio::spawn(async move {
+                delete_started.notify_one();
+                cache.delete_user(&access_key, UserType::Sts).await
+            })
+        };
+        delete_started.notified().await;
+        tokio::task::yield_now().await;
+        let delete_waited_for_notification = !delete.is_finished();
+
+        store.release_account_load.notify_one();
+        notification.await.expect("notification task").expect("notification refresh");
+        delete.await.expect("delete task").expect("temporary account delete");
+
+        assert!(delete_waited_for_notification, "delete must wait for the in-flight STS cache refresh");
+        assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
+        assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
+    }
+
+    #[tokio::test]
+    async fn sts_auth_reload_cannot_restore_concurrent_delete() {
+        let store = DelayedTempUserVisibilityStore::new(0);
+        let cache = Arc::new(build_test_iam_cache(store.clone()));
+        let credentials = build_test_temp_credentials();
+        let access_key = credentials.access_key.clone();
+        cache
+            .set_temp_user(&access_key, &credentials, None)
+            .await
+            .expect("seed temporary account");
+        cache.cache.delete_sts_account(&access_key, OffsetDateTime::now_utc());
+
+        store.block_account_load.store(true, Ordering::SeqCst);
+        let reload = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            tokio::spawn(async move { cache.load_user(&access_key).await })
+        };
+        store.account_load_started.notified().await;
+
+        let delete_started = Arc::new(Notify::new());
+        let delete = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            let delete_started = Arc::clone(&delete_started);
+            tokio::spawn(async move {
+                delete_started.notify_one();
+                cache.delete_user(&access_key, UserType::Sts).await
+            })
+        };
+        delete_started.notified().await;
+        tokio::task::yield_now().await;
+        let delete_waited_for_reload = !delete.is_finished();
+
+        store.release_account_load.notify_one();
+        reload.await.expect("reload task").expect("authentication cache reload");
+        delete.await.expect("delete task").expect("temporary account delete");
+
+        assert!(delete_waited_for_reload, "delete must wait for the in-flight authentication reload");
+        assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
+        assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
+    }
+
+    #[tokio::test]
+    async fn sts_create_cannot_restore_concurrent_delete() {
+        let store = DelayedTempUserVisibilityStore::new(0);
+        store.block_account_save.store(true, Ordering::SeqCst);
+        let cache = Arc::new(build_test_iam_cache(store.clone()));
+        let credentials = build_test_temp_credentials();
+        let access_key = credentials.access_key.clone();
+
+        let create = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            tokio::spawn(async move { cache.set_temp_user(&access_key, &credentials, None).await })
+        };
+        store.account_save_started.notified().await;
+
+        let delete_started = Arc::new(Notify::new());
+        let delete = {
+            let cache = Arc::clone(&cache);
+            let access_key = access_key.clone();
+            let delete_started = Arc::clone(&delete_started);
+            tokio::spawn(async move {
+                delete_started.notify_one();
+                cache.delete_user(&access_key, UserType::Sts).await
+            })
+        };
+        delete_started.notified().await;
+        tokio::task::yield_now().await;
+        let delete_waited_for_create = !delete.is_finished();
+
+        store.block_account_save.store(false, Ordering::SeqCst);
+        store.release_account_save.notify_one();
+        create.await.expect("create task").expect("temporary account create");
+        delete.await.expect("delete task").expect("temporary account delete");
+
+        assert!(delete_waited_for_create, "delete must wait for the in-flight STS create");
+        assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
+        assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
+    }
+
     #[tokio::test]
     async fn test_init_keeps_error_state_when_initial_load_fails() {
         let (sender, receiver) = mpsc::channel::<i64>(1);

crates/iam/src/store/object.rs

@@ -28,7 +28,10 @@ use crate::{
 use futures::future::join_all;
 use rustfs_io_metrics::record_system_path_failure;
 use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
-use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
+use rustfs_utils::{
+    MaskedAccessKey,
+    path::{SLASH_SEPARATOR, path_join_buf},
+};
 use serde::{Serialize, de::DeserializeOwned};
 use std::sync::{LazyLock, Mutex};
 use std::time::{Duration, Instant};
@@ -600,10 +603,10 @@ impl ObjectStore {
             .await
             .map_err(|err| {
                 if is_err_config_not_found(&err) {
-                    warn!(name, user_type = ?user_type, "IAM user identity missing");
+                    debug!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity missing");
                     Error::NoSuchUser(name.to_owned())
                 } else {
-                    warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
+                    warn!(name = %MaskedAccessKey(name), user_type = ?user_type, error = ?err, "IAM user identity load failed");
                     err
                 }
             })?;
@@ -611,7 +614,7 @@ impl ObjectStore {
         if u.credentials.is_expired() {
             let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
             let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
-            warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
+            warn!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity expired and was removed");
             return Err(Error::NoSuchUser(name.to_owned()));
         }
 
@@ -635,7 +638,7 @@ impl ObjectStore {
                         let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
                         let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
                     }
-                    warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
+                    warn!(name = %MaskedAccessKey(name), user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
                     return Err(Error::NoSuchUser(name.to_owned()));
                 }
             }
@@ -873,13 +876,7 @@ impl Store for ObjectStore {
     async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
         self.delete_iam_config(get_user_identity_path(name, user_type))
             .await
-            .map_err(|err| {
-                if is_err_config_not_found(&err) {
-                    Error::NoSuchPolicy
-                } else {
-                    err
-                }
-            })?;
+            .map_err(|err| map_delete_user_identity_error(name, err))?;
         Ok(())
     }
     async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity> {
@@ -1327,9 +1324,18 @@ impl Store for ObjectStore {
     }
 }
 
+fn map_delete_user_identity_error(name: &str, err: Error) -> Error {
+    if is_err_config_not_found(&err) {
+        Error::NoSuchUser(name.to_owned())
+    } else {
+        err
+    }
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{DecryptSource, LoadMode, ObjectStore};
+    use super::{DecryptSource, LoadMode, ObjectStore, map_delete_user_identity_error};
+    use crate::error::Error;
     use crate::keyring;
     use rustfs_credentials::{Credentials, init_global_action_credentials};
     use serial_test::serial;
@@ -1352,6 +1358,12 @@ mod tests {
         assert!(!LoadMode::Locked.read_opts().no_lock);
     }
 
+    #[test]
+    fn missing_user_identity_delete_maps_to_no_such_user() {
+        let err = map_delete_user_identity_error("missing-sts", Error::ConfigNotFound);
+        assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
+    }
+
     fn test_cred() -> Credentials {
         if let Some(cred) = crate::root_credentials::credentials() {
             return cred;

crates/iam/src/sys.rs

@@ -48,6 +48,16 @@ use time::OffsetDateTime;
 use tokio::sync::RwLock;
 use tracing::{error, info, warn};
 
+#[cfg(not(test))]
+const STS_INVALIDATION_RETRY_INITIAL_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
+#[cfg(test)]
+const STS_INVALIDATION_RETRY_INITIAL_DELAY: std::time::Duration = std::time::Duration::from_millis(1);
+const STS_INVALIDATION_MAX_ATTEMPTS: usize = 3;
+#[cfg(not(test))]
+const STS_INVALIDATION_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
+#[cfg(test)]
+const STS_INVALIDATION_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(20);
+
 pub const MAX_SVCSESSION_POLICY_SIZE: usize = 4096;
 pub const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0";
 
@@ -393,17 +403,74 @@ impl<T: Store> IamSys<T> {
     /// associated session token. This is the primitive used by the admin
     /// `revoke-tokens` endpoint to revoke STS credentials for a parent user.
     pub async fn delete_temp_account(&self, access_key: &str, notify: bool) -> Result<()> {
-        self.store.delete_user(access_key, UserType::Sts).await?;
-
-        if notify && !self.has_watcher() {
-            for r in notify_iam_delete_user(access_key).await {
-                if let Some(err) = r.err {
-                    warn!("notify delete_temp_account failed: {}", err);
-                }
-            }
+        if !notify || self.has_watcher() {
+            return self.store.delete_user(access_key, UserType::Sts).await;
         }
 
-        Ok(())
+        let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
+        #[cfg(test)]
+        let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
+        #[cfg(test)]
+        let notification_available = notification_probe.is_some() || crate::runtime_sources::notification_sys().is_some();
+        #[cfg(not(test))]
+        let notification_available = crate::runtime_sources::notification_sys().is_some();
+        if !notification_available {
+            return Err(Error::other("IAM peer notification system is unavailable"));
+        }
+
+        let store = Arc::clone(&self.store);
+        let access_key = access_key.to_string();
+
+        let operation = async move {
+            store.delete_user(&access_key, UserType::Sts).await?;
+
+            let mut delay = STS_INVALIDATION_RETRY_INITIAL_DELAY;
+            for attempt in 1..=STS_INVALIDATION_MAX_ATTEMPTS {
+                let attempt_error =
+                    match tokio::time::timeout(STS_INVALIDATION_ATTEMPT_TIMEOUT, notify_iam_load_user(&access_key, true)).await {
+                        Ok(results) => results.into_iter().find_map(|result| result.err).map(Error::other),
+                        Err(_) => Some(Error::other("peer STS invalidation timed out")),
+                    };
+                let Some(err) = attempt_error else {
+                    return Ok(());
+                };
+                if attempt == STS_INVALIDATION_MAX_ATTEMPTS {
+                    return Err(Error::other(err));
+                }
+                tokio::time::sleep(delay).await;
+                delay = delay.saturating_mul(2);
+            }
+            unreachable!("STS invalidation retry loop always returns")
+        };
+
+        #[cfg(test)]
+        let task = runtime.spawn(async move {
+            if let Some(probe) = notification_probe {
+                return crate::LOAD_USER_NOTIFICATION_PROBE.scope(probe, operation).await;
+            }
+            operation.await
+        });
+        #[cfg(not(test))]
+        let task = runtime.spawn(operation);
+
+        task.await.map_err(Error::other)?
+    }
+
+    #[cfg(test)]
+    fn load_user_notification_probe(
+        failures_before_success: usize,
+        block: bool,
+        panic: bool,
+    ) -> Arc<crate::LoadUserNotificationProbe> {
+        Arc::new(crate::LoadUserNotificationProbe {
+            observed: std::sync::Mutex::new(None),
+            remaining_failures: std::sync::atomic::AtomicUsize::new(failures_before_success),
+            attempts: std::sync::atomic::AtomicUsize::new(0),
+            panic,
+            started: tokio::sync::Notify::new(),
+            release: block.then(tokio::sync::Notify::new),
+            completed: tokio::sync::Notify::new(),
+        })
     }
 
     async fn notify_for_user(&self, name: &str, is_temp: bool) {
@@ -1855,6 +1922,11 @@ mod tests {
         empty_policies: bool,
         saved_sts_users: Arc<Mutex<HashMap<String, UserIdentity>>>,
         saved_service_account_count: Arc<Mutex<usize>>,
+        fail_delete: Arc<std::sync::atomic::AtomicBool>,
+        deleted_mapped_policies: Arc<Mutex<Vec<(String, UserType)>>>,
+        block_delete: Arc<std::sync::atomic::AtomicBool>,
+        delete_started: Arc<tokio::sync::Notify>,
+        release_delete: Arc<tokio::sync::Notify>,
     }
 
     impl StsTestMockStore {
@@ -1863,6 +1935,11 @@ mod tests {
                 empty_policies,
                 saved_sts_users: Arc::new(Mutex::new(HashMap::new())),
                 saved_service_account_count: Arc::new(Mutex::new(0)),
+                fail_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                deleted_mapped_policies: Arc::new(Mutex::new(Vec::new())),
+                block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                delete_started: Arc::new(tokio::sync::Notify::new()),
+                release_delete: Arc::new(tokio::sync::Notify::new()),
             }
         }
 
@@ -1913,6 +1990,13 @@ mod tests {
         }
 
         async fn delete_user_identity(&self, name: &str, _user_type: UserType) -> Result<()> {
+            if self.block_delete.load(std::sync::atomic::Ordering::SeqCst) {
+                self.delete_started.notify_one();
+                self.release_delete.notified().await;
+            }
+            if self.fail_delete.load(std::sync::atomic::Ordering::SeqCst) {
+                return Err(Error::Io(std::io::Error::other("delete temporary account failed")));
+            }
             self.saved_sts_users
                 .lock()
                 .expect("saved_sts_users mutex poisoned")
@@ -1930,7 +2014,7 @@ mod tests {
         }
 
         async fn load_user(&self, name: &str, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()> {
-            if name == "deleted-notify-user" {
+            if matches!(name, "deleted-notify-user" | "deleted-notify-sts") {
                 return Err(Error::NoSuchUser(name.to_string()));
             }
 
@@ -2008,7 +2092,11 @@ mod tests {
             Err(Error::InvalidArgument)
         }
 
-        async fn delete_mapped_policy(&self, _name: &str, _user_type: UserType, _is_group: bool) -> Result<()> {
+        async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
+            self.deleted_mapped_policies
+                .lock()
+                .expect("deleted_mapped_policies mutex poisoned")
+                .push((name.to_string(), user_type));
             Err(Error::InvalidArgument)
         }
 
@@ -3867,6 +3955,363 @@ mod tests {
         assert!(iam_sys.store.cache.snapshot().sts_policies.contains_key("notify-sts-parent"));
     }
 
+    #[tokio::test]
+    async fn delete_temp_account_notifies_peers_as_sts_user() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
+
+        crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("deleted-notify-sts", true)
+                    .await
+                    .expect("delete temporary account");
+            })
+            .await;
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some(("deleted-notify-sts", true)),
+            "peer notification must retain the STS access key and user type"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_reports_peer_invalidation_failure() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(STS_INVALIDATION_MAX_ATTEMPTS, false, false);
+
+        crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                let result = iam_sys.delete_temp_account("deleted-notify-sts", true).await;
+                let err = result.expect_err("failed peer invalidation must fail STS revocation");
+                assert!(err.to_string().contains("peer notification failed"));
+            })
+            .await;
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some(("deleted-notify-sts", true)),
+            "failed notification must retain the STS access key and user type"
+        );
+        assert_eq!(probe.attempts.load(std::sync::atomic::Ordering::SeqCst), STS_INVALIDATION_MAX_ATTEMPTS);
+    }
+
+    #[tokio::test]
+    async fn transient_peer_invalidation_retries_after_local_delete() {
+        const ACCESS_KEY: &str = "retryable-revoked-sts";
+
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(2, false, false);
+        crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), iam_sys.delete_temp_account(ACCESS_KEY, true))
+            .await
+            .expect("transient peer invalidation should converge within the retry budget");
+
+        assert_eq!(
+            probe.attempts.load(std::sync::atomic::Ordering::SeqCst),
+            STS_INVALIDATION_MAX_ATTEMPTS,
+            "peer invalidation must retry until it succeeds"
+        );
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some((ACCESS_KEY, true))
+        );
+    }
+
+    #[tokio::test]
+    async fn stalled_peer_invalidation_is_bounded_by_attempt_timeouts() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, true, false);
+
+        let err = crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("stalled-peer-sts", true)
+                    .await
+                    .expect_err("stalled peer invalidation must fail after bounded attempts")
+            })
+            .await;
+
+        assert!(err.to_string().contains("peer STS invalidation timed out"));
+        assert_eq!(probe.attempts.load(std::sync::atomic::Ordering::SeqCst), STS_INVALIDATION_MAX_ATTEMPTS);
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_reports_local_deletion_failure_without_notifying() {
+        let store = StsTestMockStore::new(false);
+        store.fail_delete.store(true, std::sync::atomic::Ordering::SeqCst);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
+
+        let err = crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("deleted-notify-sts", true)
+                    .await
+                    .expect_err("local deletion failure must fail STS revocation")
+            })
+            .await;
+
+        assert!(err.to_string().contains("delete temporary account failed"));
+        assert!(
+            probe.observed.lock().expect("notification probe mutex poisoned").is_none(),
+            "peer invalidation must not run after local deletion fails"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_reports_notification_task_panic() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, true);
+
+        let err = crate::LOAD_USER_NOTIFICATION_PROBE
+            .scope(Arc::clone(&probe), async {
+                iam_sys
+                    .delete_temp_account("deleted-notify-sts", true)
+                    .await
+                    .expect_err("notification task panic must fail STS revocation")
+            })
+            .await;
+
+        assert!(err.to_string().contains("panicked"));
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_notification_survives_caller_cancellation() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = Arc::new(IamSys::new(cache_manager));
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, true, false);
+        let call = {
+            let iam_sys = Arc::clone(&iam_sys);
+            crate::LOAD_USER_NOTIFICATION_PROBE.scope(Arc::clone(&probe), async move {
+                iam_sys.delete_temp_account("deleted-notify-sts", true).await
+            })
+        };
+        let call = tokio::spawn(call);
+        probe.started.notified().await;
+
+        call.abort();
+        assert!(call.await.expect_err("caller task should be cancelled").is_cancelled());
+        probe
+            .release
+            .as_ref()
+            .expect("blocking probe must have a release signal")
+            .notify_one();
+        probe.completed.notified().await;
+
+        assert_eq!(
+            probe
+                .observed
+                .lock()
+                .expect("notification probe mutex poisoned")
+                .as_ref()
+                .map(|(access_key, temp)| (access_key.as_str(), *temp)),
+            Some(("deleted-notify-sts", true)),
+            "background peer invalidation must complete with the STS access key and user type"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_local_delete_survives_caller_cancellation() {
+        const ACCESS_KEY: &str = "cancelled-during-local-delete";
+
+        let store = StsTestMockStore::new(false);
+        store.saved_sts_users.lock().expect("saved_sts_users mutex poisoned").insert(
+            ACCESS_KEY.to_string(),
+            UserIdentity::from(Credentials {
+                access_key: ACCESS_KEY.to_string(),
+                secret_key: "temporary-user-secret".to_string(),
+                session_token: "session-token".to_string(),
+                status: ACCOUNT_ON.to_string(),
+                ..Default::default()
+            }),
+        );
+        store.block_delete.store(true, std::sync::atomic::Ordering::SeqCst);
+        let store_probe = store.clone();
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = Arc::new(IamSys::new(cache_manager));
+        let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
+        let call = {
+            let iam_sys = Arc::clone(&iam_sys);
+            crate::LOAD_USER_NOTIFICATION_PROBE
+                .scope(Arc::clone(&probe), async move { iam_sys.delete_temp_account(ACCESS_KEY, true).await })
+        };
+        let call = tokio::spawn(call);
+        store_probe.delete_started.notified().await;
+
+        call.abort();
+        assert!(call.await.expect_err("caller task should be cancelled").is_cancelled());
+        store_probe.release_delete.notify_one();
+        probe.completed.notified().await;
+
+        assert!(
+            !store_probe
+                .saved_sts_users
+                .lock()
+                .expect("saved_sts_users mutex poisoned")
+                .contains_key(ACCESS_KEY)
+        );
+    }
+
+    #[test]
+    fn notified_delete_without_tokio_runtime_returns_error() {
+        let runtime = tokio::runtime::Runtime::new().expect("create test runtime");
+        let iam_sys = runtime.block_on(async {
+            let store = StsTestMockStore::new(false);
+            let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+            IamSys::new(cache_manager)
+        });
+        drop(runtime);
+
+        let result = futures::executor::block_on(iam_sys.delete_temp_account("deleted-notify-sts", true));
+        let err = result.expect_err("notified deletion without a Tokio runtime must return an error");
+        assert!(err.to_string().contains("Tokio"));
+    }
+
+    #[tokio::test]
+    async fn notified_delete_without_notification_system_returns_error() {
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+
+        let err = iam_sys
+            .delete_temp_account("deleted-notify-sts", true)
+            .await
+            .expect_err("missing peer notification system must fail STS revocation");
+        assert!(err.to_string().contains("peer notification system is unavailable"));
+    }
+
+    #[tokio::test]
+    async fn missing_sts_notification_evicts_only_sts_cache_entry() {
+        const ACCESS_KEY: &str = "deleted-notify-sts";
+        const GROUP: &str = "deleted-notify-sts-group";
+
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let regular_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "regular-user-secret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let sts_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "temporary-user-secret".to_string(),
+            session_token: "session-token".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let mapped_policy = MappedPolicy::new("readwrite");
+        let membership = HashSet::from([GROUP.to_string()]);
+        let group = GroupInfo::new(vec![ACCESS_KEY.to_string()]);
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(ACCESS_KEY, &regular_user, now);
+            cache.add_or_update_user_policy(ACCESS_KEY, &mapped_policy, now);
+            cache.add_or_update_group(GROUP, &group, now);
+            cache.add_or_update_user_group_membership(ACCESS_KEY, &membership, now);
+            cache.add_or_update_sts_account(ACCESS_KEY, &sts_user, now);
+        });
+
+        iam_sys
+            .load_user(ACCESS_KEY, UserType::Sts)
+            .await
+            .expect("process missing STS user notification");
+
+        let cache = iam_sys.store.cache.snapshot();
+        assert!(!cache.sts_accounts.contains_key(ACCESS_KEY));
+        assert!(
+            cache.users.contains_key(ACCESS_KEY),
+            "STS invalidation must not evict a same-name regular user"
+        );
+        assert!(cache.user_policies.contains_key(ACCESS_KEY));
+        assert!(cache.user_group_memberships.contains_key(ACCESS_KEY));
+        assert!(
+            cache
+                .groups
+                .get(GROUP)
+                .is_some_and(|group| group.members.contains(&ACCESS_KEY.to_string())),
+            "STS invalidation must preserve same-name regular-user group membership"
+        );
+    }
+
+    #[tokio::test]
+    async fn delete_temp_account_preserves_same_name_regular_cache_state() {
+        const ACCESS_KEY: &str = "deleted-notify-sts";
+
+        let store = StsTestMockStore::new(false);
+        let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
+        let iam_sys = IamSys::new(cache_manager);
+        let regular_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "regular-user-secret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let sts_user = UserIdentity::from(Credentials {
+            access_key: ACCESS_KEY.to_string(),
+            secret_key: "temporary-user-secret".to_string(),
+            session_token: "session-token".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        let mapped_policy = MappedPolicy::new("readwrite");
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(ACCESS_KEY, &regular_user, now);
+            cache.add_or_update_user_policy(ACCESS_KEY, &mapped_policy, now);
+            cache.add_or_update_sts_account(ACCESS_KEY, &sts_user, now);
+        });
+
+        iam_sys
+            .delete_temp_account(ACCESS_KEY, false)
+            .await
+            .expect("delete temporary account without peer notification");
+
+        let cache = iam_sys.store.cache.snapshot();
+        assert!(!cache.sts_accounts.contains_key(ACCESS_KEY));
+        assert!(cache.users.contains_key(ACCESS_KEY));
+        assert!(cache.user_policies.contains_key(ACCESS_KEY));
+        assert!(
+            iam_sys
+                .store
+                .api
+                .deleted_mapped_policies
+                .lock()
+                .expect("deleted_mapped_policies mutex poisoned")
+                .is_empty(),
+            "deleting one STS identity must not delete a parent-scoped STS policy mapping"
+        );
+    }
+
     #[tokio::test]
     async fn test_missing_user_notification_cleans_related_cache_state() {
         let store = StsTestMockStore::new(false);