fix(replication): close IAM snapshot, marker purge and broadcast gaps (#7195)

rustfs/src/admin/handlers/user.rs

@@ -1131,6 +1131,7 @@ impl Operation for ImportIam {
                         expiration: req.expiration,
                         allow_site_replicator_account: false,
                         claims: Some(req.claims),
+                        status: None,
                     };
 
                     let groups = if req.groups.is_empty() { None } else { Some(req.groups) };

rustfs/src/admin/storage_api.rs

@@ -335,6 +335,25 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await
     }
 
+    /// [`update_if_incarnation`] stamping the config with a replicated edit's
+    /// source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn update_if_incarnation_at(
+        bucket: &str,
+        config_file: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: uuid::Uuid,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::update_if_incarnation_at(
+            bucket,
+            config_file,
+            data,
+            expected_incarnation_id,
+            updated_at,
+        )
+        .await
+    }
+
     pub(crate) async fn update_quota_if_incarnation(
         bucket: &str,
         data: Vec<u8>,
@@ -344,6 +363,25 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await
     }
 
+    /// [`update_quota_if_incarnation`] stamping the quota with a replicated
+    /// edit's source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn update_quota_if_incarnation_at(
+        bucket: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: uuid::Uuid,
+        proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::update_quota_if_incarnation_at(
+            bucket,
+            data,
+            expected_incarnation_id,
+            proof,
+            updated_at,
+        )
+        .await
+    }
+
     pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> {
         super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await
     }
@@ -398,6 +436,18 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::delete_if_incarnation(bucket, config_file, expected_incarnation_id).await
     }
 
+    /// [`delete_if_incarnation`] stamping the cleared config with a replicated
+    /// deletion's source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn delete_if_incarnation_at(
+        bucket: &str,
+        config_file: &str,
+        expected_incarnation_id: uuid::Uuid,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::delete_if_incarnation_at(bucket, config_file, expected_incarnation_id, updated_at)
+            .await
+    }
+
     pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await
     }

rustfs/src/app/storage_api.rs

@@ -718,6 +718,8 @@ pub(crate) mod bucket {
                     delete_marker_version_id: None,
                     delete_marker: false,
                     delete_marker_mtime: None,
+                    target_delete_marker_version_ids: Default::default(),
+                    target_delete_marker_version_ids_corrupt: false,
                     target_arns,
                     force_delete_id: Some(operation_id),
                     force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)),

rustfs/src/site_replication/hooks.rs

@@ -302,7 +302,164 @@ pub(crate) fn site_replication_state_replicates_ilm_expiry(state: &SiteReplicati
     state.peers.values().any(|peer| peer.replicate_ilm_expiry)
 }
 
-pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteReplicationBootstrapPlan> {
+/// Secret-bearing half of the IAM snapshot. `SRInfo` is served to admin
+/// callers (`site-replication/info`, status, add preflight) and must stay
+/// secret-free, so the bootstrap plan receives credentials through this
+/// separate value, built only on the paths that deliver to peers (site add
+/// bootstrap, repair, retry snapshot resend). Never persisted, never served.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct SiteReplicationIamCredentials {
+    /// Built-in users (access key -> credential); temp and service accounts
+    /// are excluded, external/IdP users never appear here.
+    pub(crate) users: BTreeMap<String, SiteReplicationUserCredential>,
+    /// Every service account except the site replicator's own, already
+    /// shaped as the `service-account` create item the live hook emits.
+    pub(crate) service_accounts: Vec<SiteReplicationServiceAccountSnapshot>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SiteReplicationUserCredential {
+    pub(crate) secret_key: String,
+    pub(crate) status: AccountStatus,
+    /// The user record's own update time (the axis the receiver's staleness
+    /// check compares against), unlike `UserInfo::updated_at` which
+    /// `list_users` overwrites with the policy mapping's time.
+    pub(crate) updated_at: Option<OffsetDateTime>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SiteReplicationServiceAccountSnapshot {
+    pub(crate) create: SRSvcAccCreate,
+    pub(crate) envelope: Option<SRSvcAccReplicationEnvelope>,
+    pub(crate) updated_at: Option<OffsetDateTime>,
+}
+
+pub(crate) const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2;
+
+pub(crate) fn encode_service_account_replication_policy(
+    claims: &HashMap<String, Value>,
+    session_policy: Option<&str>,
+) -> S3Result<(SRSessionPolicy, Option<SRSvcAccReplicationEnvelope>)> {
+    if !claims.contains_key(OIDC_VIRTUAL_PARENT_CLAIM) {
+        return session_policy
+            .map(SRSessionPolicy::from_json)
+            .transpose()
+            .map(|policy| policy.unwrap_or_default())
+            .map(|policy| (policy, None))
+            .map_err(|err| s3_error!(InvalidArgument, "marshal policy failed: {:?}", err));
+    }
+
+    let policy = match session_policy {
+        Some(policy) => serde_json::from_str::<Policy>(policy)
+            .map_err(|err| s3_error!(InvalidArgument, "invalid service account replication policy: {:?}", err))?,
+        None => Policy::default(),
+    };
+    if policy.statements.is_empty() && (!policy.id.is_empty() || !policy.version.is_empty())
+        || policy.version.is_empty() && !policy.statements.is_empty()
+    {
+        return Err(s3_error!(InvalidArgument, "service account replication policy is not normalized"));
+    }
+    let policy = serde_json::to_string(&policy)
+        .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?;
+    let policy = SRSessionPolicy::from_json(&policy)
+        .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?;
+    Ok((
+        policy,
+        Some(SRSvcAccReplicationEnvelope {
+            version: SERVICE_ACCOUNT_ENVELOPE_VERSION,
+        }),
+    ))
+}
+
+/// Read the credentials the IAM snapshot needs straight from the IAM store:
+/// `list_users` deliberately strips secret keys and skips service accounts,
+/// which is right for an admin listing and wrong for a peer snapshot (the
+/// plan builder used to drop every user for lack of a secret, so a status
+/// change or secret rotation committed while a peer was unreachable never
+/// reached it — backlog#2289).
+pub(crate) async fn build_sr_iam_credentials() -> S3Result<SiteReplicationIamCredentials> {
+    let mut credentials = SiteReplicationIamCredentials::default();
+    let Some(iam_sys) = current_iam_handle() else {
+        return Ok(credentials);
+    };
+
+    let mut users = HashMap::new();
+    iam_sys.load_users(UserType::Reg, &mut users).await.map_err(ApiError::from)?;
+    for (access_key, identity) in users {
+        if identity.credentials.is_temp() || identity.credentials.is_service_account() {
+            continue;
+        }
+        credentials.users.insert(
+            access_key,
+            SiteReplicationUserCredential {
+                secret_key: identity.credentials.secret_key,
+                status: if identity.credentials.status == "off" {
+                    AccountStatus::Disabled
+                } else {
+                    AccountStatus::Enabled
+                },
+                updated_at: identity.update_at,
+            },
+        );
+    }
+
+    let mut service_accounts = HashMap::new();
+    iam_sys
+        .load_users(UserType::Svc, &mut service_accounts)
+        .await
+        .map_err(ApiError::from)?;
+    let mut service_accounts: Vec<_> = service_accounts.into_iter().collect();
+    service_accounts.sort_by(|(a, _), (b, _)| a.cmp(b));
+    for (access_key, identity) in service_accounts {
+        // The replicator account is installed by join / rotate, never by a snapshot.
+        if access_key == SITE_REPLICATOR_SERVICE_ACCOUNT || !identity.credentials.is_service_account() {
+            continue;
+        }
+        let claims = iam_sys.get_claims_for_svc_acc(&access_key).await.map_err(ApiError::from)?;
+        let (account, session_policy) = iam_sys.get_service_account(&access_key).await.map_err(ApiError::from)?;
+        let session_policy = session_policy
+            .map(|policy| serde_json::to_string(&policy))
+            .transpose()
+            .map_err(|err| {
+                S3Error::with_message(
+                    S3ErrorCode::InternalError,
+                    format!("marshal service account session policy failed: {err:?}"),
+                )
+            })?;
+        let (session_policy, envelope) = encode_service_account_replication_policy(&claims, session_policy.as_deref())?;
+        credentials.service_accounts.push(SiteReplicationServiceAccountSnapshot {
+            create: SRSvcAccCreate {
+                parent: identity.credentials.parent_user,
+                access_key,
+                secret_key: identity.credentials.secret_key,
+                groups: identity.credentials.groups.unwrap_or_default(),
+                claims,
+                session_policy,
+                status: identity.credentials.status,
+                name: account.name.unwrap_or_default(),
+                description: account.description.unwrap_or_default(),
+                expiration: account.expiration,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            },
+            envelope,
+            updated_at: identity.update_at,
+        });
+    }
+
+    Ok(credentials)
+}
+
+/// The bootstrap plan for peer delivery: `info` (secret-free) plus the IAM
+/// credentials read at this moment.
+pub(crate) async fn build_site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteReplicationBootstrapPlan> {
+    let credentials = build_sr_iam_credentials().await?;
+    site_replication_bootstrap_plan(info, &credentials)
+}
+
+pub(crate) fn site_replication_bootstrap_plan(
+    info: &SRInfo,
+    credentials: &SiteReplicationIamCredentials,
+) -> S3Result<SiteReplicationBootstrapPlan> {
     let mut plan = SiteReplicationBootstrapPlan::default();
     let replicate_ilm_expiry = site_replication_info_replicates_ilm_expiry(info);
 
@@ -318,24 +475,57 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
     }
 
     for (access_key, user) in &info.user_info_map {
-        if let Some(secret_key) = &user.secret_key {
-            plan.iam_items.push(SRIAMItem {
-                r#type: "iam-user".to_string(),
-                iam_user: Some(rustfs_madmin::SRIAMUser {
-                    access_key: access_key.clone(),
-                    is_delete_req: false,
-                    user_req: Some(AddOrUpdateUserReq {
-                        secret_key: secret_key.clone(),
-                        policy: user.policy_name.clone(),
-                        status: user.status.clone(),
-                    }),
-                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        // Credentials come from the store snapshot; an inline `secret_key` on
+        // the SRInfo entry (older callers, tests) is accepted as a fallback.
+        // Users with neither (external / IdP identities) have nothing a peer
+        // could install and are skipped.
+        let credential = credentials.users.get(access_key);
+        let Some(secret_key) = credential
+            .map(|credential| credential.secret_key.clone())
+            .or_else(|| user.secret_key.clone())
+            .filter(|secret_key| !secret_key.is_empty())
+        else {
+            continue;
+        };
+        let status = credential
+            .map(|credential| credential.status.clone())
+            .unwrap_or_else(|| user.status.clone());
+        let updated_at = credential.and_then(|credential| credential.updated_at).or(user.updated_at);
+        plan.iam_items.push(SRIAMItem {
+            r#type: "iam-user".to_string(),
+            iam_user: Some(rustfs_madmin::SRIAMUser {
+                access_key: access_key.clone(),
+                is_delete_req: false,
+                user_req: Some(AddOrUpdateUserReq {
+                    secret_key,
+                    policy: user.policy_name.clone(),
+                    status,
                 }),
-                updated_at: user.updated_at,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        });
+    }
+
+    // Service accounts follow their parents: the receiver creates a missing
+    // account under `parent` and updates an existing one (secret, status,
+    // session policy), so a rotation or disable committed during an outage
+    // converges through the same snapshot as users do.
+    for account in &credentials.service_accounts {
+        plan.iam_items.push(SRIAMItem {
+            r#type: "service-account".to_string(),
+            svc_acc_change: Some(SRSvcAccChange {
+                create: Some(account.create.clone()),
+                oidc_service_account_envelope: account.envelope.clone(),
                 api_version: Some(SITE_REPL_API_VERSION.to_string()),
                 ..Default::default()
-            });
-        }
+            }),
+            updated_at: account.updated_at,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        });
     }
 
     for (name, desc) in &info.group_desc_map {
@@ -518,7 +708,12 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
     } else {
         path
     };
-    broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await?;
+    // Both steps run to completion on their own: the broadcast attempts every
+    // peer and reports the first failure (backlog#2293), so stopping here on
+    // that error would skip `configure-replication` for the peers whose
+    // `make` just succeeded — and nothing records a retry for that gap. The
+    // failed peer's retry events cover both steps independently.
+    let make_result = broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await;
 
     let configure_path = bootstrap_bucket_op_path(bucket, "configure-replication");
     let configure_path = if let Some(token) = bootstrap_token {
@@ -526,7 +721,8 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
     } else {
         configure_path
     };
-    broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
+    let configure_result = broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await;
+    make_result.and(configure_result)
 }
 
 const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
@@ -832,6 +1028,21 @@ pub async fn site_replication_iam_change_hook(item: SRIAMItem) -> S3Result<()> {
     let Some(runtime) = runtime_site_replication_targets().await? else {
         return Ok(());
     };
+    // A local revoke must out-rank a stale grant a peer delivers later, so its
+    // mark is committed before the broadcast (backlog#2291). The broadcast
+    // still goes out when the mark cannot be persisted: the peers' own records
+    // remain the primary gate, the mark only covers the deleted case.
+    if let Err(err) = record_iam_deletion_marks_for_item(&item).await {
+        warn!(
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
+            event = EVENT_ADMIN_SITE_REPLICATION_STATE,
+            item_type = %item.r#type,
+            result = "iam_deletion_mark_not_recorded",
+            error = ?err,
+            "failed to record local IAM deletion mark before broadcast"
+        );
+    }
     let mut first_error: Option<S3Error> = None;
     for peer in runtime.state.peers.values() {
         if peer.deployment_id == runtime.local_peer.deployment_id

rustfs/src/site_replication/mod.rs

@@ -79,13 +79,16 @@ use http::header::{CONTENT_TYPE, HOST};
 use http::{HeaderMap, HeaderValue, Uri};
 use hyper::{Method, StatusCode};
 use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH};
+use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM;
 use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type};
 use rustfs_iam::sys::SITE_REPLICATOR_SERVICE_ACCOUNT;
 use rustfs_madmin::{
-    AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus, SITE_REPL_API_VERSION,
-    SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq, SRResyncOpStatus,
-    SRRetryStats, SRStateInfo, SyncStatus,
+    AccountStatus, AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus,
+    SITE_REPL_API_VERSION, SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq,
+    SRResyncOpStatus, SRRetryStats, SRSessionPolicy, SRStateInfo, SRSvcAccChange, SRSvcAccCreate, SRSvcAccDelete,
+    SRSvcAccReplicationEnvelope, SyncStatus,
 };
+use rustfs_policy::policy::Policy;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration};
@@ -107,6 +110,26 @@ use tracing::{info, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
+/// Serialize `value` with every JSON object's keys sorted, for hashing and
+/// equality checks. `HashMap` fields (service-account claims) iterate in a
+/// per-instance random order and `serde_json` is built with `preserve_order`,
+/// so two identical plans would otherwise hash differently: the repair
+/// preflight token went stale between dry-run and execute, and a retry
+/// snapshot resend never looked "stable" (backlog#2289 follow-up).
+pub(crate) fn canonical_json_vec<T: Serialize>(value: &T) -> serde_json::Result<Vec<u8>> {
+    fn sort_keys(value: Value) -> Value {
+        match value {
+            Value::Object(map) => {
+                let sorted: BTreeMap<String, Value> = map.into_iter().map(|(key, value)| (key, sort_keys(value))).collect();
+                Value::Object(sorted.into_iter().collect())
+            }
+            Value::Array(items) => Value::Array(items.into_iter().map(sort_keys).collect()),
+            other => other,
+        }
+    }
+    serde_json::to_vec(&sort_keys(serde_json::to_value(value)?))
+}
+
 pub(crate) const LOG_COMPONENT_ADMIN: &str = "admin";
 
 pub(crate) const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication";

rustfs/src/site_replication/repair.rs

@@ -234,9 +234,9 @@ impl SiteReplicationRepairTask<'_> {
 
     pub(crate) fn id(&self) -> S3Result<String> {
         let payload = match self {
-            Self::Iam(item) => serde_json::to_vec(item),
+            Self::Iam(item) => canonical_json_vec(item),
             Self::BucketMake(_) | Self::Replication(_) => serde_json::to_vec(&serde_json::json!({})),
-            Self::BucketMetadata(item) => serde_json::to_vec(item),
+            Self::BucketMetadata(item) => canonical_json_vec(item),
         }
         .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair task failed: {err}")))?;
         let mut digest = Sha256::new();
@@ -726,7 +726,7 @@ pub(crate) async fn execute_site_replication_repair_locked(
         return Err(s3_error!(InvalidRequest, "site replication is not configured"));
     }
     let info = build_sr_info(&state, &request.local_peer).await?;
-    let plan = site_replication_bootstrap_plan(&info)?;
+    let plan = build_site_replication_bootstrap_plan(&info).await?;
     let plan_token = site_replication_repair_plan_token(&state, &plan)?;
     let preflight_token = site_replication_repair_preflight_token(&state, &plan, request.signing_key.as_bytes())?;
     let sites = site_replication_repair_sites(&state, &request.local_peer, &plan, request.signing_key.as_bytes())?;

rustfs/src/site_replication/retry.rs

@@ -397,12 +397,12 @@ pub(crate) fn iam_deletion_replay_matches(record: &SiteReplicationIamDeletionRep
 /// newer revision of one another.
 pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
     match item.r#type.as_str() {
-        "policy" if item.policy.is_none() => Some(format!("policy:{}", item.name)),
+        "policy" if item.policy.is_none() => Some(iam_policy_deletion_mark_entity(&item.name)),
         "iam-user" => item
             .iam_user
             .as_ref()
             .filter(|user| user.is_delete_req)
-            .map(|user| format!("iam-user:{}", user.access_key)),
+            .map(|user| iam_user_deletion_mark_entity(&user.access_key)),
         "group-info" => item
             .group_info
             .as_ref()
@@ -416,7 +416,7 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
             .policy_mapping
             .as_ref()
             .filter(|mapping| mapping.policy.is_empty())
-            .map(|mapping| format!("policy-mapping:{}:{}:{}", mapping.user_or_group, mapping.user_type, mapping.is_group)),
+            .map(|mapping| iam_policy_mapping_deletion_mark_entity(&mapping.user_or_group, mapping.user_type, mapping.is_group)),
         "service-account" => item
             .svc_acc_change
             .as_ref()
@@ -426,6 +426,82 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
     }
 }
 
+/// The entities whose deletion a deletion-shaped IAM item commits, keyed the
+/// way the receive-side staleness gate looks them up once the local record is
+/// gone (backlog#2291); empty for creates and updates. Group member removal
+/// yields one entity per removed member so a stale re-add of that member can
+/// be judged, and a group delete (no members) yields the group itself.
+pub(crate) fn iam_item_deletion_mark_entities(item: &SRIAMItem) -> Vec<String> {
+    if item.r#type == "group-info" {
+        let Some(update) = item
+            .group_info
+            .as_ref()
+            .map(|group| &group.update_req)
+            .filter(|update| update.is_remove)
+        else {
+            return Vec::new();
+        };
+        if update.members.is_empty() {
+            return vec![iam_group_deletion_mark_entity(&update.group)];
+        }
+        return update
+            .members
+            .iter()
+            .map(|member| iam_group_member_deletion_mark_entity(&update.group, member))
+            .collect();
+    }
+    iam_item_deletion_entity(item).into_iter().collect()
+}
+
+pub(crate) fn iam_policy_deletion_mark_entity(name: &str) -> String {
+    format!("policy:{name}")
+}
+
+pub(crate) fn iam_user_deletion_mark_entity(access_key: &str) -> String {
+    format!("iam-user:{access_key}")
+}
+
+/// `user_type` is the SR wire integer, as carried by the item on both sides.
+pub(crate) fn iam_policy_mapping_deletion_mark_entity(user_or_group: &str, user_type: i64, is_group: bool) -> String {
+    format!("policy-mapping:{user_or_group}:{user_type}:{is_group}")
+}
+
+pub(crate) fn iam_group_deletion_mark_entity(group: &str) -> String {
+    format!("group:{group}")
+}
+
+pub(crate) fn iam_group_member_deletion_mark_entity(group: &str, member: &str) -> String {
+    format!("group-member:{group}:{member}")
+}
+
+/// Persist the deletion marks of `item` (its source `updated_at` per entity
+/// of [`iam_item_deletion_mark_entities`]) through the state transaction.
+/// No-op for creates/updates and for items without a source timestamp
+/// (older peers): a mark without a source clock could not be ordered against
+/// later items. Called before a local deletion is broadcast and after a
+/// replicated deletion is applied, so both sides out-rank a stale grant that
+/// arrives later.
+pub(crate) async fn record_iam_deletion_marks_for_item(item: &SRIAMItem) -> S3Result<()> {
+    let entities = iam_item_deletion_mark_entities(item);
+    let Some(deleted_at) = item.updated_at.filter(|_| !entities.is_empty()) else {
+        return Ok(());
+    };
+    commit_iam_deletion_marks(entities, deleted_at).await
+}
+
+/// [`record_iam_deletion_marks`] under the state transaction; the write is
+/// skipped when no mark moves.
+pub(crate) async fn commit_iam_deletion_marks(entities: Vec<String>, deleted_at: OffsetDateTime) -> S3Result<()> {
+    update_site_replication_state_when_changed(move |state| {
+        Ok(if record_iam_deletion_marks(state, &entities, deleted_at) {
+            StateCommit::Changed(())
+        } else {
+            StateCommit::Unchanged(())
+        })
+    })
+    .await
+}
+
 /// Failure bookkeeping for one IAM item delivery: upsert the collapsed retry
 /// event and, when the item is a deletion, record its body for replay. Both
 /// live in the same state so the caller commits them in one transaction — a
@@ -791,8 +867,8 @@ impl RetrySnapshot {
 
     pub(crate) fn fingerprint(&self) -> S3Result<Vec<Vec<u8>>> {
         let mut payloads = match self {
-            Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::<Result<Vec<_>, _>>(),
-            Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::<Result<Vec<_>, _>>(),
+            Self::Iam(items) => items.iter().map(canonical_json_vec).collect::<Result<Vec<_>, _>>(),
+            Self::BucketMetadata(items) => items.iter().map(canonical_json_vec).collect::<Result<Vec<_>, _>>(),
         }
         .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?;
         payloads.sort_unstable();
@@ -954,6 +1030,7 @@ pub(crate) enum IamSnapshotKey {
     User(String),
     Group(String),
     PolicyMapping { target: String, user_type: i64, is_group: bool },
+    ServiceAccount(String),
 }
 
 pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option<IamSnapshotKey> {
@@ -972,6 +1049,11 @@ pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option<IamSnapshotKey> {
             user_type: mapping.user_type,
             is_group: mapping.is_group,
         }),
+        "service-account" => item
+            .svc_acc_change
+            .as_ref()
+            .and_then(|change| change.create.as_ref())
+            .map(|create| IamSnapshotKey::ServiceAccount(create.access_key.clone())),
         _ => None,
     }
 }
@@ -1006,6 +1088,24 @@ pub(crate) fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateT
                 mapping.policy.clear();
             }
         }
+        "service-account" => {
+            let Some(access_key) = item
+                .svc_acc_change
+                .as_ref()
+                .and_then(|change| change.create.as_ref())
+                .map(|create| create.access_key.clone())
+            else {
+                return Vec::new();
+            };
+            tombstone.svc_acc_change = Some(SRSvcAccChange {
+                delete: Some(SRSvcAccDelete {
+                    access_key,
+                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                }),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            });
+        }
         _ => return Vec::new(),
     }
     vec![tombstone]
@@ -1701,7 +1801,7 @@ pub(crate) async fn drain_site_replication_retry_queue_locked(
     // tick and only when a snapshot resend is actually due.
     let plan = if needs_plan {
         let info = build_sr_info(&runtime.state, &runtime.local_peer).await?;
-        Some(site_replication_bootstrap_plan(&info)?)
+        Some(build_site_replication_bootstrap_plan(&info).await?)
     } else {
         None
     };
@@ -1841,7 +1941,7 @@ pub(crate) async fn drain_one_site_replication_retry_event(
                     }
                 }
                 let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?;
-                let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?;
+                let fresh_plan = build_site_replication_bootstrap_plan(&fresh_info).await?;
                 let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot");
                 if fresh_snapshot.fingerprint()? == current_fingerprint {
                     if is_iam {

rustfs/src/site_replication/state.rs

@@ -64,6 +64,104 @@ pub(crate) struct SiteReplicationState {
     /// newer edit that already landed.
     #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub(crate) applied_edit_generations: BTreeMap<String, u64>,
+    /// Source timestamp of the newest IAM deletion committed on this site,
+    /// keyed by the deleted entity (`iam_item_deletion_mark_entities`). A
+    /// deletion leaves no local record to judge a later item against, so this
+    /// is what lets the receive-side staleness gate reject a grant that is
+    /// older than the revoke it would otherwise undo (backlog#2291). Marks
+    /// are kept for [`SITE_REPLICATION_IAM_DELETION_MARK_RETENTION`] and never
+    /// evicted by count: see that constant for why a count bound would open
+    /// exactly the window the marks exist to close.
+    #[serde(default, with = "rfc3339_map", skip_serializing_if = "BTreeMap::is_empty")]
+    pub(crate) iam_deletion_marks: BTreeMap<String, OffsetDateTime>,
+}
+
+/// How long an IAM deletion mark outlives the deletion it records.
+///
+/// A mark fences the delivery paths that can still carry an older grant for
+/// the deleted entity: a live delivery delayed in transit, the same grant
+/// arriving on a sibling node while the revoke is being applied, and a
+/// snapshot (bootstrap / repair / resend) built by a peer that has not yet
+/// received the deletion — which is bounded by this site's own retry queue
+/// towards that peer, whose backoff tops out at one day
+/// (`SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS`). The retry drain itself
+/// never replays a stale grant: it resends snapshots of the current records
+/// and the recorded deletion bodies. Thirty days is an order of magnitude
+/// beyond every one of those windows. Marks are pruned by age only — a count
+/// bound would drop a mark that is still inside the delivery window as soon
+/// as enough newer deletions happen, letting the delayed grant re-create the
+/// entity, which is the very hole the marks close.
+pub(crate) const SITE_REPLICATION_IAM_DELETION_MARK_RETENTION: time::Duration = time::Duration::days(30);
+
+/// Record that deletions of `entities` with source timestamp `deleted_at`
+/// were committed here. Newest wins per entity: an older deletion never
+/// lowers a mark. Marks older than the retention are pruned in the same
+/// pass. Returns whether the state changed.
+pub(crate) fn record_iam_deletion_marks(
+    state: &mut SiteReplicationState,
+    entities: &[String],
+    deleted_at: OffsetDateTime,
+) -> bool {
+    record_iam_deletion_marks_at(state, entities, deleted_at, OffsetDateTime::now_utc())
+}
+
+/// [`record_iam_deletion_marks`] pruning against an explicit `now`.
+pub(crate) fn record_iam_deletion_marks_at(
+    state: &mut SiteReplicationState,
+    entities: &[String],
+    deleted_at: OffsetDateTime,
+    now: OffsetDateTime,
+) -> bool {
+    let mut changed = false;
+    for entity in entities {
+        if state
+            .iam_deletion_marks
+            .get(entity)
+            .is_some_and(|existing| *existing >= deleted_at)
+        {
+            continue;
+        }
+        state.iam_deletion_marks.insert(entity.clone(), deleted_at);
+        changed = true;
+    }
+    let expired_before = now - SITE_REPLICATION_IAM_DELETION_MARK_RETENTION;
+    let before = state.iam_deletion_marks.len();
+    state.iam_deletion_marks.retain(|_, deleted_at| *deleted_at >= expired_before);
+    changed || state.iam_deletion_marks.len() != before
+}
+
+/// Newest deletion mark among `entities`, or `None` when no deletion of any
+/// of them was recorded here. The receive-side staleness gate feeds this in
+/// as the local timestamp when the targeted record is absent.
+pub(crate) fn iam_deletion_mark(state: &SiteReplicationState, entities: &[String]) -> Option<OffsetDateTime> {
+    entities
+        .iter()
+        .filter_map(|entity| state.iam_deletion_marks.get(entity).copied())
+        .max()
+}
+
+/// RFC 3339 map values, matching the other timestamps in the state object
+/// (`time::serde::rfc3339` only applies to a single field).
+mod rfc3339_map {
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+    use std::collections::BTreeMap;
+    use time::OffsetDateTime;
+
+    #[derive(Serialize, Deserialize)]
+    #[serde(transparent)]
+    struct Stamp(#[serde(with = "time::serde::rfc3339")] OffsetDateTime);
+
+    pub(super) fn serialize<S: Serializer>(map: &BTreeMap<String, OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.collect_map(map.iter().map(|(entity, deleted_at)| (entity, Stamp(*deleted_at))))
+    }
+
+    pub(super) fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<BTreeMap<String, OffsetDateTime>, D::Error> {
+        let map = BTreeMap::<String, Stamp>::deserialize(deserializer)?;
+        Ok(map
+            .into_iter()
+            .map(|(entity, Stamp(deleted_at))| (entity, deleted_at))
+            .collect())
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -323,6 +421,33 @@ where
     update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await
 }
 
+/// The state transaction for work that has to await inside it: an IAM write
+/// that must be ordered with the staleness verdict taken before it and the
+/// deletion mark committed after it (backlog#2291). Same boundary as
+/// [`update_site_replication_state`] — load and persist under the
+/// distributed state-object write lock, so two nodes of this site cannot
+/// interleave their verdicts and writes — and the same rules inside: no peer
+/// network calls and no other config locks. The closure hands the state back
+/// as `Some` when it changed it; `None` skips the write.
+pub(crate) async fn with_site_replication_state_transaction<T, F, Fut>(transaction: F) -> S3Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce(SiteReplicationState) -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = S3Result<(T, Option<SiteReplicationState>)>> + Send + 'static,
+{
+    with_site_replication_state_lock(move || async move {
+        let store = current_object_store_handle()
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
+        let state = load_site_replication_state_no_lock(store.clone()).await?;
+        let (result, changed) = transaction(state).await?;
+        if let Some(state) = changed {
+            persist_site_replication_state_no_lock(store, state).await?;
+        }
+        Ok(result)
+    })
+    .await
+}
+
 /// [`update_site_replication_state`] for closures that may find nothing to
 /// do — see [`StateCommit`].
 pub(crate) async fn update_site_replication_state_when_changed<T, F>(update: F) -> S3Result<T>

rustfs/src/site_replication/tests.rs

@@ -554,6 +554,145 @@ fn test_iam_item_deletion_entity_shapes() {
     assert!(iam_item_deletion_entity(&policy_set).is_none());
 }
 
+/// Deletion marks (backlog#2291) key on the same entities as the replay
+/// records, except that a group member removal is marked per member (so a
+/// stale re-add of one member can be judged) and a group delete marks the
+/// group itself. Creates and updates leave no mark.
+#[test]
+fn test_iam_item_deletion_mark_entities_shapes() {
+    assert_eq!(
+        iam_item_deletion_mark_entities(&user_delete_item("alice")),
+        vec!["iam-user:alice".to_string()]
+    );
+    assert_eq!(
+        iam_item_deletion_mark_entities(&policy_delete_item("readonly")),
+        vec!["policy:readonly".to_string()]
+    );
+
+    let mut group_remove = SRIAMItem {
+        r#type: "group-info".to_string(),
+        group_info: Some(SRGroupInfo {
+            update_req: GroupAddRemove {
+                group: "devs".to_string(),
+                members: vec!["bob".to_string(), "alice".to_string()],
+                status: GroupStatus::Enabled,
+                is_remove: true,
+            },
+            api_version: None,
+        }),
+        ..Default::default()
+    };
+    assert_eq!(
+        iam_item_deletion_mark_entities(&group_remove),
+        vec!["group-member:devs:bob".to_string(), "group-member:devs:alice".to_string()]
+    );
+    group_remove
+        .group_info
+        .as_mut()
+        .expect("group info")
+        .update_req
+        .members
+        .clear();
+    assert_eq!(
+        iam_item_deletion_mark_entities(&group_remove),
+        vec!["group:devs".to_string()],
+        "a removal without members deletes the group"
+    );
+    group_remove.group_info.as_mut().expect("group info").update_req.is_remove = false;
+    assert!(iam_item_deletion_mark_entities(&group_remove).is_empty());
+
+    let mapping_clear = SRIAMItem {
+        r#type: "policy-mapping".to_string(),
+        policy_mapping: Some(SRPolicyMapping {
+            user_or_group: "alice".to_string(),
+            user_type: 0,
+            is_group: false,
+            policy: String::new(),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    assert_eq!(
+        iam_item_deletion_mark_entities(&mapping_clear),
+        vec!["policy-mapping:alice:0:false".to_string()]
+    );
+
+    let mut user_create = user_delete_item("alice");
+    user_create.iam_user.as_mut().expect("iam user").is_delete_req = false;
+    assert!(iam_item_deletion_mark_entities(&user_create).is_empty());
+}
+
+/// Newest wins per entity, marks are pruned by age only (never by count: a
+/// count bound would drop a mark still inside the delivery window as soon as
+/// enough newer deletions happen), and the timestamps survive the state
+/// object as RFC 3339.
+#[test]
+fn test_record_iam_deletion_marks_newest_wins_and_expires_by_age_only() {
+    let at = |seconds: i64| OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds);
+    let now = at(1_000_000);
+    let mut state = SiteReplicationState::default();
+    let alice = vec!["iam-user:alice".to_string()];
+
+    assert!(record_iam_deletion_marks_at(&mut state, &alice, at(20), now));
+    assert!(
+        !record_iam_deletion_marks_at(&mut state, &alice, at(10), now),
+        "an older deletion does not move the mark"
+    );
+    assert!(
+        !record_iam_deletion_marks_at(&mut state, &alice, at(20), now),
+        "a replayed deletion is not a change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(20)));
+    assert!(record_iam_deletion_marks_at(&mut state, &alice, at(30), now));
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)));
+    assert_eq!(iam_deletion_mark(&state, &["iam-user:bob".to_string()]), None);
+    assert!(!record_iam_deletion_marks_at(&mut state, &[], at(40), now));
+
+    // Many newer deletions never evict an older mark that is still within the retention.
+    let members: Vec<String> = (0..4096).map(|index| format!("group-member:devs:user-{index:04}")).collect();
+    for (index, member) in members.iter().enumerate() {
+        record_iam_deletion_marks_at(&mut state, std::slice::from_ref(member), at(100 + index as i64), now);
+    }
+    assert_eq!(state.iam_deletion_marks.len(), members.len() + 1);
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)), "no count-based eviction");
+
+    // Marks older than the retention are pruned, on the pass that records a
+    // newer one and on a pass that changes nothing else; younger ones stay.
+    let later = at(100) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION;
+    assert!(
+        record_iam_deletion_marks_at(&mut state, &["iam-user:carol".to_string()], at(200_000), later),
+        "pruning alone is a change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), None, "alice's mark aged out");
+    assert_eq!(
+        iam_deletion_mark(&state, &members[..1]),
+        Some(at(100)),
+        "a mark exactly at the retention edge stays, and so do the younger ones"
+    );
+    assert_eq!(state.iam_deletion_marks.len(), members.len() + 1);
+    assert_eq!(iam_deletion_mark(&state, &["iam-user:carol".to_string()]), Some(at(200_000)));
+    let mut state = SiteReplicationState::default();
+    record_iam_deletion_marks_at(&mut state, &alice, at(30), now);
+    let past_edge = at(30) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION + time::Duration::seconds(1);
+    assert!(
+        record_iam_deletion_marks_at(&mut state, &[], at(0), past_edge),
+        "a pass that only prunes reports the change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), None);
+    record_iam_deletion_marks_at(&mut state, &alice, at(30), now);
+
+    let json = serde_json::to_value(&state).expect("serialize state");
+    assert_eq!(json["iam_deletion_marks"]["iam-user:alice"], serde_json::json!("1970-01-01T00:00:30Z"));
+    let reloaded = parse_site_replication_state(&serde_json::to_vec(&state).expect("serialize state")).expect("parse state");
+    assert_eq!(reloaded.iam_deletion_marks, state.iam_deletion_marks);
+    assert!(
+        parse_site_replication_state(br#"{"name":"a","service_account_access_key":"","service_account_parent":"","peers":{},"updated_at":null,"resync_status":{}}"#)
+            .expect("state without marks")
+            .iam_deletion_marks
+            .is_empty()
+    );
+}
+
 /// A failed deletion delivery persists a replay record next to the collapsed
 /// retry entry; a fresh entry is stamped `deletions_recorded` so a later
 /// replay can settle it, and a repeated deletion of the same entity keeps the
@@ -1679,7 +1818,8 @@ fn test_site_replication_bootstrap_plan_includes_replayable_snapshot_items() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     assert_eq!(plan.iam_items.iter().map(|item| item.r#type.as_str()).collect::<Vec<_>>(), {
         vec!["policy", "iam-user", "group-info", "policy-mapping"]
@@ -1717,7 +1857,8 @@ fn test_site_replication_bootstrap_plan_skips_lifecycle_by_default() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     assert!(!plan.bucket_items.iter().any(|item| item.r#type == "lc-config"));
 }
@@ -1748,7 +1889,8 @@ fn test_site_replication_bootstrap_plan_emits_timestamped_lifecycle_delete() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     let item = plan
         .bucket_items
@@ -1935,8 +2077,8 @@ fn test_site_replication_repair_preflight_token_is_deterministic_for_equal_state
         },
     );
 
-    let plan_a = site_replication_bootstrap_plan(&info).expect("first plan");
-    let plan_b = site_replication_bootstrap_plan(&info).expect("second plan");
+    let plan_a = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("first plan");
+    let plan_b = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("second plan");
     let token_a = site_replication_repair_preflight_token(&state, &plan_a, b"test-signing-key").expect("first token");
     let token_b = site_replication_repair_preflight_token(&state, &plan_b, b"test-signing-key").expect("second token");
 
@@ -3219,3 +3361,345 @@ fn test_reconcile_adds_missing_peer_rules_to_existing_config() {
     assert!(rule_ids.contains(&"site-repl-dep-b"));
     assert!(rule_ids.contains(&"site-repl-dep-c"));
 }
+
+/// backlog#2289: the IAM snapshot (retry resend, repair, site-add bootstrap)
+/// used to be built from `list_users`, whose `UserInfo` never carries a
+/// secret key, so the plan dropped every user and a status change or secret
+/// rotation committed while a peer was unreachable never reached it. The
+/// credentials now come from a separate store read; SRInfo stays secret-free.
+#[test]
+fn test_bootstrap_plan_carries_users_from_the_credential_snapshot() {
+    let mut info = SRInfo::default();
+    // Exactly what `list_users` builds: status, policy, updated_at — never secret_key.
+    info.user_info_map.insert(
+        "alice".to_string(),
+        rustfs_madmin::UserInfo {
+            status: rustfs_madmin::AccountStatus::Disabled,
+            policy_name: Some("readwrite".to_string()),
+            updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
+            ..Default::default()
+        },
+    );
+    info.user_info_map.insert(
+        "external-idp-user".to_string(),
+        rustfs_madmin::UserInfo {
+            status: rustfs_madmin::AccountStatus::Enabled,
+            ..Default::default()
+        },
+    );
+    let user_updated_at = OffsetDateTime::from_unix_timestamp(1_700_000_500).expect("timestamp");
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Disabled,
+            updated_at: Some(user_updated_at),
+        },
+    );
+
+    let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build");
+
+    let users: Vec<_> = plan.iam_items.iter().filter(|item| item.r#type == "iam-user").collect();
+    assert_eq!(users.len(), 1, "only the user with a credential travels: {:?}", plan.iam_items);
+    let alice = users[0].iam_user.as_ref().expect("iam user body");
+    assert_eq!(alice.access_key, "alice");
+    let req = alice.user_req.as_ref().expect("user request");
+    assert_eq!(req.secret_key, "alice-secret");
+    assert_eq!(req.status, rustfs_madmin::AccountStatus::Disabled);
+    assert_eq!(req.policy.as_deref(), Some("readwrite"));
+    // the user record's own axis, not the policy-mapping time list_users reports
+    assert_eq!(users[0].updated_at, Some(user_updated_at));
+}
+
+fn service_account_snapshot(access_key: &str, parent: &str, status: &str) -> SiteReplicationServiceAccountSnapshot {
+    SiteReplicationServiceAccountSnapshot {
+        create: rustfs_madmin::SRSvcAccCreate {
+            parent: parent.to_string(),
+            access_key: access_key.to_string(),
+            secret_key: format!("{access_key}-secret"),
+            groups: Vec::new(),
+            claims: HashMap::new(),
+            session_policy: SRSessionPolicy::default(),
+            status: status.to_string(),
+            name: String::new(),
+            description: String::new(),
+            expiration: None,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        },
+        envelope: None,
+        updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_600).expect("timestamp")),
+    }
+}
+
+/// backlog#2289: service accounts were absent from every snapshot (the
+/// listing filters them). They now travel as the create item the live hook
+/// emits — after their parents — carrying secret and status.
+#[test]
+fn test_bootstrap_plan_emits_service_accounts_after_their_parents() {
+    let mut info = SRInfo::default();
+    info.user_info_map
+        .insert("alice".to_string(), rustfs_madmin::UserInfo::default());
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Enabled,
+            updated_at: None,
+        },
+    );
+    credentials
+        .service_accounts
+        .push(service_account_snapshot("alice-svc", "alice", "off"));
+
+    let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build");
+
+    let types: Vec<_> = plan.iam_items.iter().map(|item| item.r#type.as_str()).collect();
+    assert_eq!(types, vec!["iam-user", "service-account"]);
+    let change = plan.iam_items[1].svc_acc_change.as_ref().expect("service account change");
+    let create = change.create.as_ref().expect("create body");
+    assert_eq!((create.access_key.as_str(), create.parent.as_str()), ("alice-svc", "alice"));
+    assert_eq!(create.secret_key, "alice-svc-secret");
+    assert_eq!(create.status, "off", "a disabled account must arrive disabled");
+    assert!(change.delete.is_none() && change.update.is_none());
+}
+
+/// A service account present in the previous snapshot but gone from the
+/// fresh one is replayed as an explicit delete, like the other IAM kinds.
+#[test]
+fn test_retry_snapshot_tombstones_removed_service_accounts() {
+    let observed_at = OffsetDateTime::from_unix_timestamp(1_700_001_000).expect("timestamp");
+    let mut info = SRInfo::default();
+    info.user_info_map
+        .insert("alice".to_string(), rustfs_madmin::UserInfo::default());
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Enabled,
+            updated_at: None,
+        },
+    );
+    let mut with_account = credentials.clone();
+    with_account
+        .service_accounts
+        .push(service_account_snapshot("alice-svc", "alice", "on"));
+    let previous = site_replication_bootstrap_plan(&info, &with_account).expect("previous plan");
+    let fresh = site_replication_bootstrap_plan(&info, &credentials).expect("fresh plan");
+
+    let replay = RetrySnapshot::replay_after_change(
+        &RetrySnapshot::Iam(previous.iam_items),
+        &RetrySnapshot::Iam(fresh.iam_items),
+        observed_at,
+    );
+    let RetrySnapshot::Iam(items) = replay else {
+        panic!("IAM snapshot expected");
+    };
+    let tombstone = items
+        .iter()
+        .find(|item| item.r#type == "service-account")
+        .expect("service account tombstone");
+    let change = tombstone.svc_acc_change.as_ref().expect("change");
+    assert_eq!(change.delete.as_ref().map(|delete| delete.access_key.as_str()), Some("alice-svc"));
+    assert!(change.create.is_none());
+    assert_eq!(tombstone.updated_at, Some(observed_at));
+}
+
+/// Spawns a one-shot HTTP peer that answers 200 and flips the returned flag
+/// once a request head has arrived.
+async fn spawn_reached_probe_peer() -> (String, Arc<AtomicBool>, tokio::task::JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind healthy peer");
+    let endpoint = format!("http://{}", listener.local_addr().expect("healthy peer address"));
+    let reached = Arc::new(AtomicBool::new(false));
+    let reached_by_server = reached.clone();
+    let server = tokio::spawn(async move {
+        let Ok((mut stream, _)) = listener.accept().await else {
+            return;
+        };
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        loop {
+            let Ok(read) = stream.read(&mut buffer).await else {
+                return;
+            };
+            if read == 0 {
+                return;
+            }
+            request.extend_from_slice(&buffer[..read]);
+            if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                break;
+            }
+        }
+        reached_by_server.store(true, Ordering::SeqCst);
+        let _ = stream
+            .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok")
+            .await;
+    });
+    (endpoint, reached, server)
+}
+
+/// Three-peer runtime whose local peer is `local`; BTreeMap order visits the
+/// failing peer `b` before the healthy peer `c`.
+fn broadcast_runtime_with_failing_peer_before_healthy(failing_endpoint: &str, healthy_endpoint: &str) -> SiteReplicationRuntime {
+    let local_peer = PeerInfo {
+        deployment_id: "local".to_string(),
+        ..peer("local", "http://127.0.0.1:9")
+    };
+    let mut state = SiteReplicationState {
+        name: "local".to_string(),
+        service_account_access_key: "site-replicator-0".to_string(),
+        ..Default::default()
+    };
+    state.peers.insert("local".to_string(), local_peer.clone());
+    state.peers.insert(
+        "b".to_string(),
+        PeerInfo {
+            deployment_id: "b".to_string(),
+            ..peer("b", failing_endpoint)
+        },
+    );
+    state.peers.insert(
+        "c".to_string(),
+        PeerInfo {
+            deployment_id: "c".to_string(),
+            ..peer("c", healthy_endpoint)
+        },
+    );
+    SiteReplicationRuntime {
+        state,
+        local_peer,
+        service_account_secret_key: "site-replicator-secret".to_string(),
+    }
+}
+
+const BROADCAST_PROBE_DELETE_BUCKET_PATH: &str =
+    "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket";
+
+/// The generic JSON broadcast (bucket make/delete, bucket-meta hook, bucket
+/// ops) attempts every remote peer: a peer whose request fails must not stop
+/// delivery to the peers that follow it in deployment-id order, and the
+/// failure is still reported to the caller (backlog#2293).
+#[tokio::test]
+#[serial]
+async fn test_broadcast_json_reaches_healthy_peers_after_a_failed_peer() {
+    // Peer "b": nothing listens on the port, so the connect is refused.
+    let refused = TcpListener::bind("127.0.0.1:0").await.expect("bind refused-peer probe");
+    let refused_endpoint = format!("http://{}", refused.local_addr().expect("refused-peer address"));
+    drop(refused);
+
+    let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await;
+    let runtime = broadcast_runtime_with_failing_peer_before_healthy(&refused_endpoint, &healthy_endpoint);
+
+    let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async {
+        broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await
+    })
+    .await;
+
+    let err = result.expect_err("peer b refuses connections, the broadcast must report it");
+    assert!(
+        reached.load(Ordering::SeqCst),
+        "peer c never received the broadcast once peer b failed: {err}"
+    );
+    server.abort();
+}
+
+/// Same guarantee when the failing peer never gets a transport: an endpoint
+/// that `PeerTransport::for_runtime_peer` rejects must be skipped past (and
+/// reported), not abort the broadcast before the healthy peers (backlog#2293).
+#[tokio::test]
+#[serial]
+async fn test_broadcast_json_reaches_healthy_peers_after_a_peer_without_transport() {
+    // Peer "b": a scheme the peer connection validator refuses outright.
+    let forbidden_endpoint = "ftp://peer-b.example.com";
+
+    let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await;
+    let runtime = broadcast_runtime_with_failing_peer_before_healthy(forbidden_endpoint, &healthy_endpoint);
+
+    let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async {
+        broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await
+    })
+    .await;
+
+    let err = result.expect_err("peer b has no usable transport, the broadcast must report it");
+    assert!(
+        err.to_string().contains("invalid persisted site replication peer"),
+        "the reported error must be peer b's transport failure: {err}"
+    );
+    assert!(
+        reached.load(Ordering::SeqCst),
+        "peer c never received the broadcast once peer b failed to get a transport: {err}"
+    );
+    server.abort();
+}
+
+fn service_account_item_with_claims(order: &[&str]) -> SRIAMItem {
+    let mut claims = HashMap::new();
+    for key in order {
+        claims.insert((*key).to_string(), serde_json::json!(format!("value-of-{key}")));
+    }
+    SRIAMItem {
+        r#type: "service-account".to_string(),
+        svc_acc_change: Some(SRSvcAccChange {
+            create: Some(rustfs_madmin::SRSvcAccCreate {
+                parent: "alice".to_string(),
+                access_key: "alice-svc".to_string(),
+                secret_key: "alice-svc-secret".to_string(),
+                groups: Vec::new(),
+                claims,
+                session_policy: SRSessionPolicy::default(),
+                status: "on".to_string(),
+                name: String::new(),
+                description: String::new(),
+                expiration: None,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        }),
+        updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
+        api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        ..Default::default()
+    }
+}
+
+/// The repair preflight token and the retry-snapshot fingerprint hash the
+/// serialized items. Service-account claims live in a `HashMap`, whose
+/// iteration order differs between instances, so the hash must not depend on
+/// it (the real-VM repair returned 412 "preflight is stale" between dry-run
+/// and execute once snapshots carried service accounts).
+#[test]
+fn test_repair_task_id_and_retry_fingerprint_ignore_claim_map_order() {
+    let forward = service_account_item_with_claims(&["accessKey", "exp", "parent", "sa-policy", "sub", "tenant"]);
+    let backward = service_account_item_with_claims(&["tenant", "sub", "sa-policy", "parent", "exp", "accessKey"]);
+
+    let canonical = canonical_json_vec(&forward).expect("canonical json");
+    let text = String::from_utf8(canonical).expect("utf8");
+    let positions: Vec<usize> = [
+        "\"accessKey\"",
+        "\"exp\"",
+        "\"parent\"",
+        "\"sa-policy\"",
+        "\"sub\"",
+        "\"tenant\"",
+    ]
+    .iter()
+    .map(|key| text.find(key).expect("claim key present"))
+    .collect();
+    assert!(
+        positions.windows(2).all(|pair| pair[0] < pair[1]),
+        "claim keys must serialize sorted: {text}"
+    );
+
+    assert_eq!(
+        SiteReplicationRepairTask::Iam(&forward).id().expect("id"),
+        SiteReplicationRepairTask::Iam(&backward).id().expect("id"),
+        "identical items must yield the same repair task id regardless of claim map order"
+    );
+    assert_eq!(
+        RetrySnapshot::Iam(vec![forward]).fingerprint().expect("fingerprint"),
+        RetrySnapshot::Iam(vec![backward]).fingerprint().expect("fingerprint"),
+        "identical snapshots must fingerprint equal regardless of claim map order"
+    );
+}

rustfs/src/site_replication/transport.rs

@@ -876,6 +876,14 @@ pub(crate) async fn broadcast_site_replication_json<T: Serialize>(path: &str, bo
     broadcast_site_replication_json_with_runtime(&runtime, path, body).await
 }
 
+/// PUT `body` to `path` on every remote peer of the runtime.
+///
+/// Every peer is attempted: one peer's failure — transport construction
+/// included — must not skip the peers that follow it in deployment-id order,
+/// or they silently miss the change with no retry record (backlog#2293). A
+/// success settles the peer/path's queued retry event, a failure enqueues one
+/// under the request `path` (so the drain classifies it as today), and the
+/// first error is returned once all peers were attempted.
 pub(crate) async fn broadcast_site_replication_json_with_runtime<T: Serialize>(
     runtime: &SiteReplicationRuntime,
     path: &str,
@@ -883,20 +891,30 @@ pub(crate) async fn broadcast_site_replication_json_with_runtime<T: Serialize>(
 ) -> S3Result<()> {
     let state = &runtime.state;
     let local_peer = &runtime.local_peer;
+    let mut first_error: Option<S3Error> = None;
 
     for peer in state.peers.values() {
         if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
             continue;
         }
 
-        let transport = PeerTransport::for_runtime_peer(peer).await?;
-        PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key)
-            .with_client(&transport.client)
-            .send_with_retry_event(peer, &runtime.service_account_secret_key, body)
-            .await?;
+        let sent = match PeerTransport::for_runtime_peer(peer).await {
+            Ok(transport) => PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key)
+                .with_client(&transport.client)
+                .send_with_retry_event(peer, &runtime.service_account_secret_key, body)
+                .await
+                .map(|_| ()),
+            Err(err) => {
+                enqueue_site_replication_retry_event(peer, path, &err).await;
+                Err(err)
+            }
+        };
+        if let Err(err) = sent {
+            first_error.get_or_insert(err);
+        }
     }
 
-    Ok(())
+    first_error.map_or(Ok(()), Err)
 }
 
 pub(crate) fn parse_endpoint_refresh_status(peer: &PeerInfo, body: &[u8]) -> S3Result<()> {