chore: integrate current main for namespace target validation

rustfs/fixtures/on_demand_migration/source_config_e2a.rs

@@ -0,0 +1,80 @@
+// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01.
+// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted.
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+const REDACTED: &str = "REDACTED";
+
+/// The external S3-compatible source bucket.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SourceConfig {
+    pub provider: Provider,
+    /// `http(s)://host[:port]` with no path or query. Optional only for
+    /// [`Provider::Aws`], where it is derived from `region`.
+    #[serde(default)]
+    pub endpoint: Option<String>,
+    pub region: String,
+    pub bucket: String,
+    #[serde(default)]
+    pub path_style: PathStyle,
+    /// `None` means anonymous access to a public source bucket.
+    #[serde(default)]
+    pub credentials: Option<SourceCredentials>,
+    #[serde(default)]
+    pub tls: TlsConfig,
+}
+
+/// Source vendor family. `azure` is deliberately absent from this version.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Provider {
+    /// Generic S3-compatible endpoint.
+    S3,
+    Aws,
+    Minio,
+    Rustfs,
+    R2,
+    /// GCS XML interoperability API with HMAC keys.
+    Gcs,
+}
+
+/// Bucket addressing style. `auto` is resolved by the source client builder.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum PathStyle {
+    #[default]
+    Auto,
+    Path,
+    Virtual,
+}
+
+/// Static credentials for the source. `Debug` never prints the secret or
+/// the session token.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SourceCredentials {
+    pub access_key: String,
+    pub secret_key: String,
+    #[serde(default)]
+    pub session_token: Option<String>,
+}
+
+impl fmt::Debug for SourceCredentials {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SourceCredentials")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &REDACTED)
+            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED))
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TlsConfig {
+    #[serde(default)]
+    pub skip_verify: bool,
+    #[serde(default)]
+    pub ca_cert_pem: Option<String>,
+}

rustfs/src/admin/handlers/heal.rs

@@ -1636,6 +1636,44 @@ mod tests {
         assert!(executed.load(Ordering::SeqCst));
     }
 
+    #[tokio::test]
+    async fn heal_start_retry_preflight_failures_do_not_create_request_identities() {
+        let hip = HealInitParams {
+            bucket: "bucket".to_string(),
+            ..Default::default()
+        };
+        let mut request_ids = Vec::new();
+        for attempt in 0..3 {
+            let executed_ids = &mut request_ids;
+            let request_params = &hip;
+            let result = execute_after_heal_control_capability(
+                || async {
+                    if attempt < 2 {
+                        Err(super::cluster_heal_control_unavailable("test_capability_failure"))
+                    } else {
+                        Ok(())
+                    }
+                },
+                || async move {
+                    let request = build_heal_channel_request(request_params);
+                    executed_ids.push(request.id);
+                    Ok(())
+                },
+            )
+            .await;
+            if attempt < 2 {
+                assert!(result.is_err(), "failed capability checks must not start a heal");
+                assert!(
+                    request_ids.is_empty(),
+                    "preflight failure must precede request construction and admission"
+                );
+            } else {
+                result.expect("restored capabilities allow the first execution");
+                assert_eq!(request_ids.len(), 1);
+            }
+        }
+    }
+
     #[test]
     fn replacement_recovery_status_response_reports_cluster_proof() {
         let local = replacement_snapshot("11111111-1111-4111-8111-111111111111");
@@ -1743,6 +1781,21 @@ mod tests {
         assert!(decoded.is_none());
     }
 
+    #[test]
+    fn heal_start_retry_conflicts_keep_actionable_public_reasons() {
+        for (reason, label) in [
+            (HealAdmissionDropReason::AlreadyRunning, "already_running"),
+            (HealAdmissionDropReason::OverlappingPaths, "overlapping_paths"),
+        ] {
+            let error = reject_heal_admission(HealAdmissionResult::Dropped(reason));
+            assert_eq!(error.code(), &S3ErrorCode::OperationAborted);
+            assert!(
+                error.to_string().contains(label),
+                "the caller must distinguish conflicts from transient coordination failure"
+            );
+        }
+    }
+
     #[test]
     fn test_reject_heal_admission_preserves_retry_semantics() {
         for admission in [

rustfs/src/admin/handlers/user.rs

@@ -1131,6 +1131,7 @@ impl Operation for ImportIam {
                         expiration: req.expiration,
                         allow_site_replicator_account: false,
                         claims: Some(req.claims),
+                        status: None,
                     };
 
                     let groups = if req.groups.is_empty() { None } else { Some(req.groups) };

rustfs/src/admin/storage_api.rs

@@ -295,6 +295,8 @@ pub(crate) mod remote_s3_client {
 }
 
 pub(crate) mod metadata_sys {
+    #[cfg(test)]
+    pub(crate) use super::ecstore_bucket::metadata_sys::ConfigWriteLockProbe;
     use std::sync::Arc;
 
     use rustfs_policy::policy::BucketPolicy;
@@ -312,6 +314,7 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::get(bucket).await
     }
 
+    #[cfg(test)]
     pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
         crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await
     }
@@ -332,6 +335,25 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await
     }
 
+    /// [`update_if_incarnation`] stamping the config with a replicated edit's
+    /// source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn update_if_incarnation_at(
+        bucket: &str,
+        config_file: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: uuid::Uuid,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::update_if_incarnation_at(
+            bucket,
+            config_file,
+            data,
+            expected_incarnation_id,
+            updated_at,
+        )
+        .await
+    }
+
     pub(crate) async fn update_quota_if_incarnation(
         bucket: &str,
         data: Vec<u8>,
@@ -341,6 +363,25 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await
     }
 
+    /// [`update_quota_if_incarnation`] stamping the quota with a replicated
+    /// edit's source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn update_quota_if_incarnation_at(
+        bucket: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: uuid::Uuid,
+        proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::update_quota_if_incarnation_at(
+            bucket,
+            data,
+            expected_incarnation_id,
+            proof,
+            updated_at,
+        )
+        .await
+    }
+
     pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> {
         super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await
     }
@@ -395,6 +436,18 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::delete_if_incarnation(bucket, config_file, expected_incarnation_id).await
     }
 
+    /// [`delete_if_incarnation`] stamping the cleared config with a replicated
+    /// deletion's source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn delete_if_incarnation_at(
+        bucket: &str,
+        config_file: &str,
+        expected_incarnation_id: uuid::Uuid,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::delete_if_incarnation_at(bucket, config_file, expected_incarnation_id, updated_at)
+            .await
+    }
+
     pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await
     }
@@ -667,7 +720,7 @@ pub(crate) mod replication {
 }
 
 pub(crate) mod target {
-    pub(crate) use super::ecstore_bucket::target::duration_from_secs_or_nanos;
+    pub(crate) use super::ecstore_bucket::target::{ARN, duration_from_secs_or_nanos};
     pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget;
     pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType;
     pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets;

rustfs/src/app/bucket_usecase.rs

@@ -2783,42 +2783,49 @@ impl DefaultBucketUsecase {
         } else {
             (None, None)
         };
-        let (object_infos, degraded) = match source_state {
-            Some(state) => {
+        let (object_infos, degraded) = match (source_state, merged_token.as_ref()) {
+            (None, Some(token)) if params.max_keys == 0 => {
+                // No source was consulted, so retain every unconsumed side and
+                // the original wire format without spending its progress budget.
+                let is_truncated = !token.local_done || !token.source_done;
+                (
+                    StorageListObjectsV2Info {
+                        is_truncated,
+                        next_continuation_token: params.decoded_continuation_token.clone().filter(|_| is_truncated),
+                        ..Default::default()
+                    },
+                    false,
+                )
+            }
+            (None, None) => {
+                let infos = store
+                    .list_objects_v2(
+                        &bucket,
+                        &params.prefix,
+                        params.decoded_continuation_token.clone(),
+                        params.delimiter.clone(),
+                        params.max_keys,
+                        fetch_owner.unwrap_or_default(),
+                        params.start_after_for_query.clone(),
+                        incl_deleted,
+                    )
+                    .await
+                    .map_err(ApiError::from)?;
+                (infos, false)
+            }
+            (state, token) => {
                 let outcome = list_through::merged_list_objects_v2(
                     &store,
-                    &state,
+                    state.as_ref(),
                     &bucket,
                     &params,
                     fetch_owner.unwrap_or_default(),
                     incl_deleted,
-                    merged_token.as_ref(),
+                    token,
                 )
                 .await?;
                 (outcome.info, outcome.degraded)
             }
-            None => {
-                let cursor = list_through::local_cursor(params.decoded_continuation_token.as_deref(), merged_token.as_ref());
-                match cursor {
-                    list_through::LocalListCursor::Exhausted => (StorageListObjectsV2Info::default(), false),
-                    list_through::LocalListCursor::Token(token) => {
-                        let infos = store
-                            .list_objects_v2(
-                                &bucket,
-                                &params.prefix,
-                                token,
-                                params.delimiter.clone(),
-                                params.max_keys,
-                                fetch_owner.unwrap_or_default(),
-                                params.start_after_for_query.clone(),
-                                incl_deleted,
-                            )
-                            .await
-                            .map_err(ApiError::from)?;
-                        (infos, false)
-                    }
-                }
-            }
         };
 
         let output = build_list_objects_v2_output(

rustfs/src/app/storage_api.rs

@@ -718,6 +718,8 @@ pub(crate) mod bucket {
                     delete_marker_version_id: None,
                     delete_marker: false,
                     delete_marker_mtime: None,
+                    target_delete_marker_version_ids: Default::default(),
+                    target_delete_marker_version_ids_corrupt: false,
                     target_arns,
                     force_delete_id: Some(operation_id),
                     force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)),

rustfs/src/on_demand_migration/azure.rs

@@ -163,6 +163,32 @@ impl AzureSourceBackend {
         Ok(request)
     }
 
+    /// A missing blob is distinct from a missing container or version. Only
+    /// object reads may use BlobNotFound as positive evidence of absence.
+    async fn send_object_request(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
+        let is_head = request.method() == Method::HEAD;
+        let versioned = request
+            .url()
+            .query_pairs()
+            .any(|(name, _)| name.eq_ignore_ascii_case("versionid") || name.eq_ignore_ascii_case("snapshot"));
+        let response = self.http.execute(request).await?;
+        if response.status() == http::StatusCode::NOT_FOUND && !versioned {
+            match header(response.headers(), HEADER_ERROR_CODE) {
+                Some("BlobNotFound") => return Err(SourceError::NotFound),
+                None | Some("ResourceNotFound") if is_head => {
+                    // HEAD may omit an error code. One successful container
+                    // probe proves key absence; a failed probe keeps its error.
+                    // These are two independently timed requests, not one deadline.
+                    drop(response);
+                    self.probe().await?;
+                    return Err(SourceError::NotFound);
+                }
+                _ => {}
+            }
+        }
+        NativeHttp::check_response(response, Some(HEADER_ERROR_CODE))
+    }
+
     /// Shared mapping for Get Blob and Get Blob Properties.
     fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
         // A customer-provided key means the service holds ciphertext it cannot
@@ -189,7 +215,7 @@ impl AzureSourceBackend {
 impl SourceBackend for AzureSourceBackend {
     async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
         let request = self.request(Method::HEAD, self.blob_url(key)?, HeaderMap::new())?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.send_object_request(request).await?;
         Self::head_from_response(response.headers())
     }
 
@@ -202,7 +228,7 @@ impl SourceBackend for AzureSourceBackend {
             );
         }
         let request = self.request(Method::GET, self.blob_url(key)?, headers)?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.send_object_request(request).await?;
         let head = Self::head_from_response(response.headers())?;
         let content_range = header(response.headers(), "content-range").map(str::to_string);
         Ok(SourceGet {
@@ -240,7 +266,7 @@ impl SourceBackend for AzureSourceBackend {
         }
 
         let request = self.request(Method::GET, url, HeaderMap::new())?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
         let body = read_text(response, MAX_XML_BYTES).await?;
         let listing = parse_list_blobs(&body)?;
 
@@ -256,7 +282,7 @@ impl SourceBackend for AzureSourceBackend {
         let mut url = self.blob_url(key)?;
         url.query_pairs_mut().append_pair("comp", "tags");
         let request = self.request(Method::GET, url, HeaderMap::new())?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
         let body = read_text(response, MAX_XML_BYTES).await?;
         parse_blob_tags(&body)
     }
@@ -265,7 +291,7 @@ impl SourceBackend for AzureSourceBackend {
         let mut url = self.container_url()?;
         url.query_pairs_mut().append_pair("restype", "container");
         let request = self.request(Method::HEAD, url, HeaderMap::new())?;
-        self.http.send(request, HEADER_ERROR_CODE).await?;
+        self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
         Ok(())
     }
 }
@@ -343,9 +369,9 @@ struct AzureListing {
 
 #[derive(Default)]
 struct BlobEntry {
-    name: String,
+    name: Option<String>,
     etag: Option<String>,
-    size: u64,
+    size: Option<u64>,
     last_modified: Option<std::time::SystemTime>,
     access_tier: Option<String>,
 }
@@ -358,6 +384,7 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
     let mut next_marker = None;
     let mut blob: Option<BlobEntry> = None;
     let mut in_blob_prefix = false;
+    let mut blob_prefix: Option<String> = None;
     // Open container elements. quick-xml reports a truncated document as a
     // plain end of input, so a non-zero depth at EOF is the only signal that
     // the page was cut short and must not be read as a complete listing.
@@ -367,6 +394,9 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
         match reader.read_event() {
             Ok(Event::Start(start)) => {
                 let name = local_name(start.name().as_ref());
+                if matches!(name.as_str(), "blob" | "blobprefix") && (blob.is_some() || in_blob_prefix) {
+                    return Err(SourceError::Other("source listing entries must not be nested".to_string()));
+                }
                 match name.as_str() {
                     "blob" => {
                         depth += 1;
@@ -385,27 +415,35 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
                         } else {
                             text
                         };
-                        apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix);
+                        apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?;
                     }
                 }
             }
             Ok(Event::Empty(empty)) => {
                 let name = local_name(empty.name().as_ref());
+                if matches!(name.as_str(), "blob" | "blobprefix") {
+                    return Err(SourceError::Other("source listing entry has no name".to_string()));
+                }
                 let text = if name == "name" {
                     decode_list_name(&empty, String::new())?
                 } else {
                     String::new()
                 };
-                apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix);
+                apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?;
             }
             Ok(Event::End(end)) => match local_name(end.name().as_ref()).as_str() {
                 "blob" => {
                     depth = depth.saturating_sub(1);
                     if let Some(entry) = blob.take() {
                         objects.push(SourceObject {
-                            key: entry.name,
+                            key: entry
+                                .name
+                                .filter(|name| !name.is_empty())
+                                .ok_or_else(|| SourceError::Other("source listing object has no name".to_string()))?,
                             etag: entry.etag,
-                            size: entry.size,
+                            size: entry
+                                .size
+                                .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?,
                             last_modified: entry.last_modified,
                             storage_class: entry.access_tier,
                             // Azure ETags carry no part count; the listing
@@ -417,6 +455,12 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
                 "blobprefix" => {
                     depth = depth.saturating_sub(1);
                     in_blob_prefix = false;
+                    prefixes.push(
+                        blob_prefix
+                            .take()
+                            .filter(|name| !name.is_empty())
+                            .ok_or_else(|| SourceError::Other("source listing prefix has no name".to_string()))?,
+                    );
                 }
                 "properties" | "blobs" | "enumerationresults" => depth = depth.saturating_sub(1),
                 _ => {}
@@ -478,16 +522,22 @@ fn apply_list_field(
     name: &str,
     text: String,
     blob: &mut Option<BlobEntry>,
-    prefixes: &mut Vec<String>,
+    blob_prefix: &mut Option<String>,
     next_marker: &mut Option<String>,
     in_blob_prefix: bool,
-) {
+) -> Result<(), SourceError> {
     match name {
         "name" => {
             if in_blob_prefix {
-                prefixes.push(text);
+                if blob_prefix.is_some() {
+                    return Err(SourceError::Other("source listing prefix has duplicate names".to_string()));
+                }
+                *blob_prefix = Some(text);
             } else if let Some(entry) = blob.as_mut() {
-                entry.name = text;
+                if entry.name.is_some() {
+                    return Err(SourceError::Other("source listing object has duplicate names".to_string()));
+                }
+                entry.name = Some(text);
             }
         }
         "nextmarker" => *next_marker = Some(text),
@@ -498,7 +548,14 @@ fn apply_list_field(
         }
         "content-length" => {
             if let Some(entry) = blob.as_mut() {
-                entry.size = text.trim().parse().unwrap_or(0);
+                if entry.size.is_some() {
+                    return Err(SourceError::Other("source listing object has duplicate sizes".to_string()));
+                }
+                entry.size = Some(
+                    text.trim()
+                        .parse()
+                        .map_err(|_| SourceError::Other("source listing object has no valid size".to_string()))?,
+                );
             }
         }
         "last-modified" => {
@@ -513,6 +570,7 @@ fn apply_list_field(
         }
         _ => {}
     }
+    Ok(())
 }
 
 /// Parses a `Get Blob Tags` response.
@@ -599,7 +657,7 @@ mod tests {
     use super::*;
     use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
     use crate::on_demand_migration::source_client::SourceError;
-    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
+    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server};
 
     const LIST_PAGE: &str = r#"<?xml version="1.0" encoding="utf-8"?>
 <EnumerationResults ServiceEndpoint="https://acct.blob.core.windows.net/" ContainerName="legacy">
@@ -769,6 +827,168 @@ mod tests {
         assert!(parse_blob_tags("<Tags><TagSet>").is_err(), "a truncated tag set must fail");
     }
 
+    #[tokio::test]
+    async fn native_listing_rejects_missing_or_invalid_required_object_fields() {
+        for entry in [
+            "<Blob />",
+            "<Blob><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name /><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name>broken</Name></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length /></Properties></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length>-1</Content-Length></Properties></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length>18446744073709551616</Content-Length></Properties></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length>not-a-size</Content-Length></Properties></Blob>",
+            "<BlobPrefix />",
+            "<BlobPrefix><Name /></BlobPrefix>",
+            "<BlobPrefix></BlobPrefix>",
+        ] {
+            // Reject the entire page even if a valid object precedes the bad
+            // entry, so callers cannot expose partial data or advance its cursor.
+            let body = format!(
+                "<EnumerationResults><Blobs><Blob><Name>valid</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker>next</NextMarker></EnumerationResults>"
+            );
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .list(&SourceListRequest {
+                    prefix: Some("dir/"),
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await
+                .expect_err("malformed object must reject the complete native page");
+            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
+            assert!(!err.is_retryable());
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2",
+                )],
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn native_listing_rejects_duplicate_fields_and_nested_entries() {
+        for entry in [
+            "<Blob><Name>a</Name><Name>b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name /><Name>b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length><Content-Length>2</Content-Length></Properties></Blob>",
+            "<BlobPrefix><Name>a/</Name><Name>b/</Name></BlobPrefix>",
+            "<BlobPrefix><Name /><Name>b/</Name></BlobPrefix>",
+            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length></Properties><Blob><Name>b</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></Blob>",
+            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length></Properties><BlobPrefix><Name>b/</Name></BlobPrefix></Blob>",
+            "<BlobPrefix><Name>a/</Name><Blob><Name>b</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></BlobPrefix>",
+            "<BlobPrefix><Name>a/</Name><BlobPrefix><Name>b/</Name></BlobPrefix></BlobPrefix>",
+        ] {
+            let body = format!(
+                "<EnumerationResults><Blobs><Blob><Name>valid</Name><Properties><Content-Length>0</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker>next</NextMarker></EnumerationResults>"
+            );
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let result = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .list(&SourceListRequest {
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await;
+            let err = result.expect_err("ambiguous entries must reject the entire page and its cursor");
+            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
+            assert!(!err.is_retryable(), "{entry}: {err:?}");
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2",
+                )],
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() {
+        let body = "<EnumerationResults><Blobs><Blob><Name>目录/空 &amp; file</Name><Properties><Content-Length>0</Content-Length></Properties></Blob><BlobPrefix><Name>目录/子/</Name></BlobPrefix></Blobs><NextMarker>opaque+/=</NextMarker></EnumerationResults>";
+        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+        let page = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+            .list(&SourceListRequest {
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("valid native page");
+        assert_eq!(page.objects.len(), 1);
+        assert_eq!(page.objects[0].key, "目录/空 & file");
+        assert_eq!(page.objects[0].size, 0);
+        assert_eq!(page.common_prefixes, ["目录/子/"]);
+        assert!(page.is_truncated);
+        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
+        assert_requests(&recorded, &[("GET", "/legacy?restype=container&comp=list&maxresults=2")]);
+    }
+
+    #[tokio::test]
+    async fn encoded_listing_preserves_required_field_and_entry_validation() {
+        for (entry, expected_error) in [
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name></Blob>"#,
+                "source listing object has no valid size",
+            ),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Properties><Content-Length>-1</Content-Length></Properties></Blob>"#,
+                "source listing object has no valid size",
+            ),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Name>a/b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>"#,
+                "source listing object has duplicate names",
+            ),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Properties><Content-Length>1</Content-Length><Content-Length>2</Content-Length></Properties></Blob>"#,
+                "source listing object has duplicate sizes",
+            ),
+            (
+                r#"<BlobPrefix><Name Encoded="true">a%2F</Name><Name>a/</Name></BlobPrefix>"#,
+                "source listing prefix has duplicate names",
+            ),
+            (r#"<BlobPrefix><Name Encoded="true" /></BlobPrefix>"#, "source listing prefix has no name"),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Properties><Content-Length>1</Content-Length></Properties><Blob><Name Encoded="true">c%2Fd</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></Blob>"#,
+                "source listing entries must not be nested",
+            ),
+            (
+                r#"<BlobPrefix><Name Encoded="true">a%2F</Name><BlobPrefix><Name Encoded="true">b%2F</Name></BlobPrefix></BlobPrefix>"#,
+                "source listing entries must not be nested",
+            ),
+        ] {
+            let body = format!(
+                r#"<EnumerationResults><Blobs><Blob><Name Encoded="true">valid%252F</Name><Properties><Content-Length>0</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker Encoded="true">opaque%2B+marker</NextMarker></EnumerationResults>"#
+            );
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .list(&SourceListRequest {
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque%2B+marker"),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await
+                .expect_err("encoded names cannot bypass whole-page validation");
+            assert!(!err.is_retryable(), "{entry}: {err:?}");
+            let SourceError::Other(message) = err else {
+                panic!("wrong error class for {entry}: {err:?}");
+            };
+            assert_eq!(message, expected_error, "{entry}");
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%252B%2Bmarker&maxresults=2",
+                )],
+            );
+        }
+    }
+
     #[test]
     fn blob_tags_parse_into_the_shared_tag_map() {
         let tags = parse_blob_tags(TAGS).expect("tags should parse");
@@ -917,6 +1137,18 @@ mod tests {
         assert!(head.sse.is_none());
     }
 
+    #[tokio::test]
+    async fn dot_segment_keys_fail_before_any_source_request() {
+        let (endpoint, recorded) = scripted_server(Vec::new()).await;
+        let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
+        for key in [".", "..", "dir/./key", "dir/../key", "\u{fffe}/../key"] {
+            assert!(matches!(backend.head(key).await, Err(SourceError::Unsupported(_))), "HEAD {key:?}");
+            assert!(matches!(backend.get(key, None).await, Err(SourceError::Unsupported(_))), "GET {key:?}");
+            assert!(matches!(backend.tagging(key).await, Err(SourceError::Unsupported(_))), "tags {key:?}");
+        }
+        assert!(recorded.lock().expect("recorder lock").is_empty());
+    }
+
     #[tokio::test]
     async fn sas_credentials_travel_in_the_query_and_never_sign() {
         let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, blob_headers(), String::new())]).await;
@@ -1239,6 +1471,184 @@ mod tests {
         ]
     }
 
+    #[tokio::test]
+    async fn object_not_found_requires_provider_evidence_or_one_successful_head_probe() {
+        for method in [Method::HEAD, Method::GET] {
+            for (status, code, expected) in [
+                (404, Some("BlobNotFound"), "not_found"),
+                (403, Some("BlobNotFound"), "access_denied"),
+                (404, Some("ContainerNotFound"), "other"),
+                (404, Some("BlobVersionNotFound"), "other"),
+                (404, Some("UnrecognizedError"), "other"),
+                (404, None, if method == Method::HEAD { "not_found" } else { "other" }),
+                (404, Some("ResourceNotFound"), if method == Method::HEAD { "not_found" } else { "other" }),
+            ] {
+                let probes = method == Method::HEAD && status == 404 && matches!(code, None | Some("ResourceNotFound"));
+                let headers = code
+                    .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())])
+                    .unwrap_or_default();
+                let mut responses = vec![ScriptedResponse::new(status, headers, "untrusted-error-body".to_string())];
+                if probes {
+                    responses.push(ScriptedResponse::new(200, Vec::new(), String::new()));
+                }
+                let (endpoint, recorded) = scripted_server(responses).await;
+                let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
+                let result = if method == Method::HEAD {
+                    backend.head("missing").await.map(|_| ())
+                } else {
+                    backend.get("missing", None).await.map(|_| ())
+                };
+                let err = result.expect_err("object error must remain an error");
+                assert_eq!(err.class_label(), expected, "{method} {status} {code:?}: {err:?}");
+                assert!(!err.is_retryable(), "{err:?}");
+                assert!(!err.to_string().contains("untrusted-error-body"));
+                let mut requests = vec![(method.as_str(), "/legacy/missing")];
+                if probes {
+                    requests.push(("HEAD", "/legacy?restype=container"));
+                }
+                assert_requests(&recorded, &requests);
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn s3_not_found_alias_never_proves_native_object_absence() {
+        for selector in [None, Some("versionid"), Some("snapshot")] {
+            for operation in ["head", "get", "list", "tags", "probe"] {
+                if selector.is_some() && !matches!(operation, "head" | "get") {
+                    continue;
+                }
+                for (status, expected, retryable) in [
+                    (403, "access_denied", false),
+                    (404, "other", false),
+                    (416, "other", false),
+                    (500, "server_error", true),
+                ] {
+                    let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                        status,
+                        vec![(HEADER_ERROR_CODE, "NoSuchKey".to_string())],
+                        "untrusted-error-body".to_string(),
+                    )])
+                    .await;
+                    let credential = selector.map_or_else(
+                        || Credential::SharedKey(vec![7_u8; 32]),
+                        |selector| Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]),
+                    );
+                    let backend = backend(&endpoint, credential);
+                    let result = match operation {
+                        "head" => backend.head("missing").await.map(|_| ()),
+                        "get" => backend.get("missing", None).await.map(|_| ()),
+                        "list" => backend.list(&SourceListRequest::default()).await.map(|_| ()),
+                        "tags" => backend.tagging("missing").await.map(|_| ()),
+                        "probe" => backend.probe().await,
+                        _ => unreachable!(),
+                    };
+                    let err = result.expect_err("an S3 error alias is not Azure absence evidence");
+                    assert_eq!(err.class_label(), expected, "{operation} {selector:?} HTTP {status}: {err:?}");
+                    assert_eq!(err.is_retryable(), retryable, "{operation} {selector:?} HTTP {status}: {err:?}");
+                    if status == 500 {
+                        assert!(matches!(err, SourceError::ServerError(500)));
+                    }
+                    assert!(!err.to_string().contains("untrusted-error-body"));
+                    let (method, mut target) = match operation {
+                        "head" => ("HEAD", "/legacy/missing".to_string()),
+                        "get" => ("GET", "/legacy/missing".to_string()),
+                        "list" => ("GET", "/legacy?restype=container&comp=list".to_string()),
+                        "tags" => ("GET", "/legacy/missing?comp=tags".to_string()),
+                        "probe" => ("HEAD", "/legacy?restype=container".to_string()),
+                        _ => unreachable!(),
+                    };
+                    if let Some(selector) = selector {
+                        target.push_str(&format!("?{selector}=old-version"));
+                    }
+                    assert_requests(&recorded, &[(method, target.as_str())]);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn ambiguous_head_preserves_the_container_probe_failure() {
+        for (status, expected, retryable) in [
+            (403, "access_denied", false),
+            (404, "other", false),
+            (429, "throttled", true),
+            (500, "server_error", true),
+            (503, "throttled", true),
+        ] {
+            let (endpoint, recorded) = scripted_server(vec![
+                ScriptedResponse::new(404, Vec::new(), String::new()),
+                // A BlobNotFound header on a container request cannot prove
+                // that the object is missing, regardless of this status.
+                ScriptedResponse::new(status, vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())], String::new()),
+            ])
+            .await;
+            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .head("missing")
+                .await
+                .expect_err("failed probe must not become object absence");
+            assert_eq!(err.class_label(), expected, "probe {status}: {err:?}");
+            assert_eq!(err.is_retryable(), retryable, "probe {status}: {err:?}");
+            if status == 500 {
+                assert!(matches!(err, SourceError::ServerError(500)));
+            }
+            assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("HEAD", "/legacy?restype=container")]);
+        }
+    }
+
+    #[tokio::test]
+    async fn version_and_snapshot_absence_are_not_missing_current_blobs() {
+        for selector in ["versionid", "snapshot"] {
+            for code in [None, Some("BlobNotFound"), Some("ResourceNotFound")] {
+                for method in [Method::HEAD, Method::GET] {
+                    let headers = code
+                        .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())])
+                        .unwrap_or_default();
+                    let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(404, headers, String::new())]).await;
+                    let backend = backend(&endpoint, Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]));
+                    let result = if method == Method::HEAD {
+                        backend.head("object").await.map(|_| ())
+                    } else {
+                        backend.get("object", None).await.map(|_| ())
+                    };
+                    let err = result.expect_err("missing selected version must remain a source error");
+                    assert!(matches!(err, SourceError::Other(_)), "{method} {selector} {code:?}: {err:?}");
+                    assert_requests(&recorded, &[(method.as_str(), &format!("/legacy/object?{selector}=old-version"))]);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn blob_not_found_header_is_not_object_absence_for_list_or_tags() {
+        for tags in [false, true] {
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                404,
+                vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())],
+                String::new(),
+            )])
+            .await;
+            let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
+            let result = if tags {
+                backend.tagging("missing").await.map(|_| ())
+            } else {
+                backend.list(&SourceListRequest::default()).await.map(|_| ())
+            };
+            assert!(matches!(result, Err(SourceError::Other(_))), "tags={tags}: {result:?}");
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    if tags {
+                        "/legacy/missing?comp=tags"
+                    } else {
+                        "/legacy?restype=container&comp=list"
+                    },
+                )],
+            );
+        }
+    }
+
     #[tokio::test]
     async fn azure_backend_satisfies_the_shared_backend_contract() {
         let mut ranged = contract_blob_headers();
@@ -1246,7 +1656,7 @@ mod tests {
         // A HEAD reports the object size with no body, exactly as Azure does.
         let mut head_only = contract_blob_headers();
         head_only.push(("Content-Length", "5".to_string()));
-        let (endpoint, _) = scripted_server(vec![
+        let (endpoint, recorded) = scripted_server(vec![
             ScriptedResponse::new(200, head_only, String::new()),
             ScriptedResponse::new(200, contract_blob_headers(), "hello".to_string()),
             ScriptedResponse::new(206, ranged, "ell".to_string()),
@@ -1276,6 +1686,23 @@ mod tests {
             },
         )
         .await;
+        assert_requests(
+            &recorded,
+            &[
+                ("HEAD", "/legacy/dir/a.txt"),
+                ("GET", "/legacy/dir/a.txt"),
+                ("GET", "/legacy/dir/a.txt"),
+                ("GET", "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&maxresults=2"),
+                (
+                    "GET",
+                    "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=cursor-1&maxresults=2",
+                ),
+                ("GET", "/legacy/dir/a.txt?comp=tags"),
+                ("HEAD", "/legacy?restype=container"),
+                ("HEAD", "/legacy/missing"),
+                ("HEAD", "/legacy/secret"),
+            ],
+        );
     }
 
     #[tokio::test]

rustfs/src/on_demand_migration/config.rs

@@ -106,12 +106,12 @@ pub struct SourceConfig {
     pub tls: TlsConfig,
     /// Required for [`Provider::Azure`] and rejected for every other
     /// provider.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub azure: Option<AzureSourceConfig>,
     /// Required for [`Provider::GcsNative`] and rejected for every other
     /// provider. [`Provider::Gcs`] keeps using `credentials` because it
     /// speaks the S3 interoperability API.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub gcs: Option<GcsSourceConfig>,
 }
 
@@ -808,6 +808,10 @@ impl EndpointKey {
 mod tests {
     use super::*;
 
+    mod before_native_sources {
+        include!("../../fixtures/on_demand_migration/source_config_e2a.rs");
+    }
+
     const FULL_JSON: &str = r#"{
   "version": 1,
   "enabled": true,
@@ -878,6 +882,32 @@ mod tests {
         assert_eq!(minimal.policy.source_timeout.first_byte_ms, 15_000);
     }
 
+    #[test]
+    fn s3_config_writes_remain_readable_by_the_strict_pre_native_reader() {
+        // FULL_JSON is the complete config fixture already present in e2a921bc.
+        for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] {
+            let mut old_wire: serde_json::Value = serde_json::from_str(FULL_JSON).expect("historical config fixture");
+            old_wire["source"]["provider"] = provider.into();
+            let config = OnDemandMigrationConfig::from_json(&serde_json::to_vec(&old_wire).expect("historical wire"))
+                .expect("current reader accepts the historical source");
+            let wire = config.to_json().expect("persist current config");
+            let actual: serde_json::Value = serde_json::from_slice(&wire).expect("persisted config JSON");
+            let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone())
+                .expect("an existing S3 source must remain readable by the strict e2a source consumer");
+            assert_eq!(serde_json::to_value(old_source).expect("old reader wire"), old_wire["source"]);
+            assert_eq!(actual, old_wire, "provider={provider}: no existing config field or value may change");
+
+            for field in ["azure", "gcs"] {
+                let mut rejected = old_wire["source"].clone();
+                rejected[field] = serde_json::Value::Null;
+                assert!(
+                    serde_json::from_value::<before_native_sources::SourceConfig>(rejected).is_err(),
+                    "the frozen old reader must reject {field}, even when null"
+                );
+            }
+        }
+    }
+
     #[test]
     fn unknown_fields_are_rejected_at_every_level() {
         for (label, json) in [
@@ -1080,6 +1110,19 @@ mod tests {
         for cfg in [azure_cfg(), gcs_native_cfg()] {
             let json = cfg.to_json().expect("config must serialize");
             assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg);
+            let wire: serde_json::Value = serde_json::from_slice(&json).expect("native config JSON");
+            let (present, absent, expected) = match cfg.source.provider {
+                Provider::Azure => ("azure", "gcs", serde_json::to_value(&cfg.source.azure).expect("Azure block")),
+                Provider::GcsNative => ("gcs", "azure", serde_json::to_value(&cfg.source.gcs).expect("GCS block")),
+                _ => unreachable!("native fixture"),
+            };
+            assert!(expected.is_object(), "native credentials must be present");
+            assert_eq!(wire["source"][present], expected);
+            assert!(wire["source"].get(absent).is_none());
+            assert!(
+                serde_json::from_value::<before_native_sources::SourceConfig>(wire["source"].clone()).is_err(),
+                "native providers still require upgraded readers"
+            );
         }
         // The wire labels are part of the admin contract.
         assert!(

rustfs/src/on_demand_migration/gcs.rs

@@ -55,10 +55,6 @@ use url::Url;
 /// Read-only object scope: this backend never writes to the source.
 const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
 const METADATA_PREFIX: &str = "x-goog-meta-";
-/// GCS reports its error code in the response body, not a header; the shared
-/// transport takes a header name, so it is given one that never matches and
-/// classification falls back to the status.
-const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
 /// One `objects.list` page is small; refuse an unbounded document.
 const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;
 
@@ -125,7 +121,7 @@ impl GcsNativeSourceBackend {
     }
 
     async fn send_object(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
-        match self.http.send_object(request, NO_ERROR_CODE_HEADER).await {
+        match self.http.send_object(request, None).await {
             Err(SourceError::NotFound) => {
                 // An XML object URL also returns 404 when its bucket is gone.
                 // Reuse the read-only listing probe before caching a key miss.
@@ -225,7 +221,7 @@ impl SourceBackend for GcsNativeSourceBackend {
         }
 
         let request = self.request(Method::GET, url, HeaderMap::new()).await?;
-        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
+        let response = self.http.send(request, None).await?;
         let body = read_text(response, MAX_JSON_BYTES).await?;
         parse_objects_list(&body)
     }
@@ -245,7 +241,7 @@ impl SourceBackend for GcsNativeSourceBackend {
         let mut url = self.objects_url()?;
         url.query_pairs_mut().append_pair("maxResults", "1");
         let request = self.request(Method::GET, url, HeaderMap::new()).await?;
-        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
+        let response = self.http.send(request, None).await?;
         read_text(response, MAX_JSON_BYTES)
             .await
             .and_then(|body| parse_objects_list(&body))?;
@@ -284,28 +280,38 @@ struct ListedObject {
 fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
     let listing: ObjectsList =
         serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
+    if listing.prefixes.iter().any(|prefix| prefix.is_empty()) {
+        return Err(SourceError::Other("source listing prefix has no name".to_string()));
+    }
     let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
     let objects = listing
         .items
         .into_iter()
         .map(|item| {
+            if item.name.is_empty() {
+                return Err(SourceError::Other("source listing object has no name".to_string()));
+            }
+            let size = item
+                .size
+                .and_then(|size| size.parse::<u64>().ok())
+                .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?;
             let etag = item
                 .md5_hash
                 .as_deref()
                 .and_then(base64_md5_to_hex)
                 .or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
                 .filter(|etag| !etag.is_empty());
-            SourceObject {
+            Ok(SourceObject {
                 key: item.name,
                 etag,
-                size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
+                size,
                 last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
                 storage_class: item.storage_class,
                 // GCS never encodes a part count in a digest or an ETag.
                 is_multipart_etag: false,
-            }
+            })
         })
-        .collect();
+        .collect::<Result<_, SourceError>>()?;
 
     Ok(SourcePage {
         objects,
@@ -319,7 +325,7 @@ fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
 mod tests {
     use super::*;
     use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
-    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
+    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server};
     use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
 
     const LIST_PAGE_ONE: &str = r#"{
@@ -372,6 +378,17 @@ mod tests {
         ]
     }
 
+    #[tokio::test]
+    async fn dot_segment_keys_fail_before_any_source_request() {
+        let (endpoint, recorded) = scripted_server(Vec::new()).await;
+        let backend = backend(&endpoint);
+        for key in [".", "..", "dir/./key", "dir/../key", "\u{fffe}/../key"] {
+            assert!(matches!(backend.head(key).await, Err(SourceError::Unsupported(_))), "HEAD {key:?}");
+            assert!(matches!(backend.get(key, None).await, Err(SourceError::Unsupported(_))), "GET {key:?}");
+        }
+        assert!(recorded.lock().expect("recorder lock").is_empty());
+    }
+
     #[test]
     fn objects_list_maps_items_prefixes_and_the_page_token() {
         let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse");
@@ -562,6 +579,197 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn native_listing_rejects_missing_or_invalid_required_object_fields() {
+        for entry in [
+            r#"{"size":"1"}"#,
+            r#"{"name":"","size":"1"}"#,
+            r#"{"name":"broken"}"#,
+            r#"{"name":"broken","size":null}"#,
+            r#"{"name":"broken","size":""}"#,
+            r#"{"name":"broken","size":"-1"}"#,
+            r#"{"name":"broken","size":"18446744073709551616"}"#,
+            r#"{"name":"broken","size":"not-a-size"}"#,
+            r#"{"name":"broken","size":1}"#,
+        ] {
+            let body = format!(r#"{{"items":[{{"name":"valid","size":"1"}},{entry}],"nextPageToken":"next"}}"#);
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let err = backend(&endpoint)
+                .list(&SourceListRequest {
+                    prefix: Some("dir/"),
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await
+                .expect_err("malformed object must reject the complete native page");
+            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
+            assert!(!err.is_retryable());
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/storage/v1/b/legacy/o?prefix=dir%2F&delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2",
+                )],
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn native_listing_rejects_empty_prefix_entries() {
+        for body in [
+            r#"{"items":[{"name":"valid","size":"1"}],"prefixes":[""],"nextPageToken":"next"}"#,
+            r#"{"prefixes":[""],"nextPageToken":"next"}"#,
+            r#"{"prefixes":["目录/子/",""],"nextPageToken":"next"}"#,
+        ] {
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+            let result = backend(&endpoint)
+                .list(&SourceListRequest {
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await;
+            let err = result.expect_err("an empty prefix must reject the entire page and its cursor");
+            assert!(matches!(err, SourceError::Other(_)), "{body}: {err:?}");
+            assert!(!err.is_retryable());
+            assert_requests(
+                &recorded,
+                &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2")],
+            );
+        }
+
+        let body = r#"{"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#;
+        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+        let page = backend(&endpoint)
+            .list(&SourceListRequest {
+                delimiter: Some("/"),
+                max_keys: 1,
+                ..Default::default()
+            })
+            .await
+            .expect("a valid prefix-only page must remain usable");
+        assert!(page.objects.is_empty());
+        assert_eq!(page.common_prefixes, ["目录/子/"]);
+        assert!(page.is_truncated);
+        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
+        assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&maxResults=1")]);
+    }
+
+    #[tokio::test]
+    async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() {
+        let body = r#"{"items":[{"name":"目录/空 & file","size":"0"}],"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#;
+        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+        let page = backend(&endpoint)
+            .list(&SourceListRequest {
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("valid native page");
+        assert_eq!(page.objects.len(), 1);
+        assert_eq!(page.objects[0].key, "目录/空 & file");
+        assert_eq!(page.objects[0].size, 0);
+        assert_eq!(page.common_prefixes, ["目录/子/"]);
+        assert!(page.is_truncated);
+        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
+        assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?maxResults=2")]);
+    }
+
+    #[tokio::test]
+    async fn missing_object_head_requires_one_successful_bucket_probe() {
+        for (status, body, expected, retryable) in [
+            (200, "{}", "not_found", false),
+            (403, "", "access_denied", false),
+            (404, "", "other", false),
+            (429, "", "throttled", true),
+            (500, "", "server_error", true),
+            (503, "", "throttled", true),
+            (200, "not JSON", "other", false),
+        ] {
+            let (endpoint, recorded) = scripted_server(vec![
+                ScriptedResponse::new(404, Vec::new(), String::new()),
+                ScriptedResponse::new(status, Vec::new(), body.to_string()),
+            ])
+            .await;
+            let err = backend(&endpoint).head("missing").await.expect_err("missing HEAD must fail");
+            assert_eq!(err.class_label(), expected, "probe {status} {body:?}: {err:?}");
+            assert_eq!(err.is_retryable(), retryable, "probe {status} {body:?}: {err:?}");
+            if status == 500 {
+                assert!(matches!(err, SourceError::ServerError(500)));
+            }
+            assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("GET", "/storage/v1/b/legacy/o?maxResults=1")]);
+        }
+    }
+
+    #[tokio::test]
+    async fn denied_object_reads_do_not_probe_or_become_object_absence() {
+        for method in [Method::HEAD, Method::GET] {
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                403,
+                vec![("x-goog-unused-error-code", "NoSuchKey".to_string())],
+                "untrusted-error-body".to_string(),
+            )])
+            .await;
+            let backend = backend(&endpoint);
+            let result = if method == Method::HEAD {
+                backend.head("missing").await.map(|_| ())
+            } else {
+                backend.get("missing", None).await.map(|_| ())
+            };
+            let err = result.expect_err("denied object read must remain a failure");
+            assert_eq!(err.class_label(), "access_denied");
+            assert!(!err.is_retryable());
+            assert!(!err.to_string().contains("untrusted-error-body"));
+            assert_requests(&recorded, &[(method.as_str(), "/legacy/missing")]);
+        }
+    }
+    #[tokio::test]
+    async fn non_object_errors_ignore_untrusted_error_code_headers() {
+        for probe in [false, true] {
+            for (status, expected, retryable) in [(403, "access_denied", false), (500, "server_error", true)] {
+                let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                    status,
+                    vec![("x-goog-unused-error-code", "NoSuchKey".to_string())],
+                    "untrusted-error-body".to_string(),
+                )])
+                .await;
+                let backend = backend(&endpoint);
+                let result = if probe {
+                    backend.probe().await
+                } else {
+                    backend
+                        .list(&SourceListRequest {
+                            max_keys: 2,
+                            ..Default::default()
+                        })
+                        .await
+                        .map(|_| ())
+                };
+                let err = result.expect_err("a synthetic provider header cannot change the source status");
+                assert_eq!(err.class_label(), expected, "probe={probe} status={status}: {err:?}");
+                assert_eq!(err.is_retryable(), retryable);
+                assert!(!err.to_string().contains("untrusted-error-body"));
+                if status == 500 {
+                    assert!(matches!(err, SourceError::ServerError(500)));
+                }
+                assert_requests(
+                    &recorded,
+                    &[(
+                        "GET",
+                        if probe {
+                            "/storage/v1/b/legacy/o?maxResults=1"
+                        } else {
+                            "/storage/v1/b/legacy/o?maxResults=2"
+                        },
+                    )],
+                );
+            }
+        }
+    }
+
     /// GCS states its error code in the response body, which this backend never
     /// reads, so every class must follow from the status alone. The classes are
     /// what the runtime acts on: only `NotFound` is negative-cached, and only a

rustfs/src/on_demand_migration/list_through.rs

@@ -94,13 +94,16 @@ pub struct MergePick {
 }
 
 /// The continuation-token envelope. Opaque to clients: it is serialized as
-/// framed JSON and then base64-encoded by the same helper as a local marker.
+/// JSON, optionally framed, then base64-encoded like a local marker.
 ///
 /// A `null` cursor with `done = false` means "list that side from the start";
 /// `done = true` means the side is finished and must not be listed again.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct ListThroughToken {
+    /// Transport framing observed by the decoder, never an envelope field.
+    #[serde(skip)]
+    pub framed: bool,
     /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
     pub t: String,
     pub v: u32,
@@ -127,6 +130,7 @@ pub struct ListThroughToken {
 impl ListThroughToken {
     fn new(local: SideCursor, source: SideCursor, last_key: Option<String>) -> Self {
         Self {
+            framed: false,
             t: LIST_THROUGH_TOKEN_TAG.to_string(),
             v: LIST_THROUGH_TOKEN_VERSION,
             local: local.token,
@@ -141,7 +145,12 @@ impl ListThroughToken {
     pub fn encode(&self) -> String {
         // The envelope is built here from owned strings, so serialization
         // cannot fail; the fallback keeps the signature infallible.
-        format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
+        let json = serde_json::to_string(self).unwrap_or_default();
+        if self.framed {
+            format!("{LIST_THROUGH_TOKEN_PREFIX}{json}")
+        } else {
+            json
+        }
     }
 }
 
@@ -165,16 +174,30 @@ pub enum ListThroughTokenError {
 
 /// Classifies an already base64-decoded continuation token.
 ///
-/// Only a framed JSON object is read as a merged token;
-/// anything else is a local marker, so a bucket that turns `list_through` off
-/// keeps paginating with the tokens it handed out. A token that *is* an
-/// envelope but was tampered with (unknown version, unknown field, truncated
-/// JSON) is an error, never a silent fallback.
+/// Framed envelopes and complete historical writer envelopes are merged tokens.
+/// Partial JSON-shaped keys remain local markers. A key identical to a complete
+/// historical envelope is inherently ambiguous and retains merged semantics.
+/// Recognized envelopes share the same version, count and field validation.
 pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
-    let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
-        return Ok(ListThroughCursor::Local(decoded.to_string()));
+    let (payload, framed) = match decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) {
+        Some(payload) => (payload, true),
+        None if decoded.starts_with('{') => (decoded, false),
+        None => return Ok(ListThroughCursor::Local(decoded.to_string())),
     };
-    let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
+    let value = match serde_json::from_str::<serde_json::Value>(payload) {
+        Ok(value) => value,
+        Err(_) if framed => return Err(ListThroughTokenError::Malformed),
+        Err(_) => return Ok(ListThroughCursor::Local(decoded.to_string())),
+    };
+    // RUSTFS_COMPAT_TODO(odm-list-bare-envelope): old writers issued bare JSON. Remove after all supported readers understand framing and outstanding bare listings have drained or explicitly restarted.
+    if !framed
+        && (value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG)
+            || ["v", "local", "local_done", "source", "source_done", "last_key"]
+                .iter()
+                .any(|field| value.get(field).is_none()))
+    {
+        return Ok(ListThroughCursor::Local(decoded.to_string()));
+    }
     if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
         return Err(ListThroughTokenError::Malformed);
     }
@@ -198,7 +221,10 @@ pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, Lis
         None => return Err(ListThroughTokenError::Malformed),
     }
     serde_json::from_value::<ListThroughToken>(value)
-        .map(|token| ListThroughCursor::Merged(Box::new(token)))
+        .map(|mut token| {
+            token.framed = framed;
+            ListThroughCursor::Merged(Box::new(token))
+        })
         .map_err(|_| ListThroughTokenError::Malformed)
 }
 
@@ -643,6 +669,126 @@ impl Default for SourceListRateLimiter {
     }
 }
 
+// Frozen framed-only codec from e1608fbd9ca934d157b5de46c80b4393f2dd3dd6.
+// Keep its own DTO and constants: current-reader round trips cannot establish
+// whether a deployed framed-only reader accepts the bytes we issue.
+#[cfg(test)]
+pub(crate) mod e160_framed_reader {
+    use serde::{Deserialize, Serialize};
+
+    /// The continuation-token version used by ordinary progressing pages.
+    pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
+    const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
+
+    /// The sixteenth consecutive merged page without a key or new EOF fails.
+    /// This also bounds legitimate sparse listings; it is not a cycle detector.
+    pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
+
+    /// Envelope marker. A bucket that is *not* merging hands out the local
+    /// listing's own marker, so the decoder needs a positive signal before it
+    /// treats an opaque token as a merged one.
+    const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
+    // Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix),
+    // so this framing cannot collide with a local key used as an opaque marker.
+    const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:";
+
+    /// The continuation-token envelope. Opaque to clients: it is serialized as
+    /// framed JSON and then base64-encoded by the same helper as a local marker.
+    ///
+    /// A `null` cursor with `done = false` means "list that side from the start";
+    /// `done = true` means the side is finished and must not be listed again.
+    #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+    #[serde(deny_unknown_fields)]
+    pub struct ListThroughToken {
+        /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
+        pub t: String,
+        pub v: u32,
+        #[serde(default)]
+        pub local: Option<String>,
+        #[serde(default)]
+        pub local_done: bool,
+        #[serde(default)]
+        pub source: Option<String>,
+        #[serde(default)]
+        pub source_done: bool,
+        /// Last entry the previous page consumed. A side whose page was only
+        /// partially consumed is re-listed from the same cursor and everything at
+        /// or below this key is dropped, which is delimiter-safe: a rolled-up
+        /// common prefix compares as itself, never as its members.
+        #[serde(default)]
+        pub last_key: Option<String>,
+        /// Consecutive empty truncated merged pages, present only in v2 tokens.
+        /// Ordinary v1 tokens retain their original serialized shape.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        pub no_progress: Option<u8>,
+    }
+
+    impl ListThroughToken {
+        pub fn encode(&self) -> String {
+            // The envelope is built here from owned strings, so serialization
+            // cannot fail; the fallback keeps the signature infallible.
+            format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
+        }
+    }
+
+    /// What a decoded (base64-stripped) continuation token turned out to be.
+    #[derive(Clone, Debug, PartialEq, Eq)]
+    pub enum ListThroughCursor {
+        /// A plain local listing marker: the bucket was not merging when the token
+        /// was issued, or the client is paginating a non-merged listing.
+        Local(String),
+        Merged(Box<ListThroughToken>),
+    }
+
+    #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
+    pub enum ListThroughTokenError {
+        #[error("continuation token version {0} is not supported")]
+        UnsupportedVersion(u32),
+        /// The message never echoes the token: it is client-controlled input.
+        #[error("continuation token is malformed")]
+        Malformed,
+    }
+
+    /// Classifies an already base64-decoded continuation token.
+    ///
+    /// Only a framed JSON object is read as a merged token;
+    /// anything else is a local marker, so a bucket that turns `list_through` off
+    /// keeps paginating with the tokens it handed out. A token that *is* an
+    /// envelope but was tampered with (unknown version, unknown field, truncated
+    /// JSON) is an error, never a silent fallback.
+    pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
+        let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        };
+        let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
+        if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
+            return Err(ListThroughTokenError::Malformed);
+        }
+        match value.get("v").and_then(serde_json::Value::as_u64) {
+            Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
+                // v1 readers reject this field even when it is null or zero.
+                if value.get("no_progress").is_some() {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
+                if !value
+                    .get("no_progress")
+                    .and_then(serde_json::Value::as_u64)
+                    .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
+                {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
+            None => return Err(ListThroughTokenError::Malformed),
+        }
+        serde_json::from_value::<ListThroughToken>(value)
+            .map(|token| ListThroughCursor::Merged(Box::new(token)))
+            .map_err(|_| ListThroughTokenError::Malformed)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -797,6 +943,7 @@ mod tests {
     #[test]
     fn a_degraded_page_keeps_the_source_cursor_for_the_next_one() {
         let resume = ListThroughToken {
+            framed: false,
             t: LIST_THROUGH_TOKEN_TAG.to_string(),
             v: LIST_THROUGH_TOKEN_VERSION,
             local: Some("local-1".to_string()),
@@ -1033,7 +1180,7 @@ mod tests {
 
     #[test]
     fn token_round_trips_and_rejects_tampering() {
-        let token = ListThroughToken::new(
+        let mut token = ListThroughToken::new(
             SideCursor {
                 token: Some("l".to_string()),
                 done: false,
@@ -1041,6 +1188,7 @@ mod tests {
             SideCursor { token: None, done: true },
             Some("k".to_string()),
         );
+        token.framed = true;
         let encoded = token.encode();
         assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
 
@@ -1089,35 +1237,148 @@ mod tests {
 
     #[test]
     fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
-        fn framed(payload: &str) -> String {
-            format!("{LIST_THROUGH_TOKEN_PREFIX}{payload}")
-        }
-
         let token = progress_token(None, true, false);
         assert_eq!(
             token.encode(),
-            concat!(
-                "\0odm-list:",
-                r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
-            )
+            r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
         );
-        for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
-            let token = progress_token(Some(count), true, false);
-            assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
-        }
-        for version in [1, 2] {
-            for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
-                let encoded = framed(&format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#));
+        for framed in [false, true] {
+            let prefix = if framed { LIST_THROUGH_TOKEN_PREFIX } else { "" };
+            for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
+                let mut token = progress_token(Some(count), true, false);
+                token.framed = framed;
+                assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
+            }
+            // Bare recognition requires the complete shape emitted by old writers;
+            // partial JSON objects are also valid local keys.
+            for version in [1, 2] {
+                for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
+                    let encoded = format!(
+                        r#"{prefix}{{"t":"odm-list","v":{version},"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":{value}}}"#
+                    );
+                    assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
+                }
+            }
+            for encoded in [
+                r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1}"#,
+                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#,
+                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"extra":true}"#,
+                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"framed":true}"#,
+            ] {
+                let encoded = format!("{prefix}{encoded}");
                 assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
             }
+            let bumped = format!("{prefix}{}", token.encode().replace("\"v\":1", "\"v\":9"));
+            assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(9)));
         }
-        for payload in [
-            r#"{"t":"odm-list","v":1,"no_progress":1}"#,
-            r#"{"t":"odm-list","v":2}"#,
-            r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
+    }
+
+    // Frozen decoder from 447f3c704, before framing was introduced. Keeping this
+    // independent of the current decoder catches a default-writer rollout break.
+    fn decode_before_framing(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
+        if !decoded.starts_with('{') {
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        }
+        let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
+            // Not JSON at all: an object key may legitimately start with '{'.
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        };
+        if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        }
+        match value.get("v").and_then(serde_json::Value::as_u64) {
+            Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
+                // v1 readers reject this field even when it is null or zero.
+                if value.get("no_progress").is_some() {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
+                if !value
+                    .get("no_progress")
+                    .and_then(serde_json::Value::as_u64)
+                    .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
+                {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
+            None => return Err(ListThroughTokenError::Malformed),
+        }
+        serde_json::from_value::<ListThroughToken>(value)
+            .map(|token| ListThroughCursor::Merged(Box::new(token)))
+            .map_err(|_| ListThroughTokenError::Malformed)
+    }
+
+    #[test]
+    fn historical_writer_fixtures_and_default_output_remain_readable() {
+        for (wire, version, count) in [
+            (
+                r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#,
+                1,
+                None,
+            ),
+            (
+                r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#,
+                2,
+                Some(15),
+            ),
         ] {
-            let encoded = framed(payload);
-            assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
+            let ListThroughCursor::Merged(mut token) = decode_continuation_token(wire).expect("historical issued token") else {
+                panic!("a historical cursor must not silently become a local marker, even if a key has identical JSON");
+            };
+            assert_eq!(token.local.as_deref(), Some("local-2"));
+            assert_eq!(token.source.as_deref(), Some("source-2"));
+            assert_eq!(token.last_key.as_deref(), Some("k"));
+            assert_eq!(token.v, version);
+            assert_eq!(token.no_progress, count);
+            assert!(!token.framed);
+            assert_eq!(token.encode(), wire, "bare output retains the historical bytes");
+            assert_eq!(decode_before_framing(&token.encode()), Ok(ListThroughCursor::Merged(token.clone())));
+            token.framed = true;
+            let framed = format!("\0odm-list:{wire}");
+            assert_eq!(token.encode(), framed, "framing leaves the JSON payload unchanged");
+            assert_eq!(decode_continuation_token(&framed), Ok(ListThroughCursor::Merged(token)));
+        }
+    }
+
+    #[test]
+    fn frozen_e160_reader_distinguishes_framing_and_keeps_strict_budget_validation() {
+        use super::e160_framed_reader as old;
+
+        for raw in [
+            r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#,
+            r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#,
+        ] {
+            assert_eq!(old::decode_continuation_token(raw), Ok(old::ListThroughCursor::Local(raw.to_string())));
+            let framed = format!("\0odm-list:{raw}");
+            let old::ListThroughCursor::Merged(old_token) = old::decode_continuation_token(&framed).expect("old writer bytes")
+            else {
+                panic!("e160 recognizes its own frame");
+            };
+            assert_eq!(old_token.encode(), framed);
+            let ListThroughCursor::Merged(current) = decode_continuation_token(&framed).expect("dual reader") else {
+                panic!("dual readers preserve old framed chains");
+            };
+            assert_eq!(current.encode(), framed);
+            assert_eq!(current.local, old_token.local);
+            assert_eq!(current.local_done, old_token.local_done);
+            assert_eq!(current.source, old_token.source);
+            assert_eq!(current.source_done, old_token.source_done);
+            assert_eq!(current.last_key, old_token.last_key);
+            assert_eq!(current.v, old_token.v);
+            assert_eq!(current.no_progress, old_token.no_progress);
+        }
+        for count in ["null", "0", "16", "-1", "1.5", "\"1\"", "256"] {
+            let raw = format!(
+                "\0odm-list:{{\"t\":\"odm-list\",\"v\":2,\"local\":null,\"local_done\":true,\"source\":\"A\",\"source_done\":false,\"last_key\":null,\"no_progress\":{count}}}"
+            );
+            assert_eq!(
+                old::decode_continuation_token(&raw),
+                Err(old::ListThroughTokenError::Malformed),
+                "{count}"
+            );
+            assert_eq!(decode_continuation_token(&raw), Err(ListThroughTokenError::Malformed), "{count}");
         }
     }
 

rustfs/src/on_demand_migration/native_http.rs

@@ -99,6 +99,7 @@ impl NativeHttp {
     pub(super) fn for_test(endpoint: Url) -> Self {
         Self {
             client: reqwest::Client::builder()
+                .no_proxy()
                 .redirect(reqwest::redirect::Policy::none())
                 .build()
                 .expect("test http client should build"),
@@ -116,19 +117,27 @@ impl NativeHttp {
                 .path_segments_mut()
                 .map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?;
             path.clear();
-            path.extend(segments);
+            for segment in segments {
+                // URL normalization drops standalone dot segments. Sending
+                // that URL could fetch another object and backfill its bytes
+                // under the originally requested key.
+                if matches!(segment, "." | "..") {
+                    return Err(SourceError::Unsupported("source path contains an unsupported dot segment".to_string()));
+                }
+                path.push(segment);
+            }
         }
         Ok(url)
     }
 
     /// Sends the request and returns the response only for a 2xx status.
-    /// Non-2xx statuses are classified from the status and the provider's own
+    /// Non-2xx statuses are classified from the status and an optional provider
     /// error-code header; response bodies are not read, so no provider message
     /// can smuggle credentials or markup into a log line.
     pub(super) async fn send(
         &self,
         request: reqwest::Request,
-        error_code_header: &str,
+        error_code_header: Option<&str>,
     ) -> Result<reqwest::Response, SourceError> {
         self.send_classified(request, error_code_header, false).await
     }
@@ -137,7 +146,7 @@ impl NativeHttp {
     pub(super) async fn send_object(
         &self,
         request: reqwest::Request,
-        error_code_header: &str,
+        error_code_header: Option<&str>,
     ) -> Result<reqwest::Response, SourceError> {
         self.send_classified(request, error_code_header, true).await
     }
@@ -145,26 +154,42 @@ impl NativeHttp {
     async fn send_classified(
         &self,
         request: reqwest::Request,
-        error_code_header: &str,
+        error_code_header: Option<&str>,
         not_found_on_404_without_code: bool,
     ) -> Result<reqwest::Response, SourceError> {
-        let response = self.client.execute(request).await.map_err(classify_transport_error)?;
+        let response = self.execute(request).await?;
+        let status = response.status();
+        match Self::check_response(response, error_code_header) {
+            Err(SourceError::Other(_)) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
+            result => result,
+        }
+    }
+
+    pub(super) async fn execute(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
+        self.client.execute(request).await.map_err(classify_transport_error)
+    }
+
+    pub(super) fn check_response(
+        response: reqwest::Response,
+        error_code_header: Option<&str>,
+    ) -> Result<reqwest::Response, SourceError> {
         let status = response.status();
         if status.is_success() {
             return Ok(response);
         }
-        let code = response
-            .headers()
-            .get(error_code_header)
+        let code = error_code_header
+            .and_then(|header| response.headers().get(header))
             .and_then(|value| value.to_str().ok())
             .map(str::to_string);
         let message = match &code {
             Some(code) => format!("source returned HTTP {status} ({code})"),
             None => format!("source returned HTTP {status}"),
         };
-        match classify_status(status.as_u16(), code.as_deref(), message) {
-            SourceError::Other(_) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
-            err => Err(err),
+        match classify_status(status.as_u16(), code.as_deref(), message.clone()) {
+            // Native object absence needs provider-specific evidence or a
+            // successful bucket probe, never an alias from the S3 classifier.
+            SourceError::NotFound => Err(classify_status(status.as_u16(), None, message)),
+            error => Err(error),
         }
     }
 }
@@ -431,4 +456,44 @@ mod tests {
         assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt");
         assert_eq!(url.query(), None, "a key with '?' must not become a query");
     }
+
+    #[test]
+    fn native_http_refuses_dot_segments_instead_of_addressing_another_object() {
+        let http = NativeHttp::for_test(Url::parse("https://source.example.com").expect("origin"));
+        for key in [
+            ".",
+            "..",
+            "./key",
+            "../key",
+            "dir/./key",
+            "dir/../key",
+            "dir/.",
+            "dir/..",
+            "\u{fffe}/../key",
+        ] {
+            let error = http
+                .url(std::iter::once("bucket").chain(key.split('/')))
+                .expect_err("dot segments must not disappear");
+            assert!(matches!(error, SourceError::Unsupported(_)), "{key:?}: {error}");
+        }
+    }
+
+    #[test]
+    fn native_http_preserves_ordinary_dots_empty_segments_and_literal_escapes() {
+        let http = NativeHttp::for_test(Url::parse("https://source.example.com").expect("origin"));
+        for (key, path) in [
+            ("file.txt", "/bucket/file.txt"),
+            (".hidden/.../tail.", "/bucket/.hidden/.../tail."),
+            ("/dir//key/", "/bucket//dir//key/"),
+            ("%2e/%2E%2E/key", "/bucket/%252e/%252E%252E/key"),
+            ("a+b &?#", "/bucket/a+b%20&%3F%23"),
+        ] {
+            let url = http
+                .url(std::iter::once("bucket").chain(key.split('/')))
+                .expect("representable key");
+            assert_eq!(url.path(), path, "{key:?}");
+            assert!(url.query().is_none());
+            assert!(url.fragment().is_none());
+        }
+    }
 }

rustfs/src/on_demand_migration/source_client.rs

@@ -337,7 +337,7 @@ const THROTTLE_CODES: &[&str] = &[
     "RequestThrottled",
     "ServerBusy",
 ];
-const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "BlobNotFound"];
+const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"];
 const ACCESS_DENIED_CODES: &[&str] = &[
     "AccessDenied",
     "InvalidAccessKeyId",
@@ -1813,10 +1813,11 @@ mod tests {
 
     /// The S3 backend behind the scripted connector, without the prefix-mapping
     /// client on top: the contract is a property of the backend itself.
-    async fn scripted_s3_backend(responses: Vec<Scripted>) -> S3SourceBackend {
+    async fn scripted_s3_backend(responses: Vec<Scripted>) -> (S3SourceBackend, Recorded) {
         let spec = spec(None);
+        let requests: Recorded = Arc::new(Mutex::new(Vec::new()));
         let connector = SharedHttpConnector::new(ScriptedConnector {
-            requests: Arc::new(Mutex::new(Vec::new())),
+            requests: Arc::clone(&requests),
             responses: Arc::new(Mutex::new(responses.into_iter().collect())),
         });
         let http_client = http_client_fn(move |_settings, _components| connector.clone());
@@ -1826,17 +1827,20 @@ mod tests {
             .expect("test spec should build")
             .http_client(http_client)
             .interceptor(SourceProxyMarkerInterceptor::new());
-        S3SourceBackend {
-            client: S3Client::from_conf(config.build()),
-            bucket: spec.bucket.clone(),
-        }
+        (
+            S3SourceBackend {
+                client: S3Client::from_conf(config.build()),
+                bucket: spec.bucket.clone(),
+            },
+            requests,
+        )
     }
 
     #[tokio::test]
     async fn s3_backend_satisfies_the_shared_backend_contract() {
         let mut ranged = contract_object_headers(3);
         ranged.push(("content-range", "bytes 1-3/5".to_string()));
-        let backend = scripted_s3_backend(vec![
+        let (backend, requests) = scripted_s3_backend(vec![
             ok(contract_object_headers(5), ""),
             ok(contract_object_headers(5), "hello"),
             ok(ranged, "ell"),
@@ -1845,6 +1849,7 @@ mod tests {
             ok(Vec::new(), CONTRACT_TAGGING),
             ok(Vec::new(), ""),
             status(404, ""),
+            // An object HEAD 404 requires the existing S3 bucket HEAD probe.
             ok(Vec::new(), ""),
             status(403, ACCESS_DENIED_BODY),
         ])
@@ -1859,6 +1864,32 @@ mod tests {
             },
         )
         .await;
+        let requests = recorded(&requests);
+        let actual: Vec<_> = requests
+            .iter()
+            .map(|request| {
+                (
+                    request.method.as_str(),
+                    url::Url::parse(&request.uri).expect("recorded S3 URL").path().to_string(),
+                )
+            })
+            .collect();
+        let expected = [
+            ("HEAD", "/source-bucket/dir/a.txt"),
+            ("GET", "/source-bucket/dir/a.txt"),
+            ("GET", "/source-bucket/dir/a.txt"),
+            ("GET", "/source-bucket/"),
+            ("GET", "/source-bucket/"),
+            ("GET", "/source-bucket/dir/a.txt"),
+            ("HEAD", "/source-bucket/"),
+            ("HEAD", "/source-bucket/missing"),
+            ("HEAD", "/source-bucket/"),
+            ("HEAD", "/source-bucket/secret"),
+        ];
+        assert_eq!(actual, expected.map(|(method, path)| (method, path.to_string())));
+        for request in &requests {
+            assert_outbound_markers(request);
+        }
     }
 
     fn prefix_client(prefix: Option<String>) -> SourceClient {

rustfs/src/on_demand_migration/test_http_fixture.rs

@@ -56,6 +56,16 @@ impl RecordedRequest {
 
 pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;
 
+/// Checks the full request sequence, including the absence of extra probes.
+pub(super) fn assert_requests(recorder: &Recorder, expected: &[(&str, &str)]) {
+    let recorded = recorder.lock().expect("recorder lock");
+    let actual: Vec<_> = recorded
+        .iter()
+        .map(|request| (request.method.as_str(), request.target.as_str()))
+        .collect();
+    assert_eq!(actual, expected, "unexpected native source request sequence");
+}
+
 /// Binds a loopback listener that answers `responses` in order and returns its
 /// origin plus the recorder. The task ends once the script is exhausted.
 pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {

rustfs/src/site_replication/hooks.rs

@@ -302,7 +302,164 @@ pub(crate) fn site_replication_state_replicates_ilm_expiry(state: &SiteReplicati
     state.peers.values().any(|peer| peer.replicate_ilm_expiry)
 }
 
-pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteReplicationBootstrapPlan> {
+/// Secret-bearing half of the IAM snapshot. `SRInfo` is served to admin
+/// callers (`site-replication/info`, status, add preflight) and must stay
+/// secret-free, so the bootstrap plan receives credentials through this
+/// separate value, built only on the paths that deliver to peers (site add
+/// bootstrap, repair, retry snapshot resend). Never persisted, never served.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct SiteReplicationIamCredentials {
+    /// Built-in users (access key -> credential); temp and service accounts
+    /// are excluded, external/IdP users never appear here.
+    pub(crate) users: BTreeMap<String, SiteReplicationUserCredential>,
+    /// Every service account except the site replicator's own, already
+    /// shaped as the `service-account` create item the live hook emits.
+    pub(crate) service_accounts: Vec<SiteReplicationServiceAccountSnapshot>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SiteReplicationUserCredential {
+    pub(crate) secret_key: String,
+    pub(crate) status: AccountStatus,
+    /// The user record's own update time (the axis the receiver's staleness
+    /// check compares against), unlike `UserInfo::updated_at` which
+    /// `list_users` overwrites with the policy mapping's time.
+    pub(crate) updated_at: Option<OffsetDateTime>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SiteReplicationServiceAccountSnapshot {
+    pub(crate) create: SRSvcAccCreate,
+    pub(crate) envelope: Option<SRSvcAccReplicationEnvelope>,
+    pub(crate) updated_at: Option<OffsetDateTime>,
+}
+
+pub(crate) const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2;
+
+pub(crate) fn encode_service_account_replication_policy(
+    claims: &HashMap<String, Value>,
+    session_policy: Option<&str>,
+) -> S3Result<(SRSessionPolicy, Option<SRSvcAccReplicationEnvelope>)> {
+    if !claims.contains_key(OIDC_VIRTUAL_PARENT_CLAIM) {
+        return session_policy
+            .map(SRSessionPolicy::from_json)
+            .transpose()
+            .map(|policy| policy.unwrap_or_default())
+            .map(|policy| (policy, None))
+            .map_err(|err| s3_error!(InvalidArgument, "marshal policy failed: {:?}", err));
+    }
+
+    let policy = match session_policy {
+        Some(policy) => serde_json::from_str::<Policy>(policy)
+            .map_err(|err| s3_error!(InvalidArgument, "invalid service account replication policy: {:?}", err))?,
+        None => Policy::default(),
+    };
+    if policy.statements.is_empty() && (!policy.id.is_empty() || !policy.version.is_empty())
+        || policy.version.is_empty() && !policy.statements.is_empty()
+    {
+        return Err(s3_error!(InvalidArgument, "service account replication policy is not normalized"));
+    }
+    let policy = serde_json::to_string(&policy)
+        .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?;
+    let policy = SRSessionPolicy::from_json(&policy)
+        .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?;
+    Ok((
+        policy,
+        Some(SRSvcAccReplicationEnvelope {
+            version: SERVICE_ACCOUNT_ENVELOPE_VERSION,
+        }),
+    ))
+}
+
+/// Read the credentials the IAM snapshot needs straight from the IAM store:
+/// `list_users` deliberately strips secret keys and skips service accounts,
+/// which is right for an admin listing and wrong for a peer snapshot (the
+/// plan builder used to drop every user for lack of a secret, so a status
+/// change or secret rotation committed while a peer was unreachable never
+/// reached it — backlog#2289).
+pub(crate) async fn build_sr_iam_credentials() -> S3Result<SiteReplicationIamCredentials> {
+    let mut credentials = SiteReplicationIamCredentials::default();
+    let Some(iam_sys) = current_iam_handle() else {
+        return Ok(credentials);
+    };
+
+    let mut users = HashMap::new();
+    iam_sys.load_users(UserType::Reg, &mut users).await.map_err(ApiError::from)?;
+    for (access_key, identity) in users {
+        if identity.credentials.is_temp() || identity.credentials.is_service_account() {
+            continue;
+        }
+        credentials.users.insert(
+            access_key,
+            SiteReplicationUserCredential {
+                secret_key: identity.credentials.secret_key,
+                status: if identity.credentials.status == "off" {
+                    AccountStatus::Disabled
+                } else {
+                    AccountStatus::Enabled
+                },
+                updated_at: identity.update_at,
+            },
+        );
+    }
+
+    let mut service_accounts = HashMap::new();
+    iam_sys
+        .load_users(UserType::Svc, &mut service_accounts)
+        .await
+        .map_err(ApiError::from)?;
+    let mut service_accounts: Vec<_> = service_accounts.into_iter().collect();
+    service_accounts.sort_by(|(a, _), (b, _)| a.cmp(b));
+    for (access_key, identity) in service_accounts {
+        // The replicator account is installed by join / rotate, never by a snapshot.
+        if access_key == SITE_REPLICATOR_SERVICE_ACCOUNT || !identity.credentials.is_service_account() {
+            continue;
+        }
+        let claims = iam_sys.get_claims_for_svc_acc(&access_key).await.map_err(ApiError::from)?;
+        let (account, session_policy) = iam_sys.get_service_account(&access_key).await.map_err(ApiError::from)?;
+        let session_policy = session_policy
+            .map(|policy| serde_json::to_string(&policy))
+            .transpose()
+            .map_err(|err| {
+                S3Error::with_message(
+                    S3ErrorCode::InternalError,
+                    format!("marshal service account session policy failed: {err:?}"),
+                )
+            })?;
+        let (session_policy, envelope) = encode_service_account_replication_policy(&claims, session_policy.as_deref())?;
+        credentials.service_accounts.push(SiteReplicationServiceAccountSnapshot {
+            create: SRSvcAccCreate {
+                parent: identity.credentials.parent_user,
+                access_key,
+                secret_key: identity.credentials.secret_key,
+                groups: identity.credentials.groups.unwrap_or_default(),
+                claims,
+                session_policy,
+                status: identity.credentials.status,
+                name: account.name.unwrap_or_default(),
+                description: account.description.unwrap_or_default(),
+                expiration: account.expiration,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            },
+            envelope,
+            updated_at: identity.update_at,
+        });
+    }
+
+    Ok(credentials)
+}
+
+/// The bootstrap plan for peer delivery: `info` (secret-free) plus the IAM
+/// credentials read at this moment.
+pub(crate) async fn build_site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteReplicationBootstrapPlan> {
+    let credentials = build_sr_iam_credentials().await?;
+    site_replication_bootstrap_plan(info, &credentials)
+}
+
+pub(crate) fn site_replication_bootstrap_plan(
+    info: &SRInfo,
+    credentials: &SiteReplicationIamCredentials,
+) -> S3Result<SiteReplicationBootstrapPlan> {
     let mut plan = SiteReplicationBootstrapPlan::default();
     let replicate_ilm_expiry = site_replication_info_replicates_ilm_expiry(info);
 
@@ -318,24 +475,57 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
     }
 
     for (access_key, user) in &info.user_info_map {
-        if let Some(secret_key) = &user.secret_key {
-            plan.iam_items.push(SRIAMItem {
-                r#type: "iam-user".to_string(),
-                iam_user: Some(rustfs_madmin::SRIAMUser {
-                    access_key: access_key.clone(),
-                    is_delete_req: false,
-                    user_req: Some(AddOrUpdateUserReq {
-                        secret_key: secret_key.clone(),
-                        policy: user.policy_name.clone(),
-                        status: user.status.clone(),
-                    }),
-                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        // Credentials come from the store snapshot; an inline `secret_key` on
+        // the SRInfo entry (older callers, tests) is accepted as a fallback.
+        // Users with neither (external / IdP identities) have nothing a peer
+        // could install and are skipped.
+        let credential = credentials.users.get(access_key);
+        let Some(secret_key) = credential
+            .map(|credential| credential.secret_key.clone())
+            .or_else(|| user.secret_key.clone())
+            .filter(|secret_key| !secret_key.is_empty())
+        else {
+            continue;
+        };
+        let status = credential
+            .map(|credential| credential.status.clone())
+            .unwrap_or_else(|| user.status.clone());
+        let updated_at = credential.and_then(|credential| credential.updated_at).or(user.updated_at);
+        plan.iam_items.push(SRIAMItem {
+            r#type: "iam-user".to_string(),
+            iam_user: Some(rustfs_madmin::SRIAMUser {
+                access_key: access_key.clone(),
+                is_delete_req: false,
+                user_req: Some(AddOrUpdateUserReq {
+                    secret_key,
+                    policy: user.policy_name.clone(),
+                    status,
                 }),
-                updated_at: user.updated_at,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        });
+    }
+
+    // Service accounts follow their parents: the receiver creates a missing
+    // account under `parent` and updates an existing one (secret, status,
+    // session policy), so a rotation or disable committed during an outage
+    // converges through the same snapshot as users do.
+    for account in &credentials.service_accounts {
+        plan.iam_items.push(SRIAMItem {
+            r#type: "service-account".to_string(),
+            svc_acc_change: Some(SRSvcAccChange {
+                create: Some(account.create.clone()),
+                oidc_service_account_envelope: account.envelope.clone(),
                 api_version: Some(SITE_REPL_API_VERSION.to_string()),
                 ..Default::default()
-            });
-        }
+            }),
+            updated_at: account.updated_at,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        });
     }
 
     for (name, desc) in &info.group_desc_map {
@@ -518,7 +708,12 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
     } else {
         path
     };
-    broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await?;
+    // Both steps run to completion on their own: the broadcast attempts every
+    // peer and reports the first failure (backlog#2293), so stopping here on
+    // that error would skip `configure-replication` for the peers whose
+    // `make` just succeeded — and nothing records a retry for that gap. The
+    // failed peer's retry events cover both steps independently.
+    let make_result = broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await;
 
     let configure_path = bootstrap_bucket_op_path(bucket, "configure-replication");
     let configure_path = if let Some(token) = bootstrap_token {
@@ -526,7 +721,8 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
     } else {
         configure_path
     };
-    broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
+    let configure_result = broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await;
+    make_result.and(configure_result)
 }
 
 const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
@@ -832,6 +1028,21 @@ pub async fn site_replication_iam_change_hook(item: SRIAMItem) -> S3Result<()> {
     let Some(runtime) = runtime_site_replication_targets().await? else {
         return Ok(());
     };
+    // A local revoke must out-rank a stale grant a peer delivers later, so its
+    // mark is committed before the broadcast (backlog#2291). The broadcast
+    // still goes out when the mark cannot be persisted: the peers' own records
+    // remain the primary gate, the mark only covers the deleted case.
+    if let Err(err) = record_iam_deletion_marks_for_item(&item).await {
+        warn!(
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
+            event = EVENT_ADMIN_SITE_REPLICATION_STATE,
+            item_type = %item.r#type,
+            result = "iam_deletion_mark_not_recorded",
+            error = ?err,
+            "failed to record local IAM deletion mark before broadcast"
+        );
+    }
     let mut first_error: Option<S3Error> = None;
     for peer in runtime.state.peers.values() {
         if peer.deployment_id == runtime.local_peer.deployment_id

rustfs/src/site_replication/mod.rs

@@ -79,13 +79,16 @@ use http::header::{CONTENT_TYPE, HOST};
 use http::{HeaderMap, HeaderValue, Uri};
 use hyper::{Method, StatusCode};
 use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH};
+use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM;
 use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type};
 use rustfs_iam::sys::SITE_REPLICATOR_SERVICE_ACCOUNT;
 use rustfs_madmin::{
-    AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus, SITE_REPL_API_VERSION,
-    SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq, SRResyncOpStatus,
-    SRRetryStats, SRStateInfo, SyncStatus,
+    AccountStatus, AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus,
+    SITE_REPL_API_VERSION, SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq,
+    SRResyncOpStatus, SRRetryStats, SRSessionPolicy, SRStateInfo, SRSvcAccChange, SRSvcAccCreate, SRSvcAccDelete,
+    SRSvcAccReplicationEnvelope, SyncStatus,
 };
+use rustfs_policy::policy::Policy;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration};
@@ -107,6 +110,26 @@ use tracing::{info, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
+/// Serialize `value` with every JSON object's keys sorted, for hashing and
+/// equality checks. `HashMap` fields (service-account claims) iterate in a
+/// per-instance random order and `serde_json` is built with `preserve_order`,
+/// so two identical plans would otherwise hash differently: the repair
+/// preflight token went stale between dry-run and execute, and a retry
+/// snapshot resend never looked "stable" (backlog#2289 follow-up).
+pub(crate) fn canonical_json_vec<T: Serialize>(value: &T) -> serde_json::Result<Vec<u8>> {
+    fn sort_keys(value: Value) -> Value {
+        match value {
+            Value::Object(map) => {
+                let sorted: BTreeMap<String, Value> = map.into_iter().map(|(key, value)| (key, sort_keys(value))).collect();
+                Value::Object(sorted.into_iter().collect())
+            }
+            Value::Array(items) => Value::Array(items.into_iter().map(sort_keys).collect()),
+            other => other,
+        }
+    }
+    serde_json::to_vec(&sort_keys(serde_json::to_value(value)?))
+}
+
 pub(crate) const LOG_COMPONENT_ADMIN: &str = "admin";
 
 pub(crate) const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication";

rustfs/src/site_replication/repair.rs

@@ -234,9 +234,9 @@ impl SiteReplicationRepairTask<'_> {
 
     pub(crate) fn id(&self) -> S3Result<String> {
         let payload = match self {
-            Self::Iam(item) => serde_json::to_vec(item),
+            Self::Iam(item) => canonical_json_vec(item),
             Self::BucketMake(_) | Self::Replication(_) => serde_json::to_vec(&serde_json::json!({})),
-            Self::BucketMetadata(item) => serde_json::to_vec(item),
+            Self::BucketMetadata(item) => canonical_json_vec(item),
         }
         .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair task failed: {err}")))?;
         let mut digest = Sha256::new();
@@ -726,7 +726,7 @@ pub(crate) async fn execute_site_replication_repair_locked(
         return Err(s3_error!(InvalidRequest, "site replication is not configured"));
     }
     let info = build_sr_info(&state, &request.local_peer).await?;
-    let plan = site_replication_bootstrap_plan(&info)?;
+    let plan = build_site_replication_bootstrap_plan(&info).await?;
     let plan_token = site_replication_repair_plan_token(&state, &plan)?;
     let preflight_token = site_replication_repair_preflight_token(&state, &plan, request.signing_key.as_bytes())?;
     let sites = site_replication_repair_sites(&state, &request.local_peer, &plan, request.signing_key.as_bytes())?;

rustfs/src/site_replication/retry.rs

@@ -397,12 +397,12 @@ pub(crate) fn iam_deletion_replay_matches(record: &SiteReplicationIamDeletionRep
 /// newer revision of one another.
 pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
     match item.r#type.as_str() {
-        "policy" if item.policy.is_none() => Some(format!("policy:{}", item.name)),
+        "policy" if item.policy.is_none() => Some(iam_policy_deletion_mark_entity(&item.name)),
         "iam-user" => item
             .iam_user
             .as_ref()
             .filter(|user| user.is_delete_req)
-            .map(|user| format!("iam-user:{}", user.access_key)),
+            .map(|user| iam_user_deletion_mark_entity(&user.access_key)),
         "group-info" => item
             .group_info
             .as_ref()
@@ -416,7 +416,7 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
             .policy_mapping
             .as_ref()
             .filter(|mapping| mapping.policy.is_empty())
-            .map(|mapping| format!("policy-mapping:{}:{}:{}", mapping.user_or_group, mapping.user_type, mapping.is_group)),
+            .map(|mapping| iam_policy_mapping_deletion_mark_entity(&mapping.user_or_group, mapping.user_type, mapping.is_group)),
         "service-account" => item
             .svc_acc_change
             .as_ref()
@@ -426,6 +426,82 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
     }
 }
 
+/// The entities whose deletion a deletion-shaped IAM item commits, keyed the
+/// way the receive-side staleness gate looks them up once the local record is
+/// gone (backlog#2291); empty for creates and updates. Group member removal
+/// yields one entity per removed member so a stale re-add of that member can
+/// be judged, and a group delete (no members) yields the group itself.
+pub(crate) fn iam_item_deletion_mark_entities(item: &SRIAMItem) -> Vec<String> {
+    if item.r#type == "group-info" {
+        let Some(update) = item
+            .group_info
+            .as_ref()
+            .map(|group| &group.update_req)
+            .filter(|update| update.is_remove)
+        else {
+            return Vec::new();
+        };
+        if update.members.is_empty() {
+            return vec![iam_group_deletion_mark_entity(&update.group)];
+        }
+        return update
+            .members
+            .iter()
+            .map(|member| iam_group_member_deletion_mark_entity(&update.group, member))
+            .collect();
+    }
+    iam_item_deletion_entity(item).into_iter().collect()
+}
+
+pub(crate) fn iam_policy_deletion_mark_entity(name: &str) -> String {
+    format!("policy:{name}")
+}
+
+pub(crate) fn iam_user_deletion_mark_entity(access_key: &str) -> String {
+    format!("iam-user:{access_key}")
+}
+
+/// `user_type` is the SR wire integer, as carried by the item on both sides.
+pub(crate) fn iam_policy_mapping_deletion_mark_entity(user_or_group: &str, user_type: i64, is_group: bool) -> String {
+    format!("policy-mapping:{user_or_group}:{user_type}:{is_group}")
+}
+
+pub(crate) fn iam_group_deletion_mark_entity(group: &str) -> String {
+    format!("group:{group}")
+}
+
+pub(crate) fn iam_group_member_deletion_mark_entity(group: &str, member: &str) -> String {
+    format!("group-member:{group}:{member}")
+}
+
+/// Persist the deletion marks of `item` (its source `updated_at` per entity
+/// of [`iam_item_deletion_mark_entities`]) through the state transaction.
+/// No-op for creates/updates and for items without a source timestamp
+/// (older peers): a mark without a source clock could not be ordered against
+/// later items. Called before a local deletion is broadcast and after a
+/// replicated deletion is applied, so both sides out-rank a stale grant that
+/// arrives later.
+pub(crate) async fn record_iam_deletion_marks_for_item(item: &SRIAMItem) -> S3Result<()> {
+    let entities = iam_item_deletion_mark_entities(item);
+    let Some(deleted_at) = item.updated_at.filter(|_| !entities.is_empty()) else {
+        return Ok(());
+    };
+    commit_iam_deletion_marks(entities, deleted_at).await
+}
+
+/// [`record_iam_deletion_marks`] under the state transaction; the write is
+/// skipped when no mark moves.
+pub(crate) async fn commit_iam_deletion_marks(entities: Vec<String>, deleted_at: OffsetDateTime) -> S3Result<()> {
+    update_site_replication_state_when_changed(move |state| {
+        Ok(if record_iam_deletion_marks(state, &entities, deleted_at) {
+            StateCommit::Changed(())
+        } else {
+            StateCommit::Unchanged(())
+        })
+    })
+    .await
+}
+
 /// Failure bookkeeping for one IAM item delivery: upsert the collapsed retry
 /// event and, when the item is a deletion, record its body for replay. Both
 /// live in the same state so the caller commits them in one transaction — a
@@ -791,8 +867,8 @@ impl RetrySnapshot {
 
     pub(crate) fn fingerprint(&self) -> S3Result<Vec<Vec<u8>>> {
         let mut payloads = match self {
-            Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::<Result<Vec<_>, _>>(),
-            Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::<Result<Vec<_>, _>>(),
+            Self::Iam(items) => items.iter().map(canonical_json_vec).collect::<Result<Vec<_>, _>>(),
+            Self::BucketMetadata(items) => items.iter().map(canonical_json_vec).collect::<Result<Vec<_>, _>>(),
         }
         .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?;
         payloads.sort_unstable();
@@ -954,6 +1030,7 @@ pub(crate) enum IamSnapshotKey {
     User(String),
     Group(String),
     PolicyMapping { target: String, user_type: i64, is_group: bool },
+    ServiceAccount(String),
 }
 
 pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option<IamSnapshotKey> {
@@ -972,6 +1049,11 @@ pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option<IamSnapshotKey> {
             user_type: mapping.user_type,
             is_group: mapping.is_group,
         }),
+        "service-account" => item
+            .svc_acc_change
+            .as_ref()
+            .and_then(|change| change.create.as_ref())
+            .map(|create| IamSnapshotKey::ServiceAccount(create.access_key.clone())),
         _ => None,
     }
 }
@@ -1006,6 +1088,24 @@ pub(crate) fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateT
                 mapping.policy.clear();
             }
         }
+        "service-account" => {
+            let Some(access_key) = item
+                .svc_acc_change
+                .as_ref()
+                .and_then(|change| change.create.as_ref())
+                .map(|create| create.access_key.clone())
+            else {
+                return Vec::new();
+            };
+            tombstone.svc_acc_change = Some(SRSvcAccChange {
+                delete: Some(SRSvcAccDelete {
+                    access_key,
+                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                }),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            });
+        }
         _ => return Vec::new(),
     }
     vec![tombstone]
@@ -1701,7 +1801,7 @@ pub(crate) async fn drain_site_replication_retry_queue_locked(
     // tick and only when a snapshot resend is actually due.
     let plan = if needs_plan {
         let info = build_sr_info(&runtime.state, &runtime.local_peer).await?;
-        Some(site_replication_bootstrap_plan(&info)?)
+        Some(build_site_replication_bootstrap_plan(&info).await?)
     } else {
         None
     };
@@ -1841,7 +1941,7 @@ pub(crate) async fn drain_one_site_replication_retry_event(
                     }
                 }
                 let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?;
-                let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?;
+                let fresh_plan = build_site_replication_bootstrap_plan(&fresh_info).await?;
                 let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot");
                 if fresh_snapshot.fingerprint()? == current_fingerprint {
                     if is_iam {

rustfs/src/site_replication/state.rs

@@ -64,6 +64,104 @@ pub(crate) struct SiteReplicationState {
     /// newer edit that already landed.
     #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub(crate) applied_edit_generations: BTreeMap<String, u64>,
+    /// Source timestamp of the newest IAM deletion committed on this site,
+    /// keyed by the deleted entity (`iam_item_deletion_mark_entities`). A
+    /// deletion leaves no local record to judge a later item against, so this
+    /// is what lets the receive-side staleness gate reject a grant that is
+    /// older than the revoke it would otherwise undo (backlog#2291). Marks
+    /// are kept for [`SITE_REPLICATION_IAM_DELETION_MARK_RETENTION`] and never
+    /// evicted by count: see that constant for why a count bound would open
+    /// exactly the window the marks exist to close.
+    #[serde(default, with = "rfc3339_map", skip_serializing_if = "BTreeMap::is_empty")]
+    pub(crate) iam_deletion_marks: BTreeMap<String, OffsetDateTime>,
+}
+
+/// How long an IAM deletion mark outlives the deletion it records.
+///
+/// A mark fences the delivery paths that can still carry an older grant for
+/// the deleted entity: a live delivery delayed in transit, the same grant
+/// arriving on a sibling node while the revoke is being applied, and a
+/// snapshot (bootstrap / repair / resend) built by a peer that has not yet
+/// received the deletion — which is bounded by this site's own retry queue
+/// towards that peer, whose backoff tops out at one day
+/// (`SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS`). The retry drain itself
+/// never replays a stale grant: it resends snapshots of the current records
+/// and the recorded deletion bodies. Thirty days is an order of magnitude
+/// beyond every one of those windows. Marks are pruned by age only — a count
+/// bound would drop a mark that is still inside the delivery window as soon
+/// as enough newer deletions happen, letting the delayed grant re-create the
+/// entity, which is the very hole the marks close.
+pub(crate) const SITE_REPLICATION_IAM_DELETION_MARK_RETENTION: time::Duration = time::Duration::days(30);
+
+/// Record that deletions of `entities` with source timestamp `deleted_at`
+/// were committed here. Newest wins per entity: an older deletion never
+/// lowers a mark. Marks older than the retention are pruned in the same
+/// pass. Returns whether the state changed.
+pub(crate) fn record_iam_deletion_marks(
+    state: &mut SiteReplicationState,
+    entities: &[String],
+    deleted_at: OffsetDateTime,
+) -> bool {
+    record_iam_deletion_marks_at(state, entities, deleted_at, OffsetDateTime::now_utc())
+}
+
+/// [`record_iam_deletion_marks`] pruning against an explicit `now`.
+pub(crate) fn record_iam_deletion_marks_at(
+    state: &mut SiteReplicationState,
+    entities: &[String],
+    deleted_at: OffsetDateTime,
+    now: OffsetDateTime,
+) -> bool {
+    let mut changed = false;
+    for entity in entities {
+        if state
+            .iam_deletion_marks
+            .get(entity)
+            .is_some_and(|existing| *existing >= deleted_at)
+        {
+            continue;
+        }
+        state.iam_deletion_marks.insert(entity.clone(), deleted_at);
+        changed = true;
+    }
+    let expired_before = now - SITE_REPLICATION_IAM_DELETION_MARK_RETENTION;
+    let before = state.iam_deletion_marks.len();
+    state.iam_deletion_marks.retain(|_, deleted_at| *deleted_at >= expired_before);
+    changed || state.iam_deletion_marks.len() != before
+}
+
+/// Newest deletion mark among `entities`, or `None` when no deletion of any
+/// of them was recorded here. The receive-side staleness gate feeds this in
+/// as the local timestamp when the targeted record is absent.
+pub(crate) fn iam_deletion_mark(state: &SiteReplicationState, entities: &[String]) -> Option<OffsetDateTime> {
+    entities
+        .iter()
+        .filter_map(|entity| state.iam_deletion_marks.get(entity).copied())
+        .max()
+}
+
+/// RFC 3339 map values, matching the other timestamps in the state object
+/// (`time::serde::rfc3339` only applies to a single field).
+mod rfc3339_map {
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+    use std::collections::BTreeMap;
+    use time::OffsetDateTime;
+
+    #[derive(Serialize, Deserialize)]
+    #[serde(transparent)]
+    struct Stamp(#[serde(with = "time::serde::rfc3339")] OffsetDateTime);
+
+    pub(super) fn serialize<S: Serializer>(map: &BTreeMap<String, OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.collect_map(map.iter().map(|(entity, deleted_at)| (entity, Stamp(*deleted_at))))
+    }
+
+    pub(super) fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<BTreeMap<String, OffsetDateTime>, D::Error> {
+        let map = BTreeMap::<String, Stamp>::deserialize(deserializer)?;
+        Ok(map
+            .into_iter()
+            .map(|(entity, Stamp(deleted_at))| (entity, deleted_at))
+            .collect())
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -323,6 +421,33 @@ where
     update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await
 }
 
+/// The state transaction for work that has to await inside it: an IAM write
+/// that must be ordered with the staleness verdict taken before it and the
+/// deletion mark committed after it (backlog#2291). Same boundary as
+/// [`update_site_replication_state`] — load and persist under the
+/// distributed state-object write lock, so two nodes of this site cannot
+/// interleave their verdicts and writes — and the same rules inside: no peer
+/// network calls and no other config locks. The closure hands the state back
+/// as `Some` when it changed it; `None` skips the write.
+pub(crate) async fn with_site_replication_state_transaction<T, F, Fut>(transaction: F) -> S3Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce(SiteReplicationState) -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = S3Result<(T, Option<SiteReplicationState>)>> + Send + 'static,
+{
+    with_site_replication_state_lock(move || async move {
+        let store = current_object_store_handle()
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
+        let state = load_site_replication_state_no_lock(store.clone()).await?;
+        let (result, changed) = transaction(state).await?;
+        if let Some(state) = changed {
+            persist_site_replication_state_no_lock(store, state).await?;
+        }
+        Ok(result)
+    })
+    .await
+}
+
 /// [`update_site_replication_state`] for closures that may find nothing to
 /// do — see [`StateCommit`].
 pub(crate) async fn update_site_replication_state_when_changed<T, F>(update: F) -> S3Result<T>

rustfs/src/site_replication/tests.rs

@@ -554,6 +554,145 @@ fn test_iam_item_deletion_entity_shapes() {
     assert!(iam_item_deletion_entity(&policy_set).is_none());
 }
 
+/// Deletion marks (backlog#2291) key on the same entities as the replay
+/// records, except that a group member removal is marked per member (so a
+/// stale re-add of one member can be judged) and a group delete marks the
+/// group itself. Creates and updates leave no mark.
+#[test]
+fn test_iam_item_deletion_mark_entities_shapes() {
+    assert_eq!(
+        iam_item_deletion_mark_entities(&user_delete_item("alice")),
+        vec!["iam-user:alice".to_string()]
+    );
+    assert_eq!(
+        iam_item_deletion_mark_entities(&policy_delete_item("readonly")),
+        vec!["policy:readonly".to_string()]
+    );
+
+    let mut group_remove = SRIAMItem {
+        r#type: "group-info".to_string(),
+        group_info: Some(SRGroupInfo {
+            update_req: GroupAddRemove {
+                group: "devs".to_string(),
+                members: vec!["bob".to_string(), "alice".to_string()],
+                status: GroupStatus::Enabled,
+                is_remove: true,
+            },
+            api_version: None,
+        }),
+        ..Default::default()
+    };
+    assert_eq!(
+        iam_item_deletion_mark_entities(&group_remove),
+        vec!["group-member:devs:bob".to_string(), "group-member:devs:alice".to_string()]
+    );
+    group_remove
+        .group_info
+        .as_mut()
+        .expect("group info")
+        .update_req
+        .members
+        .clear();
+    assert_eq!(
+        iam_item_deletion_mark_entities(&group_remove),
+        vec!["group:devs".to_string()],
+        "a removal without members deletes the group"
+    );
+    group_remove.group_info.as_mut().expect("group info").update_req.is_remove = false;
+    assert!(iam_item_deletion_mark_entities(&group_remove).is_empty());
+
+    let mapping_clear = SRIAMItem {
+        r#type: "policy-mapping".to_string(),
+        policy_mapping: Some(SRPolicyMapping {
+            user_or_group: "alice".to_string(),
+            user_type: 0,
+            is_group: false,
+            policy: String::new(),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    assert_eq!(
+        iam_item_deletion_mark_entities(&mapping_clear),
+        vec!["policy-mapping:alice:0:false".to_string()]
+    );
+
+    let mut user_create = user_delete_item("alice");
+    user_create.iam_user.as_mut().expect("iam user").is_delete_req = false;
+    assert!(iam_item_deletion_mark_entities(&user_create).is_empty());
+}
+
+/// Newest wins per entity, marks are pruned by age only (never by count: a
+/// count bound would drop a mark still inside the delivery window as soon as
+/// enough newer deletions happen), and the timestamps survive the state
+/// object as RFC 3339.
+#[test]
+fn test_record_iam_deletion_marks_newest_wins_and_expires_by_age_only() {
+    let at = |seconds: i64| OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds);
+    let now = at(1_000_000);
+    let mut state = SiteReplicationState::default();
+    let alice = vec!["iam-user:alice".to_string()];
+
+    assert!(record_iam_deletion_marks_at(&mut state, &alice, at(20), now));
+    assert!(
+        !record_iam_deletion_marks_at(&mut state, &alice, at(10), now),
+        "an older deletion does not move the mark"
+    );
+    assert!(
+        !record_iam_deletion_marks_at(&mut state, &alice, at(20), now),
+        "a replayed deletion is not a change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(20)));
+    assert!(record_iam_deletion_marks_at(&mut state, &alice, at(30), now));
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)));
+    assert_eq!(iam_deletion_mark(&state, &["iam-user:bob".to_string()]), None);
+    assert!(!record_iam_deletion_marks_at(&mut state, &[], at(40), now));
+
+    // Many newer deletions never evict an older mark that is still within the retention.
+    let members: Vec<String> = (0..4096).map(|index| format!("group-member:devs:user-{index:04}")).collect();
+    for (index, member) in members.iter().enumerate() {
+        record_iam_deletion_marks_at(&mut state, std::slice::from_ref(member), at(100 + index as i64), now);
+    }
+    assert_eq!(state.iam_deletion_marks.len(), members.len() + 1);
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)), "no count-based eviction");
+
+    // Marks older than the retention are pruned, on the pass that records a
+    // newer one and on a pass that changes nothing else; younger ones stay.
+    let later = at(100) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION;
+    assert!(
+        record_iam_deletion_marks_at(&mut state, &["iam-user:carol".to_string()], at(200_000), later),
+        "pruning alone is a change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), None, "alice's mark aged out");
+    assert_eq!(
+        iam_deletion_mark(&state, &members[..1]),
+        Some(at(100)),
+        "a mark exactly at the retention edge stays, and so do the younger ones"
+    );
+    assert_eq!(state.iam_deletion_marks.len(), members.len() + 1);
+    assert_eq!(iam_deletion_mark(&state, &["iam-user:carol".to_string()]), Some(at(200_000)));
+    let mut state = SiteReplicationState::default();
+    record_iam_deletion_marks_at(&mut state, &alice, at(30), now);
+    let past_edge = at(30) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION + time::Duration::seconds(1);
+    assert!(
+        record_iam_deletion_marks_at(&mut state, &[], at(0), past_edge),
+        "a pass that only prunes reports the change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), None);
+    record_iam_deletion_marks_at(&mut state, &alice, at(30), now);
+
+    let json = serde_json::to_value(&state).expect("serialize state");
+    assert_eq!(json["iam_deletion_marks"]["iam-user:alice"], serde_json::json!("1970-01-01T00:00:30Z"));
+    let reloaded = parse_site_replication_state(&serde_json::to_vec(&state).expect("serialize state")).expect("parse state");
+    assert_eq!(reloaded.iam_deletion_marks, state.iam_deletion_marks);
+    assert!(
+        parse_site_replication_state(br#"{"name":"a","service_account_access_key":"","service_account_parent":"","peers":{},"updated_at":null,"resync_status":{}}"#)
+            .expect("state without marks")
+            .iam_deletion_marks
+            .is_empty()
+    );
+}
+
 /// A failed deletion delivery persists a replay record next to the collapsed
 /// retry entry; a fresh entry is stamped `deletions_recorded` so a later
 /// replay can settle it, and a repeated deletion of the same entity keeps the
@@ -1679,7 +1818,8 @@ fn test_site_replication_bootstrap_plan_includes_replayable_snapshot_items() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     assert_eq!(plan.iam_items.iter().map(|item| item.r#type.as_str()).collect::<Vec<_>>(), {
         vec!["policy", "iam-user", "group-info", "policy-mapping"]
@@ -1717,7 +1857,8 @@ fn test_site_replication_bootstrap_plan_skips_lifecycle_by_default() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     assert!(!plan.bucket_items.iter().any(|item| item.r#type == "lc-config"));
 }
@@ -1748,7 +1889,8 @@ fn test_site_replication_bootstrap_plan_emits_timestamped_lifecycle_delete() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     let item = plan
         .bucket_items
@@ -1935,8 +2077,8 @@ fn test_site_replication_repair_preflight_token_is_deterministic_for_equal_state
         },
     );
 
-    let plan_a = site_replication_bootstrap_plan(&info).expect("first plan");
-    let plan_b = site_replication_bootstrap_plan(&info).expect("second plan");
+    let plan_a = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("first plan");
+    let plan_b = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("second plan");
     let token_a = site_replication_repair_preflight_token(&state, &plan_a, b"test-signing-key").expect("first token");
     let token_b = site_replication_repair_preflight_token(&state, &plan_b, b"test-signing-key").expect("second token");
 
@@ -3219,3 +3361,345 @@ fn test_reconcile_adds_missing_peer_rules_to_existing_config() {
     assert!(rule_ids.contains(&"site-repl-dep-b"));
     assert!(rule_ids.contains(&"site-repl-dep-c"));
 }
+
+/// backlog#2289: the IAM snapshot (retry resend, repair, site-add bootstrap)
+/// used to be built from `list_users`, whose `UserInfo` never carries a
+/// secret key, so the plan dropped every user and a status change or secret
+/// rotation committed while a peer was unreachable never reached it. The
+/// credentials now come from a separate store read; SRInfo stays secret-free.
+#[test]
+fn test_bootstrap_plan_carries_users_from_the_credential_snapshot() {
+    let mut info = SRInfo::default();
+    // Exactly what `list_users` builds: status, policy, updated_at — never secret_key.
+    info.user_info_map.insert(
+        "alice".to_string(),
+        rustfs_madmin::UserInfo {
+            status: rustfs_madmin::AccountStatus::Disabled,
+            policy_name: Some("readwrite".to_string()),
+            updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
+            ..Default::default()
+        },
+    );
+    info.user_info_map.insert(
+        "external-idp-user".to_string(),
+        rustfs_madmin::UserInfo {
+            status: rustfs_madmin::AccountStatus::Enabled,
+            ..Default::default()
+        },
+    );
+    let user_updated_at = OffsetDateTime::from_unix_timestamp(1_700_000_500).expect("timestamp");
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Disabled,
+            updated_at: Some(user_updated_at),
+        },
+    );
+
+    let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build");
+
+    let users: Vec<_> = plan.iam_items.iter().filter(|item| item.r#type == "iam-user").collect();
+    assert_eq!(users.len(), 1, "only the user with a credential travels: {:?}", plan.iam_items);
+    let alice = users[0].iam_user.as_ref().expect("iam user body");
+    assert_eq!(alice.access_key, "alice");
+    let req = alice.user_req.as_ref().expect("user request");
+    assert_eq!(req.secret_key, "alice-secret");
+    assert_eq!(req.status, rustfs_madmin::AccountStatus::Disabled);
+    assert_eq!(req.policy.as_deref(), Some("readwrite"));
+    // the user record's own axis, not the policy-mapping time list_users reports
+    assert_eq!(users[0].updated_at, Some(user_updated_at));
+}
+
+fn service_account_snapshot(access_key: &str, parent: &str, status: &str) -> SiteReplicationServiceAccountSnapshot {
+    SiteReplicationServiceAccountSnapshot {
+        create: rustfs_madmin::SRSvcAccCreate {
+            parent: parent.to_string(),
+            access_key: access_key.to_string(),
+            secret_key: format!("{access_key}-secret"),
+            groups: Vec::new(),
+            claims: HashMap::new(),
+            session_policy: SRSessionPolicy::default(),
+            status: status.to_string(),
+            name: String::new(),
+            description: String::new(),
+            expiration: None,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        },
+        envelope: None,
+        updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_600).expect("timestamp")),
+    }
+}
+
+/// backlog#2289: service accounts were absent from every snapshot (the
+/// listing filters them). They now travel as the create item the live hook
+/// emits — after their parents — carrying secret and status.
+#[test]
+fn test_bootstrap_plan_emits_service_accounts_after_their_parents() {
+    let mut info = SRInfo::default();
+    info.user_info_map
+        .insert("alice".to_string(), rustfs_madmin::UserInfo::default());
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Enabled,
+            updated_at: None,
+        },
+    );
+    credentials
+        .service_accounts
+        .push(service_account_snapshot("alice-svc", "alice", "off"));
+
+    let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build");
+
+    let types: Vec<_> = plan.iam_items.iter().map(|item| item.r#type.as_str()).collect();
+    assert_eq!(types, vec!["iam-user", "service-account"]);
+    let change = plan.iam_items[1].svc_acc_change.as_ref().expect("service account change");
+    let create = change.create.as_ref().expect("create body");
+    assert_eq!((create.access_key.as_str(), create.parent.as_str()), ("alice-svc", "alice"));
+    assert_eq!(create.secret_key, "alice-svc-secret");
+    assert_eq!(create.status, "off", "a disabled account must arrive disabled");
+    assert!(change.delete.is_none() && change.update.is_none());
+}
+
+/// A service account present in the previous snapshot but gone from the
+/// fresh one is replayed as an explicit delete, like the other IAM kinds.
+#[test]
+fn test_retry_snapshot_tombstones_removed_service_accounts() {
+    let observed_at = OffsetDateTime::from_unix_timestamp(1_700_001_000).expect("timestamp");
+    let mut info = SRInfo::default();
+    info.user_info_map
+        .insert("alice".to_string(), rustfs_madmin::UserInfo::default());
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Enabled,
+            updated_at: None,
+        },
+    );
+    let mut with_account = credentials.clone();
+    with_account
+        .service_accounts
+        .push(service_account_snapshot("alice-svc", "alice", "on"));
+    let previous = site_replication_bootstrap_plan(&info, &with_account).expect("previous plan");
+    let fresh = site_replication_bootstrap_plan(&info, &credentials).expect("fresh plan");
+
+    let replay = RetrySnapshot::replay_after_change(
+        &RetrySnapshot::Iam(previous.iam_items),
+        &RetrySnapshot::Iam(fresh.iam_items),
+        observed_at,
+    );
+    let RetrySnapshot::Iam(items) = replay else {
+        panic!("IAM snapshot expected");
+    };
+    let tombstone = items
+        .iter()
+        .find(|item| item.r#type == "service-account")
+        .expect("service account tombstone");
+    let change = tombstone.svc_acc_change.as_ref().expect("change");
+    assert_eq!(change.delete.as_ref().map(|delete| delete.access_key.as_str()), Some("alice-svc"));
+    assert!(change.create.is_none());
+    assert_eq!(tombstone.updated_at, Some(observed_at));
+}
+
+/// Spawns a one-shot HTTP peer that answers 200 and flips the returned flag
+/// once a request head has arrived.
+async fn spawn_reached_probe_peer() -> (String, Arc<AtomicBool>, tokio::task::JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind healthy peer");
+    let endpoint = format!("http://{}", listener.local_addr().expect("healthy peer address"));
+    let reached = Arc::new(AtomicBool::new(false));
+    let reached_by_server = reached.clone();
+    let server = tokio::spawn(async move {
+        let Ok((mut stream, _)) = listener.accept().await else {
+            return;
+        };
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        loop {
+            let Ok(read) = stream.read(&mut buffer).await else {
+                return;
+            };
+            if read == 0 {
+                return;
+            }
+            request.extend_from_slice(&buffer[..read]);
+            if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                break;
+            }
+        }
+        reached_by_server.store(true, Ordering::SeqCst);
+        let _ = stream
+            .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok")
+            .await;
+    });
+    (endpoint, reached, server)
+}
+
+/// Three-peer runtime whose local peer is `local`; BTreeMap order visits the
+/// failing peer `b` before the healthy peer `c`.
+fn broadcast_runtime_with_failing_peer_before_healthy(failing_endpoint: &str, healthy_endpoint: &str) -> SiteReplicationRuntime {
+    let local_peer = PeerInfo {
+        deployment_id: "local".to_string(),
+        ..peer("local", "http://127.0.0.1:9")
+    };
+    let mut state = SiteReplicationState {
+        name: "local".to_string(),
+        service_account_access_key: "site-replicator-0".to_string(),
+        ..Default::default()
+    };
+    state.peers.insert("local".to_string(), local_peer.clone());
+    state.peers.insert(
+        "b".to_string(),
+        PeerInfo {
+            deployment_id: "b".to_string(),
+            ..peer("b", failing_endpoint)
+        },
+    );
+    state.peers.insert(
+        "c".to_string(),
+        PeerInfo {
+            deployment_id: "c".to_string(),
+            ..peer("c", healthy_endpoint)
+        },
+    );
+    SiteReplicationRuntime {
+        state,
+        local_peer,
+        service_account_secret_key: "site-replicator-secret".to_string(),
+    }
+}
+
+const BROADCAST_PROBE_DELETE_BUCKET_PATH: &str =
+    "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket";
+
+/// The generic JSON broadcast (bucket make/delete, bucket-meta hook, bucket
+/// ops) attempts every remote peer: a peer whose request fails must not stop
+/// delivery to the peers that follow it in deployment-id order, and the
+/// failure is still reported to the caller (backlog#2293).
+#[tokio::test]
+#[serial]
+async fn test_broadcast_json_reaches_healthy_peers_after_a_failed_peer() {
+    // Peer "b": nothing listens on the port, so the connect is refused.
+    let refused = TcpListener::bind("127.0.0.1:0").await.expect("bind refused-peer probe");
+    let refused_endpoint = format!("http://{}", refused.local_addr().expect("refused-peer address"));
+    drop(refused);
+
+    let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await;
+    let runtime = broadcast_runtime_with_failing_peer_before_healthy(&refused_endpoint, &healthy_endpoint);
+
+    let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async {
+        broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await
+    })
+    .await;
+
+    let err = result.expect_err("peer b refuses connections, the broadcast must report it");
+    assert!(
+        reached.load(Ordering::SeqCst),
+        "peer c never received the broadcast once peer b failed: {err}"
+    );
+    server.abort();
+}
+
+/// Same guarantee when the failing peer never gets a transport: an endpoint
+/// that `PeerTransport::for_runtime_peer` rejects must be skipped past (and
+/// reported), not abort the broadcast before the healthy peers (backlog#2293).
+#[tokio::test]
+#[serial]
+async fn test_broadcast_json_reaches_healthy_peers_after_a_peer_without_transport() {
+    // Peer "b": a scheme the peer connection validator refuses outright.
+    let forbidden_endpoint = "ftp://peer-b.example.com";
+
+    let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await;
+    let runtime = broadcast_runtime_with_failing_peer_before_healthy(forbidden_endpoint, &healthy_endpoint);
+
+    let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async {
+        broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await
+    })
+    .await;
+
+    let err = result.expect_err("peer b has no usable transport, the broadcast must report it");
+    assert!(
+        err.to_string().contains("invalid persisted site replication peer"),
+        "the reported error must be peer b's transport failure: {err}"
+    );
+    assert!(
+        reached.load(Ordering::SeqCst),
+        "peer c never received the broadcast once peer b failed to get a transport: {err}"
+    );
+    server.abort();
+}
+
+fn service_account_item_with_claims(order: &[&str]) -> SRIAMItem {
+    let mut claims = HashMap::new();
+    for key in order {
+        claims.insert((*key).to_string(), serde_json::json!(format!("value-of-{key}")));
+    }
+    SRIAMItem {
+        r#type: "service-account".to_string(),
+        svc_acc_change: Some(SRSvcAccChange {
+            create: Some(rustfs_madmin::SRSvcAccCreate {
+                parent: "alice".to_string(),
+                access_key: "alice-svc".to_string(),
+                secret_key: "alice-svc-secret".to_string(),
+                groups: Vec::new(),
+                claims,
+                session_policy: SRSessionPolicy::default(),
+                status: "on".to_string(),
+                name: String::new(),
+                description: String::new(),
+                expiration: None,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        }),
+        updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
+        api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        ..Default::default()
+    }
+}
+
+/// The repair preflight token and the retry-snapshot fingerprint hash the
+/// serialized items. Service-account claims live in a `HashMap`, whose
+/// iteration order differs between instances, so the hash must not depend on
+/// it (the real-VM repair returned 412 "preflight is stale" between dry-run
+/// and execute once snapshots carried service accounts).
+#[test]
+fn test_repair_task_id_and_retry_fingerprint_ignore_claim_map_order() {
+    let forward = service_account_item_with_claims(&["accessKey", "exp", "parent", "sa-policy", "sub", "tenant"]);
+    let backward = service_account_item_with_claims(&["tenant", "sub", "sa-policy", "parent", "exp", "accessKey"]);
+
+    let canonical = canonical_json_vec(&forward).expect("canonical json");
+    let text = String::from_utf8(canonical).expect("utf8");
+    let positions: Vec<usize> = [
+        "\"accessKey\"",
+        "\"exp\"",
+        "\"parent\"",
+        "\"sa-policy\"",
+        "\"sub\"",
+        "\"tenant\"",
+    ]
+    .iter()
+    .map(|key| text.find(key).expect("claim key present"))
+    .collect();
+    assert!(
+        positions.windows(2).all(|pair| pair[0] < pair[1]),
+        "claim keys must serialize sorted: {text}"
+    );
+
+    assert_eq!(
+        SiteReplicationRepairTask::Iam(&forward).id().expect("id"),
+        SiteReplicationRepairTask::Iam(&backward).id().expect("id"),
+        "identical items must yield the same repair task id regardless of claim map order"
+    );
+    assert_eq!(
+        RetrySnapshot::Iam(vec![forward]).fingerprint().expect("fingerprint"),
+        RetrySnapshot::Iam(vec![backward]).fingerprint().expect("fingerprint"),
+        "identical snapshots must fingerprint equal regardless of claim map order"
+    );
+}

rustfs/src/site_replication/transport.rs

@@ -876,6 +876,14 @@ pub(crate) async fn broadcast_site_replication_json<T: Serialize>(path: &str, bo
     broadcast_site_replication_json_with_runtime(&runtime, path, body).await
 }
 
+/// PUT `body` to `path` on every remote peer of the runtime.
+///
+/// Every peer is attempted: one peer's failure — transport construction
+/// included — must not skip the peers that follow it in deployment-id order,
+/// or they silently miss the change with no retry record (backlog#2293). A
+/// success settles the peer/path's queued retry event, a failure enqueues one
+/// under the request `path` (so the drain classifies it as today), and the
+/// first error is returned once all peers were attempted.
 pub(crate) async fn broadcast_site_replication_json_with_runtime<T: Serialize>(
     runtime: &SiteReplicationRuntime,
     path: &str,
@@ -883,20 +891,30 @@ pub(crate) async fn broadcast_site_replication_json_with_runtime<T: Serialize>(
 ) -> S3Result<()> {
     let state = &runtime.state;
     let local_peer = &runtime.local_peer;
+    let mut first_error: Option<S3Error> = None;
 
     for peer in state.peers.values() {
         if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
             continue;
         }
 
-        let transport = PeerTransport::for_runtime_peer(peer).await?;
-        PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key)
-            .with_client(&transport.client)
-            .send_with_retry_event(peer, &runtime.service_account_secret_key, body)
-            .await?;
+        let sent = match PeerTransport::for_runtime_peer(peer).await {
+            Ok(transport) => PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key)
+                .with_client(&transport.client)
+                .send_with_retry_event(peer, &runtime.service_account_secret_key, body)
+                .await
+                .map(|_| ()),
+            Err(err) => {
+                enqueue_site_replication_retry_event(peer, path, &err).await;
+                Err(err)
+            }
+        };
+        if let Err(err) = sent {
+            first_error.get_or_insert(err);
+        }
     }
 
-    Ok(())
+    first_error.map_or(Ok(()), Err)
 }
 
 pub(crate) fn parse_endpoint_refresh_status(peer: &PeerInfo, body: &[u8]) -> S3Result<()> {

rustfs/src/storage/rpc/node_service.rs

@@ -3052,6 +3052,136 @@ mod tests {
         assert_eq!(err.code(), tonic::Code::InvalidArgument);
     }
 
+    fn heal_start_retry_fixture() -> (
+        Arc<HealManager>,
+        rustfs_heal_contracts::heal_channel::HealChannelRequest,
+        rustfs_protos::heal_control::RequestMetadata,
+    ) {
+        let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None));
+        let mut request = rustfs_heal_contracts::heal_channel::create_heal_request(
+            "bucket".to_string(),
+            Some("prefix".to_string()),
+            true,
+            None,
+        );
+        request.source = rustfs_heal_contracts::heal_channel::HealRequestSource::Admin;
+        request.recursive = Some(true);
+        let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).expect("fixture clock fits in i64");
+        let metadata = rustfs_protos::heal_control::RequestMetadata::new(*Uuid::new_v4().as_bytes(), now, now + 30_000, 7);
+        (manager, request, metadata)
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_exact_forced_envelope_returns_cached_admission() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let request_id = request.id.clone();
+        let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("valid forced start");
+        let lost_response =
+            execute_heal_control_envelope_with_manager(envelope.clone(), metadata.coordinator_epoch, Some(manager.clone()))
+                .await
+                .expect("first request is admitted before its response is lost");
+        assert_eq!(manager.operations_snapshot().await.queue_length, 1);
+
+        // The caller sees no first response, but retries the original envelope.
+        let replayed = execute_heal_control_envelope_with_manager(envelope, metadata.coordinator_epoch, Some(manager.clone()))
+            .await
+            .expect("an exact envelope replay must recover its receipt");
+        assert_eq!(replayed, lost_response);
+        assert_eq!(
+            manager.operations_snapshot().await.queue_length,
+            1,
+            "forceStart must not be executed twice"
+        );
+        let outcome = rustfs_protos::heal_control::decode_result(&replayed)
+            .and_then(|result| result.into_outcome(&request_id, metadata.coordinator_epoch))
+            .expect("matching canonical receipt");
+        assert!(matches!(outcome, rustfs_protos::heal_control::Outcome::Start {
+            task_id, admission: rustfs_protos::heal_control::Admission::Accepted,
+        } if task_id == request_id));
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_new_forced_request_is_a_distinct_start() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let first_id = request.id.clone();
+        let first = rustfs_protos::heal_control::Envelope::start(request.clone(), metadata).expect("first start");
+        let _lost_response = execute_heal_control_envelope_with_manager(first, metadata.coordinator_epoch, Some(manager.clone()))
+            .await
+            .expect("first admission");
+
+        // A fresh HTTP forceStart request intentionally requests another start.
+        let mut next_request = request;
+        next_request.id = Uuid::new_v4().to_string();
+        let next_id = next_request.id.clone();
+        let next_metadata = rustfs_protos::heal_control::RequestMetadata {
+            nonce: *Uuid::new_v4().as_bytes(),
+            ..metadata
+        };
+        let next = rustfs_protos::heal_control::Envelope::start(next_request, next_metadata).expect("new forced start");
+        let response = execute_heal_control_envelope_with_manager(next, metadata.coordinator_epoch, Some(manager.clone()))
+            .await
+            .expect("forceStart preserves its explicit admission semantics");
+        let outcome = rustfs_protos::heal_control::decode_result(&response)
+            .and_then(|result| result.into_outcome(&next_id, metadata.coordinator_epoch))
+            .expect("new receipt");
+        assert!(matches!(outcome, rustfs_protos::heal_control::Outcome::Start {
+            task_id, admission: rustfs_protos::heal_control::Admission::Accepted,
+        } if task_id == next_id && task_id != first_id));
+        assert_eq!(
+            manager.operations_snapshot().await.queue_length,
+            2,
+            "a caller must not treat a new forced request as an idempotent transport retry"
+        );
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_same_id_with_changed_envelope_conflicts_before_admission() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let original = rustfs_protos::heal_control::Envelope::start(request.clone(), metadata).expect("original start");
+        let receipt =
+            execute_heal_control_envelope_with_manager(original.clone(), metadata.coordinator_epoch, Some(manager.clone()))
+                .await
+                .expect("original admission");
+        let mut changed_options = request.clone();
+        changed_options.remove_corrupted = Some(true);
+        let changed_metadata = rustfs_protos::heal_control::RequestMetadata {
+            nonce: *Uuid::new_v4().as_bytes(),
+            ..metadata
+        };
+        for changed in [
+            rustfs_protos::heal_control::Envelope::start(changed_options, metadata).expect("changed options"),
+            rustfs_protos::heal_control::Envelope::start(request, changed_metadata).expect("changed nonce"),
+        ] {
+            let error = execute_heal_control_envelope_with_manager(changed, metadata.coordinator_epoch, Some(manager.clone()))
+                .await
+                .expect_err("one request ID cannot identify different envelope bytes");
+            assert_eq!(error.code(), tonic::Code::AlreadyExists);
+            assert_eq!(manager.operations_snapshot().await.queue_length, 1);
+        }
+        assert_eq!(
+            execute_heal_control_envelope_with_manager(original, metadata.coordinator_epoch, Some(manager))
+                .await
+                .expect("conflicts must preserve the original receipt"),
+            receipt
+        );
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_wrong_coordinator_epoch_cannot_admit_locally() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let request_id = request.id.clone();
+        let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("start envelope");
+        let error = execute_heal_control_envelope_with_manager(envelope, metadata.coordinator_epoch + 1, Some(manager.clone()))
+            .await
+            .expect_err("a different coordinator epoch cannot accept the request");
+        assert_eq!(error.code(), tonic::Code::FailedPrecondition);
+        assert_eq!(manager.operations_snapshot().await.queue_length, 0);
+        assert!(matches!(
+            manager.get_task_status(&request_id).await,
+            Err(rustfs_heal::Error::TaskNotFound { .. })
+        ));
+    }
+
     #[tokio::test]
     async fn heal_control_executor_preserves_canonical_token_and_drops_query_results() {
         let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None));