chore: integrate current main for namespace target validation

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
 sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
-sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
+sha256-darwin=53b05ac745905809d3828c6994bdd8ecf9d20b2b61a8a9d80fe15eb62f932193
+sha256-linux=7c892afa4b9d1591b46bd79c976b647109a277284fddb3b98edced4b0297eda2

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15
+sha256=5db88c6fec94d4f269c7d9cfc128bd2adc27b3d7021127e2fa0b1daccc5f900f

.github/workflows/e2e-distributed.yml

@@ -22,11 +22,13 @@
 # Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
 #
 # Isolated pool filesystems: expand/decommission/rebalance cases require
-# independent `statfs` capacity. `sm-standard-4` is an ARC pod
-# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no
-# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed:
-# No such file or directory"). The prepare step therefore mounts four 1 GiB
-# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`.
+# independent `statfs` capacity. This job runs on GitHub-hosted
+# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot
+# create filesystems: `mount -o loop` fails with ENOENT (no
+# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs
+# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason
+# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare
+# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`.
 
 name: e2e-distributed
 
@@ -76,7 +78,9 @@ concurrency:
 jobs:
   distributed:
     name: Distributed 4-node 4-disk e2e
-    runs-on: sm-standard-4
+    # GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
+    # ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
+    runs-on: ubuntu-latest
     timeout-minutes: 180
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
@@ -97,7 +101,9 @@ jobs:
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-e2e-distributed
+          # Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so
+          # a shared key would mix VM and ARC pod target/ artifacts.
+          cache-shared-key: ci-e2e-distributed-hosted
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: 'false'
 
@@ -110,10 +116,14 @@ jobs:
           for pool in 0 1 2 3; do
             mountpoint="${mount_base}/pool-${pool}"
             mkdir -p "${mountpoint}"
-            # sm-standard-4 is an ARC pod without usable loop devices, so
-            # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a
-            # distinct st_dev and independent 1G statfs capacity.
-            sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"
+            # Sized tmpfs reports a distinct st_dev and independent 1G
+            # statfs capacity. Requires a VM runner (ubuntu-latest).
+            if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then
+              echo "tmpfs mount failed on $(uname -a)" >&2
+              findmnt || true
+              grep Cap /proc/self/status || true
+              exit 1
+            fi
             sudo chmod 1777 "${mountpoint}"
             roots+=("${mountpoint}")
           done

crates/config/README.md

@@ -172,6 +172,14 @@ Drive timeout profile preset:
   - Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
   - Then the profile-derived default (`default` or `high_latency`).
 
+## Admin peer probe timeout
+
+- `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS`
+  - total per-peer budget for the `server_info`/`storage_info` admin probe round; `server_info` may reconnect once and `storage_info` remains a single attempt.
+  - default is `10` seconds, preserving the previous two-attempt worst-case budget.
+  - values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
+  - the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
+
 ## Startup filesystem boundary policy
 
 - `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.

crates/config/src/constants/internode.rs

@@ -39,6 +39,15 @@ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20;
 pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
 pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30;
 
+/// Total budget for one admin peer probe round, including any reconnect retry.
+///
+/// This is intentionally separate from the transport-level RPC timeout: admin
+/// probes may retry once, but the retry must consume the same round budget.
+pub const ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS: &str = "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS";
+pub const DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 10;
+pub const MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS: u64 = 60;
+const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS);
+
 // ── Client-side internode gRPC channel tuning (P0) ──
 // These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs`
 // on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to
@@ -312,6 +321,7 @@ mod tests {
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20);
         assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30);
+        assert_eq!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS, 10);
         assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
     }
 
@@ -412,6 +422,7 @@ mod tests {
             "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
         );
         assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
+        assert_eq!(ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, "RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS");
         assert_eq!(ENV_INTERNODE_HTTP_TUNING_PROFILE, "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE");
         assert_eq!(ENV_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST, "RUSTFS_INTERNODE_HTTP_POOL_MAX_IDLE_PER_HOST");
         assert_eq!(ENV_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS, "RUSTFS_INTERNODE_HTTP_POOL_IDLE_TIMEOUT_SECS");

crates/e2e_test/src/fake_s3_target/mod.rs

@@ -451,10 +451,40 @@ impl JournaledHeaders {
 struct ControlState {
     scripts: HashMap<Operation, VecDeque<FaultAction>>,
     keyed_scripts: HashMap<(Operation, String), VecDeque<FaultAction>>,
+    held_get: Option<HeldGetObject>,
     requests: VecDeque<RequestRecord>,
     next_sequence: u64,
 }
 
+#[derive(Clone)]
+struct HeldGetObject {
+    bucket: String,
+    key: String,
+    entered: watch::Sender<usize>,
+    released: watch::Receiver<bool>,
+}
+
+/// Holds every GET of one object, including retries, until this guard is dropped.
+#[must_use = "dropping the guard releases the held GET requests"]
+pub struct GetObjectGate {
+    control: Arc<Mutex<ControlState>>,
+    entered: watch::Receiver<usize>,
+    released: watch::Sender<bool>,
+}
+
+impl GetObjectGate {
+    pub async fn wait_until_entered(&mut self) -> Result<(), watch::error::RecvError> {
+        self.entered.wait_for(|count| *count > 0).await.map(|_| ())
+    }
+}
+
+impl Drop for GetObjectGate {
+    fn drop(&mut self) {
+        lock(&self.control).held_get = None;
+        self.released.send_replace(true);
+    }
+}
+
 #[derive(Default)]
 struct StoreState {
     assign_own_version_ids: bool,
@@ -936,6 +966,30 @@ impl FakeS3Target {
             .extend(std::iter::repeat_n(action, times));
     }
 
+    /// Hold one exact bucket/key before any GET response can reach the client.
+    /// The fixture supports one live gate; request and connection deadlines still apply.
+    pub fn hold_get_object(&self, bucket: &str, key: &str) -> GetObjectGate {
+        assert!(
+            bucket.len() <= MAX_RETAINED_IDENTIFIER_BYTES && key.len() <= MAX_RETAINED_IDENTIFIER_BYTES,
+            "held GET identifiers exceed the fixture limit"
+        );
+        let mut state = lock(&self.control);
+        assert!(state.held_get.is_none(), "fake target already holds a GET gate");
+        let (entered, entered_rx) = watch::channel(0);
+        let (released, released_rx) = watch::channel(false);
+        state.held_get = Some(HeldGetObject {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            entered,
+            released: released_rx,
+        });
+        GetObjectGate {
+            control: Arc::clone(&self.control),
+            entered: entered_rx,
+            released,
+        }
+    }
+
     pub fn clear_faults(&self) {
         let mut state = lock(&self.control);
         state.scripts.clear();
@@ -2243,6 +2297,17 @@ impl S3 for FakeBackend {
         let fault = request_fault(&req);
         apply_non_body_fault(fault.as_ref(), &self.control).await?;
         let input = req.input;
+        let held_get = lock(&self.control)
+            .held_get
+            .as_ref()
+            .filter(|held| held.bucket == input.bucket && held.key == input.key)
+            .cloned();
+        if let Some(mut held) = held_get {
+            held.entered.send_modify(|count| *count += 1);
+            // Keep the gate installed when a request is cancelled or times out:
+            // a retry must cross the same boundary before returning any bytes.
+            let _ = held.released.wait_for(|released| *released).await;
+        }
         let (version, versioned) = {
             let state = lock(&self.store);
             (
@@ -2894,6 +2959,81 @@ mod tests {
         aws_sdk_s3::primitives::DateTime::from_secs(4_102_444_800)
     }
 
+    #[tokio::test]
+    async fn get_object_gate_holds_retries_and_releases_on_drop() -> Result<(), BoxError> {
+        let target = FakeS3Target::start().await?;
+        let bucket = "gated-target";
+        target.create_bucket(bucket);
+        for key in ["held", "unrelated"] {
+            target.put_seed_object(bucket, key, Bytes::from_static(b"payload"), &SeedMetadata::default());
+        }
+        {
+            let gate = target.hold_get_object(bucket, "held");
+            let request = || S3Request {
+                input: GetObjectInput {
+                    bucket: bucket.to_string(),
+                    key: "held".to_string(),
+                    ..Default::default()
+                },
+                method: Method::GET,
+                uri: Uri::from_static("/gated-target/held"),
+                headers: HeaderMap::new(),
+                extensions: http::Extensions::new(),
+                credentials: None,
+                region: None,
+                service: None,
+                trailing_headers: None,
+            };
+            // Without a fault, only the gate can suspend this backend method.
+            let mut first = target.backend.get_object(request());
+            assert!(futures::poll!(first.as_mut()).is_pending(), "the first GET must wait at the gate");
+            drop(first);
+            let mut retry = target.backend.get_object(request());
+            assert!(futures::poll!(retry.as_mut()).is_pending(), "a cancelled GET must not consume the gate");
+            drop(gate);
+            let std::task::Poll::Ready(response) = futures::poll!(retry.as_mut()) else {
+                panic!("dropping the gate must release the waiting GET");
+            };
+            let mut body = response?.output.body.expect("released GET body");
+            assert_eq!(body.next().await.transpose()?, Some(Bytes::from_static(b"payload")));
+            assert!(body.next().await.is_none(), "released GET body must be complete");
+        }
+        let client = client(&target);
+        let mut gate = target.hold_get_object(bucket, "held");
+        let mut requests = tokio::task::JoinSet::new();
+        let first = client.clone();
+        requests.spawn(async move { get_bytes(&first, bucket, "held", None).await });
+        timeout(Duration::from_secs(2), gate.wait_until_entered()).await??;
+        requests.abort_all();
+        assert!(
+            requests
+                .join_next()
+                .await
+                .expect("first GET task")
+                .expect_err("cancel the first GET attempt")
+                .is_cancelled()
+        );
+
+        let retry = client.clone();
+        requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
+        timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
+        assert_eq!(
+            timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
+            Bytes::from_static(b"payload")
+        );
+        assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
+        drop(gate);
+        assert_eq!(
+            timeout(Duration::from_secs(2), requests.join_next())
+                .await?
+                .expect("retried GET task")??,
+            Bytes::from_static(b"payload")
+        );
+        assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
+        assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
+        Ok(())
+    }
+
     #[tokio::test]
     async fn object_lock_target_requires_a_checksum_on_locked_puts() -> Result<(), BoxError> {
         use aws_sdk_s3::error::ProvideErrorMetadata;

crates/e2e_test/src/on_demand_migration/interaction_test.rs

@@ -22,8 +22,8 @@
 //! local object and what the source was asked for.
 
 use super::common::{
-    AdminResponse, BoxError, OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env,
-    start_configured_env_with,
+    ALLOW_LOOPBACK_SOURCE_ENV, AdminResponse, BackfillOp, BackfillRequest, BoxError, ODM_MODULE_SWITCH_ENV, ODM_SERVER_ENV,
+    OdmEnvOptions, OdmSourceSpec, OdmTestEnv, SeedObject, start_configured_env, start_configured_env_with,
 };
 use crate::common::{RustFSTestEnvironment, replication_fast_env, signed_request};
 use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation};
@@ -32,7 +32,7 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{
     BucketVersioningStatus, Event, FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter,
     ObjectLockRetentionMode, QueueConfiguration, S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault,
-    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, VersioningConfiguration,
+    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
 };
 use bytes::Bytes;
 use local_ip_address::local_ip;
@@ -733,6 +733,242 @@ async fn test_odm_disable_keeps_pulled_objects_and_stops_source_traffic() -> Tes
     Ok(())
 }
 
+/// The process switch preserves configured buckets and unfinished jobs while
+/// restoring local-only S3 behavior, including after an ordinary metadata write.
+#[tokio::test]
+async fn test_odm_global_disable_preserves_data_config_and_backfill_across_restarts() -> TestResult {
+    let bucket = "odm-global-disable";
+    let mut env = start_configured_env(bucket, SOURCE_BUCKET, |spec| spec.policy.list_through = true).await?;
+    let pulled_key = "migrated/pulled.bin";
+    let remote_key = "remote/untouched.bin";
+    let pending_key = "backfill/pending.bin";
+    let local_key = "local/kept.bin";
+    let source_body = Bytes::from_static(b"source payload");
+    let local_body = Bytes::from_static(b"client payload");
+    env.seed_source(
+        SOURCE_BUCKET,
+        &[
+            SeedObject::new(pulled_key, source_body.clone()),
+            SeedObject::new(remote_key, source_body.clone()),
+            SeedObject::new(pending_key, source_body.clone()),
+        ],
+    );
+    env.client
+        .put_object()
+        .bucket(bucket)
+        .key(local_key)
+        .body(local_body.clone().into())
+        .send()
+        .await?;
+    let pulled = env.raw_get(bucket, pulled_key).await?;
+    assert_eq!(pulled.status, 200);
+    assert_eq!(pulled.header(ODM_RESPONSE_HEADER), Some("source"));
+    assert_eq!(pulled.body, source_body);
+    let stored = env.raw_get(bucket, pulled_key).await?;
+    assert_eq!(stored.status, 200);
+    assert_eq!(stored.header(ODM_RESPONSE_HEADER), None, "the inline pull has committed locally");
+    assert_eq!(stored.body, source_body);
+    let config = env.get_config(bucket).await?;
+    assert_eq!(config.status, 200, "{}", config.body);
+    let config = config.json()?;
+
+    // Hold every attempt until the process has exited, so retries cannot commit
+    // the only backfill object before the crash. The start checkpoint exists.
+    let mut pending_get = env.source.hold_get_object(SOURCE_BUCKET, pending_key);
+    let started = env
+        .start_backfill(
+            bucket,
+            BackfillRequest {
+                prefix: Some("backfill/".to_string()),
+                ..BackfillRequest::default()
+            },
+        )
+        .await?;
+    assert_eq!(started.status, 200, "{}", started.body);
+    let job_id = started.json()?["job"]["job_id"].as_str().ok_or("missing job ID")?.to_string();
+    tokio::time::timeout(Duration::from_secs(10), pending_get.wait_until_entered())
+        .await
+        .expect("backfill never reached the held source GET")?;
+    let process = env.rustfs.process.as_mut().ok_or("missing RustFS process before crash")?;
+    assert!(process.try_wait()?.is_none(), "RustFS exited before the controlled crash");
+    process.kill()?;
+    let stopped = process.wait()?;
+    assert!(!stopped.success(), "the interrupted process must exit after being killed");
+    drop(env.rustfs.process.take());
+    drop(pending_get);
+    env.source.take_requests();
+    env.rustfs
+        .restart_server_preserving_data(vec![], &[(ODM_MODULE_SWITCH_ENV, "false"), (ALLOW_LOOPBACK_SOURCE_ENV, "true")])
+        .await?;
+
+    let off_config = env.get_config(bucket).await?;
+    assert_eq!(off_config.status, 200, "{}", off_config.body);
+    assert_eq!(off_config.json()?, config, "the saved configuration and timestamp survive disabling");
+    let status = env.status_json(bucket).await?;
+    assert_eq!(status["configured"], true, "{status}");
+    assert_eq!(status["enabled"], true, "the bucket remains configured as enabled: {status}");
+    assert_eq!(status["module_enabled"], false, "{status}");
+    assert_eq!(status["counters"], Value::Null, "no bucket runtime is installed: {status}");
+    let checkpoint = env.backfill_job(bucket).await?.ok_or("disabled module lost the checkpoint")?;
+    assert_eq!(checkpoint["job_id"], job_id);
+    assert_eq!(checkpoint["state"], "running", "the interrupted job is retained: {checkpoint}");
+
+    for (key, body) in [(local_key, &local_body), (pulled_key, &source_body)] {
+        let get = env.raw_get(bucket, key).await?;
+        assert_eq!(get.status, 200);
+        assert_eq!(&get.body, body);
+        assert_eq!(get.header(ODM_RESPONSE_HEADER), None);
+        let head = env.client.head_object().bucket(bucket).key(key).send().await?;
+        assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
+    }
+    for key in [remote_key, pending_key] {
+        let get = env.raw_get(bucket, key).await?;
+        assert_eq!(get.status, 404, "disabled source GET {key}: {}", String::from_utf8_lossy(&get.body));
+        let head = env.client.head_object().bucket(bucket).key(key).send().await;
+        let err = head.expect_err("a source-only object must remain absent locally");
+        assert_eq!(err.raw_response().map(|response| response.status().as_u16()), Some(404));
+    }
+
+    let replacement = Bytes::from_static(b"written while the module is off");
+    for key in [local_key, "local/deleted.bin"] {
+        env.client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(replacement.clone().into())
+            .send()
+            .await?;
+    }
+    env.client
+        .delete_object()
+        .bucket(bucket)
+        .key("local/deleted.bin")
+        .send()
+        .await?;
+    assert_eq!(env.raw_get(bucket, "local/deleted.bin").await?.status, 404);
+    assert_eq!(env.raw_get(bucket, local_key).await?.body, replacement);
+
+    // Both wire protocols must finish their local pages even though the saved
+    // configuration still requests list-through.
+    for use_v2 in [false, true] {
+        let mut cursor = None;
+        let mut listed = Vec::new();
+        for page_number in 0..2 {
+            let (keys, truncated, next) = if use_v2 {
+                let page = env
+                    .client
+                    .list_objects_v2()
+                    .bucket(bucket)
+                    .max_keys(1)
+                    .set_continuation_token(cursor)
+                    .send()
+                    .await?;
+                (
+                    page.contents()
+                        .iter()
+                        .map(|object| object.key().expect("listed key").to_string())
+                        .collect::<Vec<_>>(),
+                    page.is_truncated(),
+                    page.next_continuation_token().map(str::to_string),
+                )
+            } else {
+                let page = env
+                    .client
+                    .list_objects()
+                    .bucket(bucket)
+                    .max_keys(1)
+                    .set_marker(cursor)
+                    .send()
+                    .await?;
+                // V1 may omit NextMarker without a delimiter; clients then
+                // continue from the last returned key.
+                let next = page.next_marker().or_else(|| {
+                    if page.is_truncated() == Some(true) {
+                        page.contents().last().and_then(|object| object.key())
+                    } else {
+                        None
+                    }
+                });
+                (
+                    page.contents()
+                        .iter()
+                        .map(|object| object.key().expect("listed key").to_string())
+                        .collect::<Vec<_>>(),
+                    page.is_truncated(),
+                    next.map(str::to_string),
+                )
+            };
+            assert_eq!(keys.len(), 1, "one local key per page, V2={use_v2}");
+            assert_eq!(truncated, Some(page_number == 0), "local pagination must terminate, V2={use_v2}");
+            if page_number == 0 {
+                assert!(next.as_ref().is_some_and(|value| !value.is_empty()), "missing local cursor, V2={use_v2}");
+            }
+            cursor = next;
+            listed.extend(keys);
+        }
+        assert_eq!(listed, [local_key, pulled_key], "source-only keys must stay absent, V2={use_v2}");
+    }
+
+    let spec = env.fake_source_spec(SOURCE_BUCKET);
+    for response in [
+        env.configure_source(bucket, &spec).await?,
+        env.validate_source(bucket, &spec).await?,
+        env.backfill(bucket, BackfillOp::Start(BackfillRequest::default())).await?,
+    ] {
+        assert_eq!(response.status, 400, "{}", response.body);
+        assert!(response.body.contains("OnDemandMigrationDisabled"), "{}", response.body);
+    }
+    let tagging = Tagging::builder()
+        .tag_set(Tag::builder().key("module").value("disabled").build()?)
+        .build()?;
+    env.client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(tagging.clone())
+        .send()
+        .await?;
+    assert_eq!(env.get_config(bucket).await?.json()?, config, "an unrelated metadata write preserves ODM");
+    assert_eq!(
+        env.backfill_job(bucket).await?,
+        Some(checkpoint),
+        "no recovery or checkpoint update while disabled"
+    );
+    assert!(
+        env.source.requests().is_empty(),
+        "disabled startup and all requests must leave the source untouched"
+    );
+
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    env.wait_until_source_consulted(bucket).await?;
+    assert_eq!(
+        env.get_config(bucket).await?.json()?,
+        config,
+        "reenabling uses the persisted configuration"
+    );
+    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
+    assert_eq!(tags.tag_set(), tagging.tag_set(), "the ordinary metadata write also persists");
+    let resumed = env.raw_get(bucket, remote_key).await?;
+    assert_eq!(resumed.status, 200);
+    assert_eq!(resumed.header(ODM_RESPONSE_HEADER), Some("source"));
+    assert_eq!(resumed.body, source_body, "stored credentials still authenticate without reconfiguration");
+    let completed = env
+        .wait_for_backfill(bucket, SETTLE, |job| job["state"] == "completed")
+        .await?;
+    assert_eq!(completed["job_id"], job_id, "the interrupted job resumes without a new start");
+    assert_eq!(completed["failed"], 0, "{completed}");
+    for (key, body) in [
+        (local_key, &replacement),
+        (pulled_key, &source_body),
+        (pending_key, &source_body),
+    ] {
+        let get = env.raw_get(bucket, key).await?;
+        assert_eq!(get.status, 200);
+        assert_eq!(&get.body, body);
+        assert_eq!(get.header(ODM_RESPONSE_HEADER), None, "{key} remains stored locally");
+    }
+    Ok(())
+}
+
 /// Case 19: the admin surface an operator sees — the configuration read back
 /// without its secret, and a status document whose counters match the source
 /// journal exactly.

crates/ecstore/src/api/mod.rs

@@ -159,15 +159,17 @@ pub mod bucket {
             BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
             ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
             acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
-            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
-            get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
-            get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
-            get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
+            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_if_incarnation_at,
+            delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw,
+            get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
+            get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
+            get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
             get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
             get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
             init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
             update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
-            update_quota_if_incarnation, update_under_transaction_lock,
+            update_if_incarnation_at, update_quota_if_incarnation, update_quota_if_incarnation_at, update_under_transaction_lock,
+            update_under_transaction_lock_at,
         };
         #[cfg(feature = "test-util")]
         pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -35,6 +35,10 @@ use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::durable_namespace::{
     TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, validate_durable_ilm_record,
 };
+use crate::bucket::lifecycle::recovery_control::{
+    IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
+    load_recovery_control, observe_recovery_source, save_recovery_control_if_absent,
+};
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
     Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -78,6 +82,13 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32;
 const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16;
 const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32;
 const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
+const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1";
+const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2";
+const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown";
+const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1";
+const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2";
+const TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS: &str = "tier_delete_journal_corrupt";
+const CORRUPT_TIER_DELETE_JOURNAL_IDENTITY: &str = "corrupt";
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
@@ -5509,6 +5520,125 @@ enum TierDeleteJournalEntryRecoveryOutcome {
     Failed,
 }
 
+fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&str> {
+    let identity = object_name
+        .strip_prefix(TIER_DELETE_JOURNAL_LEGACY_PREFIX)?
+        .strip_suffix(".json")?;
+    (rustfs_utils::crypto::is_sha256_checksum(identity)
+        && !identity
+            .bytes()
+            .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()))
+    .then_some(identity)
+}
+
+fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> {
+    match entry.persisted_version {
+        1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)),
+        TIER_DELETE_JOURNAL_VERSION => Some((TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS)),
+        _ => None,
+    }
+}
+
+fn legacy_tier_delete_control_matches(
+    control: &IlmRecoveryControl,
+    identity: &IlmRecoveryControlIdentity,
+    generation: &crate::bucket::lifecycle::recovery_control::IlmRecoverySourceGeneration,
+    classification: IlmRecoveryClassification,
+    error_code: IlmRecoveryErrorCode,
+) -> bool {
+    control.identity == *identity
+        && control.observed_source_generation == *generation
+        && control.classification == classification
+        && control.last_error_code == error_code
+        && control.owner.is_none()
+        && control.attempt_count == 0
+        && control.consecutive_failure_count == 0
+}
+
+fn legacy_tier_delete_control_is_scheduler_fence(control: &IlmRecoveryControl, identity: &IlmRecoveryControlIdentity) -> bool {
+    control.identity == *identity && control.owner.is_none() && !control.classification.permits_automatic_attempt()
+}
+
+async fn persist_legacy_tier_delete_recovery_control(
+    api: Arc<ECStore>,
+    object_name: &str,
+    observed_data: &[u8],
+    stable_operation_identity: String,
+    (source_schema, record_class): (&'static str, &'static str),
+    intended_classification: IlmRecoveryClassification,
+    intended_error_code: IlmRecoveryErrorCode,
+) -> Result<()> {
+    let identity = IlmRecoveryControlIdentity {
+        protocol: IlmRecoveryProtocol::TierDeleteJournal,
+        canonical_source_path: object_name.to_string(),
+        stable_operation_identity,
+        record_class: record_class.to_string(),
+    };
+    let control_id = identity.source_operation_digest().map_err(Error::other)?;
+    match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
+        Ok(observed) if legacy_tier_delete_control_is_scheduler_fence(&observed.control, &identity) => return Ok(()),
+        Ok(_) => return Err(Error::PreconditionFailed),
+        Err(Error::ConfigNotFound) => {}
+        Err(err) => return Err(err),
+    }
+
+    let source = observe_recovery_source(api.clone(), object_name, source_schema).await?;
+    let exact_source = source.is_consistent() && source.canonical_data.as_deref() == Some(observed_data);
+    let (classification, error_code) = if exact_source {
+        (intended_classification, intended_error_code)
+    } else {
+        (IlmRecoveryClassification::Corrupt, IlmRecoveryErrorCode::SourceDivergent)
+    };
+    let candidate = IlmRecoveryControl::new(
+        identity.clone(),
+        source.generation.clone(),
+        classification,
+        i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos())
+            .map_err(|_| Error::other("tier delete journal recovery timestamp does not fit i64"))?,
+        error_code,
+    )
+    .map_err(Error::other)?;
+
+    match save_recovery_control_if_absent(api.clone(), &candidate).await {
+        Ok(()) | Err(Error::PreconditionFailed) => {}
+        Err(save_error) => match load_recovery_control(api.clone(), IlmRecoveryProtocol::TierDeleteJournal, &control_id).await {
+            Ok(observed)
+                if legacy_tier_delete_control_matches(
+                    &observed.control,
+                    &identity,
+                    &source.generation,
+                    classification,
+                    error_code,
+                ) =>
+            {
+                return Ok(());
+            }
+            Ok(_) | Err(_) => return Err(save_error),
+        },
+    }
+
+    let observed = load_recovery_control(api, IlmRecoveryProtocol::TierDeleteJournal, &control_id).await?;
+    if !legacy_tier_delete_control_matches(&observed.control, &identity, &source.generation, classification, error_code) {
+        return Err(Error::PreconditionFailed);
+    }
+    Ok(())
+}
+
+async fn retain_corrupt_legacy_tier_delete_journal(api: Arc<ECStore>, object_name: &str, data: &[u8]) -> Result<()> {
+    canonical_legacy_tier_delete_journal_identity(object_name)
+        .ok_or_else(|| Error::other("tier delete journal path is not canonical"))?;
+    persist_legacy_tier_delete_recovery_control(
+        api,
+        object_name,
+        data,
+        CORRUPT_TIER_DELETE_JOURNAL_IDENTITY.to_string(),
+        (TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_CORRUPT_RECOVERY_CLASS),
+        IlmRecoveryClassification::Corrupt,
+        IlmRecoveryErrorCode::SourceCorrupt,
+    )
+    .await
+}
+
 async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: String) -> TierDeleteJournalEntryRecoveryOutcome {
     let data = match config_boundary::read_config(api.clone(), &object_name).await {
         Ok(data) => data,
@@ -5529,6 +5659,22 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
     let je = match decode_tier_delete_journal_entry(&data) {
         Ok(je) => je,
         Err(err) => {
+            if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
+                return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
+                    Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
+                    Err(control_error) => {
+                        warn!(
+                            event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+                            component = LOG_COMPONENT_ECSTORE,
+                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                            journal_object = %object_name,
+                            error = ?control_error,
+                            "Failed to retain corrupt tier delete journal recovery control"
+                        );
+                        TierDeleteJournalEntryRecoveryOutcome::Failed
+                    }
+                };
+            }
             warn!(
                 event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
                 component = LOG_COMPONENT_ECSTORE,
@@ -5542,6 +5688,22 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
     };
 
     if tier_delete_journal_object_name(&je) != object_name {
+        if canonical_legacy_tier_delete_journal_identity(&object_name).is_some() {
+            return match retain_corrupt_legacy_tier_delete_journal(api, &object_name, &data).await {
+                Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
+                Err(err) => {
+                    warn!(
+                        event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                        journal_object = %object_name,
+                        error = ?err,
+                        "Failed to retain mismatched tier delete journal recovery control"
+                    );
+                    TierDeleteJournalEntryRecoveryOutcome::Failed
+                }
+            };
+        }
         warn!(
             event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
             component = LOG_COMPONENT_ECSTORE,
@@ -5552,6 +5714,36 @@ async fn recover_tier_delete_journal_entry(api: Arc<ECStore>, object_name: Strin
         return TierDeleteJournalEntryRecoveryOutcome::Failed;
     }
 
+    if let Some((source_schema, record_class)) = legacy_tier_delete_recovery_descriptor(&je) {
+        let stable_operation_identity = canonical_legacy_tier_delete_journal_identity(&object_name)
+            .expect("decoded legacy journal path was validated against its canonical object name")
+            .to_string();
+        return match persist_legacy_tier_delete_recovery_control(
+            api,
+            &object_name,
+            &data,
+            stable_operation_identity,
+            (source_schema, record_class),
+            IlmRecoveryClassification::RetainedAmbiguous,
+            IlmRecoveryErrorCode::RemoteVersionUnknown,
+        )
+        .await
+        {
+            Ok(()) => TierDeleteJournalEntryRecoveryOutcome::Retained,
+            Err(err) => {
+                warn!(
+                    event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                    journal_object = %object_name,
+                    error = ?err,
+                    "Failed to retain legacy tier delete journal recovery control"
+                );
+                TierDeleteJournalEntryRecoveryOutcome::Failed
+            }
+        };
+    }
+
     match api
         .durable_ilm_terminal_receipt_covers_active_source(&object_name, &data)
         .await

crates/ecstore/src/bucket/metadata.rs

@@ -791,9 +791,22 @@ impl BucketMetadata {
         }
     }
 
+    /// Replace one config payload and stamp its `*_config_updated_at` with the
+    /// local clock. This is the entry for edits that originate here: the
+    /// local write time is the edit's source time.
     pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
-        let updated = OffsetDateTime::now_utc();
+        self.update_config_at(config_file, data, OffsetDateTime::now_utc())
+    }
 
+    /// [`Self::update_config`] with an explicit `updated_at` stamp.
+    ///
+    /// For a config replicated from another site the edit's source time is
+    /// the peer's `updated_at`, not the moment it lands here: staleness of
+    /// the next incoming item is judged against the stored stamp, so stamping
+    /// the local apply time would reject a newer source edit that was merely
+    /// delivered late (backlog#2292). Only replication receivers should pass
+    /// a foreign time; local edits keep [`Self::update_config`].
+    pub fn update_config_at(&mut self, config_file: &str, data: Vec<u8>, updated: OffsetDateTime) -> Result<OffsetDateTime> {
         match config_file {
             BUCKET_POLICY_CONFIG => {
                 self.policy_config_json = data;
@@ -1525,6 +1538,39 @@ mod test {
         assert_eq!(metadata.bucket_incarnation_id, incarnation);
     }
 
+    /// backlog#2292: a replicated config is stamped with the source
+    /// `updated_at` it was given, not the local clock, while the plain
+    /// `update_config` entry keeps stamping the local clock.
+    #[test]
+    fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
+        let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
+        let mut metadata = BucketMetadata::new("source-stamped");
+
+        let stamped = metadata
+            .update_config_at(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(), source_time)
+            .unwrap();
+        assert_eq!(stamped, source_time);
+        assert_eq!(metadata.policy_config_updated_at, source_time);
+
+        let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
+        let stamped = metadata
+            .update_config_at(BUCKET_TAGGING_CONFIG, tagging, source_time)
+            .unwrap();
+        assert_eq!(stamped, source_time);
+        assert_eq!(metadata.tagging_config_updated_at, source_time);
+
+        let before = OffsetDateTime::now_utc();
+        let stamped = metadata
+            .update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec())
+            .unwrap();
+        assert!(stamped >= before, "a local edit is stamped with the local clock");
+        assert_eq!(metadata.policy_config_updated_at, stamped);
+        assert_eq!(
+            metadata.tagging_config_updated_at, source_time,
+            "restamping one config must not move another config's stamp"
+        );
+    }
+
     #[test]
     fn object_locking_requires_lock_metadata_not_plain_versioning() {
         use s3s::dto::ObjectLockEnabled;

crates/ecstore/src/bucket/metadata_sys.rs

@@ -567,6 +567,32 @@ pub async fn update_if_incarnation(
         config_file,
         data,
         Some(expected_incarnation_id),
+        None,
+    ))
+    .await
+}
+
+/// [`update_if_incarnation`] stamping the config with `updated_at` instead of
+/// the local clock.
+///
+/// For a site-replication receiver the edit's source time is the peer's
+/// `updated_at`; persisting it keeps the stored `*_config_updated_at` on the
+/// source clock so the next item's staleness is judged source-time against
+/// source-time (backlog#2292). See [`BucketMetadata::update_config_at`].
+pub async fn update_if_incarnation_at(
+    bucket: &str,
+    config_file: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    Box::pin(update_with_sys_expected(
+        get_bucket_metadata_sys()?,
+        bucket,
+        config_file,
+        data,
+        Some(expected_incarnation_id),
+        Some(updated_at),
     ))
     .await
 }
@@ -577,6 +603,30 @@ pub async fn delete_if_incarnation(bucket: &str, config_file: &str, expected_inc
         bucket,
         config_file,
         Some(expected_incarnation_id),
+        None,
+    ))
+    .await
+}
+
+/// [`delete_if_incarnation`] stamping the cleared config with `updated_at`
+/// (a replicated deletion's source time) instead of the local clock.
+///
+/// The stamp survives the deletion as the config's `*_config_updated_at`, and
+/// that is what the next incoming item is judged against: a local stamp on
+/// the delete would reject a newer source re-create that was merely delivered
+/// later (backlog#2292). See [`update_if_incarnation_at`].
+pub async fn delete_if_incarnation_at(
+    bucket: &str,
+    config_file: &str,
+    expected_incarnation_id: Uuid,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    Box::pin(delete_with_sys_expected(
+        get_bucket_metadata_sys()?,
+        bucket,
+        config_file,
+        Some(expected_incarnation_id),
+        Some(updated_at),
     ))
     .await
 }
@@ -598,34 +648,41 @@ async fn update_with_sys(
     config_file: &str,
     data: Vec<u8>,
 ) -> Result<OffsetDateTime> {
-    update_with_sys_expected(sys, bucket, config_file, data, None).await
+    update_with_sys_expected(sys, bucket, config_file, data, None, None).await
 }
 
+/// `updated_at` is the stamp persisted on the config; `None` uses the local
+/// clock (the edit originates here), `Some` carries a replicated edit's
+/// source time (backlog#2292).
 async fn update_with_sys_expected(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
     config_file: &str,
     data: Vec<u8>,
     expected_incarnation_id: Option<Uuid>,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
-    update_under_config_write_guard(sys, &guard, config_file, data).await
+    update_under_config_write_guard(sys, &guard, config_file, data, updated_at).await
 }
 
 /// [`delete`] against an explicitly supplied metadata system. See
 /// [`update_with_sys`].
 async fn delete_with_sys(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
-    delete_with_sys_expected(sys, bucket, config_file, None).await
+    delete_with_sys_expected(sys, bucket, config_file, None, None).await
 }
 
+/// `updated_at`: `None` stamps the local clock; `Some` persists a replicated
+/// deletion's source time (backlog#2292).
 async fn delete_with_sys_expected(
     sys: Arc<RwLock<BucketMetadataSys>>,
     bucket: &str,
     config_file: &str,
     expected_incarnation_id: Option<Uuid>,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
-    delete_under_config_write_guard(sys, &guard, config_file).await
+    delete_under_config_write_guard(sys, &guard, config_file, updated_at).await
 }
 
 /// Owns the complete bucket-config mutation fence.
@@ -772,7 +829,21 @@ pub async fn update_under_transaction_lock(
     data: Vec<u8>,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(bucket)?;
-    update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
+    update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, None).await
+}
+
+/// [`update_under_transaction_lock`] stamping the config with `updated_at`
+/// (a replicated edit's source time) instead of the local clock; see
+/// [`update_if_incarnation_at`] (backlog#2292).
+pub async fn update_under_transaction_lock_at(
+    guard: &BucketMetadataMutationGuard,
+    bucket: &str,
+    config_file: &str,
+    data: Vec<u8>,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    guard.ensure_valid(bucket)?;
+    update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data, Some(updated_at)).await
 }
 
 /// Clear one config file while the caller holds this bucket's transaction lock.
@@ -782,7 +853,7 @@ pub async fn delete_under_transaction_lock(
     config_file: &str,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(bucket)?;
-    delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
+    delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, None).await
 }
 
 pub async fn update_quota_if_incarnation(
@@ -790,6 +861,29 @@ pub async fn update_quota_if_incarnation(
     data: Vec<u8>,
     expected_incarnation_id: Uuid,
     proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+) -> Result<OffsetDateTime> {
+    update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, None).await
+}
+
+/// [`update_quota_if_incarnation`] stamping the quota config with
+/// `updated_at` (a replicated edit's source time) instead of the local
+/// clock; see [`update_if_incarnation_at`] (backlog#2292).
+pub async fn update_quota_if_incarnation_at(
+    bucket: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+    updated_at: OffsetDateTime,
+) -> Result<OffsetDateTime> {
+    update_quota_if_incarnation_stamped(bucket, data, expected_incarnation_id, proof, Some(updated_at)).await
+}
+
+async fn update_quota_if_incarnation_stamped(
+    bucket: &str,
+    data: Vec<u8>,
+    expected_incarnation_id: Uuid,
+    proof: &crate::services::notification_sys::CrossPoolFenceFleetProofToken,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     let sys = get_bucket_metadata_sys()?;
     let guard = Box::pin(acquire_config_write_guard_for_incarnation(
@@ -807,7 +901,7 @@ pub async fn update_quota_if_incarnation(
             achieved: 0,
         });
     }
-    update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data).await
+    update_under_config_write_guard(sys, &guard, rustfs_config::QUOTA_CONFIG_FILE, data, updated_at).await
 }
 
 pub async fn update_bucket_targets_under_transaction_lock(
@@ -823,6 +917,7 @@ async fn update_under_config_write_guard(
     guard: &BucketMetadataMutationGuard,
     config_file: &str,
     data: Vec<u8>,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(&guard.bucket)?;
     let metadata_sys = sys.read().await.clone();
@@ -834,7 +929,7 @@ async fn update_under_config_write_guard(
             Some(&guard.transaction_guard),
             &guard.bucket,
             "bucket config transaction",
-            metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id),
+            metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id, updated_at),
         ),
     )
     .await?;
@@ -846,6 +941,7 @@ async fn delete_under_config_write_guard(
     sys: Arc<RwLock<BucketMetadataSys>>,
     guard: &BucketMetadataMutationGuard,
     config_file: &str,
+    updated_at: Option<OffsetDateTime>,
 ) -> Result<OffsetDateTime> {
     guard.ensure_valid(&guard.bucket)?;
     let metadata_sys = sys.read().await.clone();
@@ -857,7 +953,7 @@ async fn delete_under_config_write_guard(
             Some(&guard.transaction_guard),
             &guard.bucket,
             "bucket config deletion transaction",
-            metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id),
+            metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id, updated_at),
         ),
     )
     .await?;
@@ -1762,15 +1858,17 @@ impl BucketMetadataSys {
     /// `update` and the config read alone). Keep these boxed.
     pub async fn update(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
         let incarnation_id = Box::pin(self.get_bucket_incarnation_id(bucket)).await?;
-        Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id)).await
+        Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id, None)).await
     }
 
     pub async fn delete(&self, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
         let incarnation_id = self.get_bucket_incarnation_id(bucket).await?;
-        self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id)
+        self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id, None)
             .await
     }
 
+    /// `updated_at`: `None` stamps the local clock; `Some` persists a
+    /// replicated edit's source time (backlog#2292).
     async fn update_checked(
         &self,
         bucket: &str,
@@ -1778,6 +1876,7 @@ impl BucketMetadataSys {
         data: Vec<u8>,
         parse: bool,
         expected_incarnation_id: Uuid,
+        updated_at: Option<OffsetDateTime>,
     ) -> Result<OffsetDateTime> {
         // Load through this system's own store, the one `save` persists to
         // (backlog#1052 S7). Reading from the ambient handle instead made the
@@ -1788,7 +1887,10 @@ impl BucketMetadataSys {
             return Err(Error::BucketNotFound(bucket.to_string()));
         }
 
-        let updated = bm.update_config(config_file, data)?;
+        let updated = match updated_at {
+            Some(updated_at) => bm.update_config_at(config_file, data, updated_at)?,
+            None => bm.update_config(config_file, data)?,
+        };
 
         Box::pin(self.save(bm)).await?;
 
@@ -3755,6 +3857,106 @@ mod tests {
         );
     }
 
+    /// backlog#2292: the explicit-stamp write path persists the given source
+    /// time as the config's `*_config_updated_at` — through the incarnation
+    /// path and through an already-held transaction guard — and survives a
+    /// reload from disk, while the plain path keeps stamping the local clock.
+    #[tokio::test]
+    async fn explicit_updated_at_is_persisted_as_the_config_stamp() {
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let bucket = "source-stamped-config";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
+        }
+        let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
+        let source_time = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
+        let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
+        let tagging = b"<Tagging><TagSet><Tag><Key>k</Key><Value>v</Value></Tag></TagSet></Tagging>".to_vec();
+
+        // Incarnation path (`update_if_incarnation_at` minus the ambient lookup).
+        let stamped =
+            update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(source_time))
+                .await
+                .expect("source-stamped policy write should persist");
+        assert_eq!(stamped, source_time);
+
+        // Held-guard path (`update_under_transaction_lock_at` minus the ambient lookup).
+        let guard = acquire_config_write_guard(sys.clone(), bucket).await.expect("write guard");
+        let stamped = update_under_config_write_guard(sys.clone(), &guard, BUCKET_TAGGING_CONFIG, tagging, Some(source_time))
+            .await
+            .expect("source-stamped tagging write should persist");
+        drop(guard);
+        assert_eq!(stamped, source_time);
+
+        let metadata_sys = sys.read().await.clone();
+        metadata_sys.metadata_map.write().await.clear();
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert_eq!(reloaded.policy_config_updated_at, source_time);
+        assert_eq!(reloaded.tagging_config_updated_at, source_time);
+
+        // The plain path is unchanged: a local edit is stamped with the local clock.
+        let before = OffsetDateTime::now_utc();
+        let stamped = update_with_sys(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy)
+            .await
+            .expect("locally stamped policy write should persist");
+        assert!(stamped >= before, "the plain write path must keep stamping the local clock");
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert_eq!(reloaded.policy_config_updated_at, stamped);
+        assert_eq!(
+            reloaded.tagging_config_updated_at, source_time,
+            "an unrelated config keeps its source stamp"
+        );
+    }
+
+    /// backlog#2292: a replicated delete persists the source time as the
+    /// cleared config's `*_config_updated_at`, so the receive-side gate
+    /// (source time against stored stamp) lets a newer source re-create land
+    /// even when the delete was applied later than the re-create's source
+    /// time; the plain delete keeps stamping the local clock.
+    #[tokio::test]
+    async fn explicit_updated_at_is_persisted_by_a_delete() {
+        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let bucket = "source-stamped-delete";
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
+        }
+        let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
+        let policy = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec();
+        let created_at = OffsetDateTime::now_utc() - Duration::from_secs(3 * 3600);
+        let deleted_at = created_at + Duration::from_secs(60);
+        let recreated_at = deleted_at + Duration::from_secs(60);
+
+        update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy.clone(), None, Some(created_at))
+            .await
+            .expect("source-stamped policy write should persist");
+        let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, Some(deleted_at))
+            .await
+            .expect("source-stamped policy delete should persist");
+        assert_eq!(stamped, deleted_at);
+
+        let metadata_sys = sys.read().await.clone();
+        metadata_sys.metadata_map.write().await.clear();
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert!(reloaded.policy_config_json.is_empty(), "the delete cleared the payload");
+        assert_eq!(reloaded.policy_config_updated_at, deleted_at, "the delete kept the source stamp");
+        assert!(
+            recreated_at >= reloaded.policy_config_updated_at,
+            "a re-create newer than the delete's source time is not stale against the stored stamp"
+        );
+
+        // The plain delete path is unchanged: stamped with the local clock.
+        update_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, policy, None, Some(recreated_at))
+            .await
+            .expect("re-create should persist");
+        let before = OffsetDateTime::now_utc();
+        let stamped = delete_with_sys_expected(sys.clone(), bucket, BUCKET_POLICY_CONFIG, None, None)
+            .await
+            .expect("locally stamped delete should persist");
+        assert!(stamped >= before, "the plain delete path must keep stamping the local clock");
+        let reloaded = metadata_sys.get_config_from_disk(bucket).await.expect("reload from disk");
+        assert_eq!(reloaded.policy_config_updated_at, stamped);
+    }
+
     /// The load and the persisted write share one write guard, so concurrent
     /// rewrites of the same config compose instead of clobbering each other.
     /// Moving the load outside that guard loses all but the last tag.
@@ -3971,10 +4173,16 @@ mod tests {
         let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
         assert_ne!(old_incarnation, new_incarnation);
 
-        let err =
-            update_with_sys_expected(sys.clone(), bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec(), Some(old_incarnation))
-                .await
-                .expect_err("a request authorized for the deleted incarnation must fail closed");
+        let err = update_with_sys_expected(
+            sys.clone(),
+            bucket,
+            BUCKET_TAGGING_CONFIG,
+            b"<Tagging/>".to_vec(),
+            Some(old_incarnation),
+            None,
+        )
+        .await
+        .expect_err("a request authorized for the deleted incarnation must fail closed");
         assert!(matches!(err, Error::BucketNotFound(name) if name == bucket));
 
         let persisted = sys.read().await.get_config_from_disk(bucket).await.unwrap();
@@ -4009,7 +4217,7 @@ mod tests {
             }],
         })
         .unwrap();
-        update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging)
+        update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging, None)
             .await
             .unwrap();
         assert!(!delete.is_finished());

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -20,9 +20,9 @@ pub use rustfs_replication::{
 pub(crate) use rustfs_replication::{
     ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
     delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
-    delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
-    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
-    resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
-    single_part_replica_etag_mismatch, target_delete_version_id,
+    delete_replication_object_opts, delete_replication_target_version_id, heal_uses_delete_replication_path,
+    is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
+    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
+    replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
+    resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -882,6 +882,20 @@ fn reconstructed_heal_delete_info(
 ) -> DeletedObjectReplicationInfo {
     let mut rstate = oi.replication_state();
     rstate.replicate_decision_str = dsc.to_string();
+    // The caller hands us a blank ObjectInfo (the source marker may already be
+    // gone), so the state above carries no target-assigned marker version ids.
+    // Restore them from the journal: `delete_marker_purge_version_id` must hit
+    // the id the target reported, not fall back to the source marker id, which
+    // a target that mints its own ids answers with an idempotent 204 that would
+    // acknowledge the intent while the real marker stays behind (backlog#2290).
+    // The corrupt flag rides along so a refusal stays a refusal after restart.
+    for (arn, version_id) in &entry.target_delete_marker_version_ids {
+        rstate
+            .target_delete_marker_version_ids
+            .entry(arn.clone())
+            .or_insert_with(|| version_id.clone());
+    }
+    rstate.target_delete_marker_version_ids_corrupt |= entry.target_delete_marker_version_ids_corrupt;
 
     let delete_marker_mtime = entry
         .delete_marker_mtime
@@ -6601,4 +6615,87 @@ mod tests {
             replacement_data
         );
     }
+
+    /// backlog#2290: a delete-marker purge intent that survives a restart
+    /// through the MRF journal addresses the marker version the TARGET
+    /// assigned, exactly as the live watcher does (see the
+    /// `requires_delayed_purge` spawn). The journal carries the per-ARN ids
+    /// (`targetDeleteMarkerVersionIDs`) and replay restores them into the
+    /// reconstructed replication state; without that the replay would fall
+    /// back to the source marker id, which a target that mints its own ids
+    /// answers with an idempotent 204 — the entry would be acknowledged while
+    /// the real marker stayed behind.
+    #[test]
+    fn mrf_delete_marker_purge_replay_preserves_target_assigned_marker_version() {
+        use super::super::replication_object_decision_boundary::{delete_marker_purge_mrf_entry, delete_marker_purge_version_id};
+
+        let arn = "arn:minio:replication::generic-target:photos".to_string();
+        let source_marker = uuid::Uuid::new_v4();
+        let remote_marker = "remote-assigned-marker-version".to_string();
+
+        let live_oi = ObjectInfo {
+            bucket: "photos".to_string(),
+            name: "obj".to_string(),
+            version_id: Some(source_marker),
+            delete_marker: true,
+            ..Default::default()
+        };
+        let mut live_state = live_oi.replication_state();
+        live_state.replicate_decision_str = replicate_decision_for_admitted_targets(std::slice::from_ref(&arn)).to_string();
+        live_state
+            .target_delete_marker_version_ids
+            .insert(arn.clone(), remote_marker.clone());
+        let live = DeletedObjectReplicationInfo {
+            delete_object: ReplicationDeletedObject {
+                object_name: "obj".to_string(),
+                delete_marker: true,
+                delete_marker_version_id: Some(source_marker),
+                replication_state: Some(live_state),
+                ..Default::default()
+            },
+            bucket: "photos".to_string(),
+            ..Default::default()
+        };
+        assert_eq!(
+            delete_marker_purge_version_id(live.delete_object.replication_state.as_ref(), &arn, source_marker),
+            Some(Some(remote_marker.clone())),
+            "the live purge addresses the recorded target version"
+        );
+
+        // Watch window exhausted: persist the intent, restart, replay it.
+        let entry = delete_marker_purge_mrf_entry(&live, vec![arn.clone()]);
+        let replay_oi = ObjectInfo {
+            bucket: entry.bucket.clone(),
+            name: entry.object.clone(),
+            version_id: entry.version_id,
+            delete_marker: entry.delete_marker,
+            ..Default::default()
+        };
+        let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
+        let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
+
+        assert_eq!(
+            delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
+            Some(Some(remote_marker)),
+            "the MRF replay must address the target-assigned marker version, not source marker {source_marker}"
+        );
+
+        // A refusal (inconsistent recorded ids) must stay a refusal across the
+        // journal round trip instead of degrading into the source-id fallback.
+        let mut refused = live;
+        refused
+            .delete_object
+            .replication_state
+            .as_mut()
+            .expect("state was set above")
+            .target_delete_marker_version_ids_corrupt = true;
+        let entry = delete_marker_purge_mrf_entry(&refused, vec![arn.clone()]);
+        assert!(entry.target_delete_marker_version_ids_corrupt);
+        let replayed = reconstructed_heal_delete_info(&entry, &replay_oi, &dsc);
+        assert_eq!(
+            delete_marker_purge_version_id(replayed.delete_object.replication_state.as_ref(), &arn, source_marker),
+            None,
+            "the MRF replay must keep refusing to guess when the recorded ids were inconsistent"
+        );
+    }
 }

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -32,11 +32,11 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec;
 use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
 use super::replication_object_decision_boundary::{
     MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-    delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
-    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
-    resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
-    target_delete_version_id,
+    delete_replication_creates_marker, delete_replication_target_version_id, heal_uses_delete_replication_path,
+    is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
+    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
+    replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
+    should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
 };
 use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
 use super::replication_resync_boundary::ResyncStatusType;
@@ -2051,7 +2051,11 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
 
     let is_version_purge = is_version_delete_replication(&dobj.delete_object);
 
-    let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object);
+    // The watcher exists to purge a replicated marker once the SOURCE marker
+    // vanishes. A version purge is that purge already (its failures reach the
+    // journal as a purge entry), so it must not spawn a second watcher that
+    // journals a duplicate intent (backlog#2290).
+    let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object) && !is_version_purge;
 
     let (replication_status, prev_status) = if !is_version_purge {
         (
@@ -2761,12 +2765,6 @@ fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str
 }
 
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
-    let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
-        version_id.to_owned()
-    } else {
-        dobj.delete_object.version_id.unwrap_or_default()
-    };
-
     let mut rinfo = dobj
         .delete_object
         .replication_state
@@ -2799,7 +2797,25 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
         return rinfo;
     }
 
-    let version_id = target_delete_version_id(version_id, is_version_purge);
+    // Purging a replicated delete marker addresses the version the target
+    // assigned (recorded when the marker was created there); see
+    // `delete_replication_target_version_id`. A corrupt record is a failure,
+    // not a guess: the entry stays visible until the metadata is repaired.
+    let Some(version_id) = delete_replication_target_version_id(&dobj.delete_object, &tgt_client.arn) else {
+        warn!(
+            event = EVENT_DELETE_MARKER_PURGE_FAILED,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            bucket = tgt_client.bucket,
+            object = dobj.delete_object.object_name,
+            arn = %tgt_client.arn,
+            reason = "recorded_target_version_inconsistent",
+            "Replicated version purge refused: recorded target delete-marker version metadata is inconsistent"
+        );
+        rinfo.version_purge_status = VersionPurgeStatusType::Failed;
+        rinfo.error = Some("recorded target delete-marker version metadata is inconsistent".to_string());
+        return rinfo;
+    };
 
     if dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
         match head_object_for_worker(

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -68,7 +68,10 @@ use std::{
     },
     time::SystemTime,
 };
-use tokio::{net::TcpStream, time::Duration};
+use tokio::{
+    net::TcpStream,
+    time::{Duration, timeout},
+};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tracing::{debug, info, warn};
@@ -874,6 +877,16 @@ impl PeerRestClient {
         self.offline.store(false, Ordering::Release);
     }
 
+    /// Prepare a retry without allowing connection-cache cleanup to extend the
+    /// caller's probe deadline. The offline gate is cleared even when eviction
+    /// times out so a cancelled cleanup cannot strand the peer in fast-fail
+    /// mode; a later request can perform a fresh eviction if needed.
+    pub async fn prepare_retry_with_timeout(&self, timeout_duration: Duration) -> bool {
+        let evicted = timeout(timeout_duration, self.evict_connection()).await.is_ok();
+        self.offline.store(false, Ordering::Release);
+        evicted
+    }
+
     /// Whether this failure means the peer is unreachable, so it should be
     /// gated offline and its connection evicted.
     ///

crates/ecstore/src/services/notification_sys.rs

@@ -72,6 +72,29 @@ const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
 /// service must not advertise this version until the conditional writer from
 /// rustfs/backlog#684 is available.
 const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
+
+fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
+    configured
+        .filter(|seconds| *seconds > 0)
+        .unwrap_or(rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS)
+        .min(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS)
+}
+
+fn admin_peer_probe_timeout() -> Duration {
+    let configured = rustfs_utils::get_env_opt_u64_with_aliases(rustfs_config::ENV_ADMIN_PEER_PROBE_TIMEOUT_SECS, &[]);
+    let seconds = resolve_admin_peer_probe_timeout_secs(configured);
+    Duration::from_secs(seconds)
+}
+
+fn remaining_admin_peer_probe_timeout(deadline: Instant) -> Option<Duration> {
+    remaining_admin_peer_probe_timeout_at(deadline, Instant::now())
+}
+
+fn remaining_admin_peer_probe_timeout_at(deadline: Instant, now: Instant) -> Option<Duration> {
+    let remaining = deadline.saturating_duration_since(now);
+    (!remaining.is_zero()).then_some(remaining)
+}
+
 type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
 
 fn cross_pool_fence_policy_results(
@@ -1538,7 +1561,7 @@ impl NotificationSys {
     {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
-        let peer_timeout = Duration::from_secs(5);
+        let peer_timeout = admin_peer_probe_timeout();
 
         for (idx, client) in self.peer_clients.iter().enumerate() {
             let endpoints = endpoints.clone();
@@ -1546,7 +1569,9 @@ impl NotificationSys {
             futures.push(async move {
                 if let Some(client) = client {
                     let host = client.host.to_string();
-                    match timeout(peer_timeout, client.local_storage_info()).await {
+                    let deadline = Instant::now() + peer_timeout;
+                    let probe_timeout = remaining_admin_peer_probe_timeout(deadline).unwrap_or_default();
+                    match timeout(probe_timeout, client.local_storage_info()).await {
                         Ok(Ok(mut info)) => {
                             normalize_and_cache_peer_storage_info(cache, &host, &mut info);
                             Some(info)
@@ -1557,7 +1582,6 @@ impl NotificationSys {
                         }
                         Err(_) => {
                             warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
-                            client.evict_connection().await;
                             handle_peer_failure(cache, &host, &endpoints)
                         }
                     }
@@ -1583,7 +1607,7 @@ impl NotificationSys {
     pub async fn server_info(&self) -> Vec<ServerProperties> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         let endpoints = runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into());
-        let peer_timeout = Duration::from_secs(5);
+        let peer_timeout = admin_peer_probe_timeout();
 
         for (idx, client) in self.peer_clients.iter().enumerate() {
             let host = self
@@ -1600,12 +1624,23 @@ impl NotificationSys {
                     };
                 };
 
+                let deadline = Instant::now() + peer_timeout;
+                let Some(first_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                };
+
                 // First attempt. A single evicted or half-open internode channel
                 // is enough to fail one probe and, before retrying, would drop
-                // the member to unknown/offline for this whole snapshot. So on any
-                // first-attempt failure we evict the channel and re-dial once
-                // before falling back (rustfs/backlog#1049, P1-B).
-                match timeout(peer_timeout, client.server_info()).await {
+                // the member to unknown/offline for this whole snapshot. On a
+                // quick failure we evict the channel and re-dial once before
+                // falling back (rustfs/backlog#1049, P1-B). A slow attempt
+                // consumes the round budget and therefore does not trigger a
+                // second full wait or an asynchronous eviction side effect.
+                match timeout(first_timeout, client.server_info()).await {
                     Ok(Ok(info)) => {
                         return PeerServerInfoProbe { host, result: Ok(info) };
                     }
@@ -1619,14 +1654,37 @@ impl NotificationSys {
                 // `evict_connection` would leave that gate up and the retry would
                 // fast-fail with "temporarily offline" instead of reconnecting
                 // (rustfs/backlog#1049 P1-B).
-                client.prepare_retry().await;
+                let Some(retry_budget) = remaining_admin_peer_probe_timeout(deadline) else {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                };
+                // Bound connection-cache cleanup too. The helper clears the offline gate even
+                // when eviction itself times out, so cancellation cannot strand this peer in
+                // fast-fail mode.
+                if !client.prepare_retry_with_timeout(retry_budget).await {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                }
 
                 // Second and final attempt on the fresh channel.
-                match timeout(peer_timeout, client.server_info()).await {
+                let Some(retry_timeout) = remaining_admin_peer_probe_timeout(deadline) else {
+                    let health = peer_disk_health_with_deadline(&host, deadline).await;
+                    return PeerServerInfoProbe {
+                        host,
+                        result: Err(PeerServerInfoProbeFailure::Rpc { health }),
+                    };
+                };
+                match timeout(retry_timeout, client.server_info()).await {
                     Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) },
                     Ok(Err(err)) => {
                         warn!("peer {host} server_info failed after retry: {err}");
-                        let health = peer_disk_health(&host).await;
+                        let health = peer_disk_health_with_deadline(&host, deadline).await;
                         PeerServerInfoProbe {
                             host,
                             result: Err(PeerServerInfoProbeFailure::Rpc { health }),
@@ -1634,8 +1692,7 @@ impl NotificationSys {
                     }
                     Err(_) => {
                         warn!("peer {host} server_info timed out after retry ({peer_timeout:?})");
-                        client.evict_connection().await;
-                        let health = peer_disk_health(&host).await;
+                        let health = peer_disk_health_with_deadline(&host, deadline).await;
                         PeerServerInfoProbe {
                             host,
                             result: Err(PeerServerInfoProbeFailure::Rpc { health }),
@@ -3023,6 +3080,11 @@ async fn peer_disk_health(host: &str) -> Option<PeerDiskHealth> {
     }
 }
 
+async fn peer_disk_health_with_deadline(host: &str, deadline: Instant) -> Option<PeerDiskHealth> {
+    let remaining = remaining_admin_peer_probe_timeout(deadline)?;
+    timeout(remaining, peer_disk_health(host)).await.ok().flatten()
+}
+
 /// Handle a peer failure for server_info: return cached data if available, or
 /// classify the member as `unknown` / `degraded` / `offline` depending on how
 /// many consecutive probes have failed and whether the peer's drives are still
@@ -4017,6 +4079,37 @@ mod tests {
         }
     }
 
+    #[test]
+    fn admin_peer_probe_timeout_rejects_zero_and_caps_large_values() {
+        assert_eq!(
+            resolve_admin_peer_probe_timeout_secs(None),
+            rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
+        );
+        assert_eq!(
+            resolve_admin_peer_probe_timeout_secs(Some(0)),
+            rustfs_config::DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS
+        );
+        assert_eq!(
+            resolve_admin_peer_probe_timeout_secs(Some(rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS + 1)),
+            rustfs_config::MAX_ADMIN_PEER_PROBE_TIMEOUT_SECS
+        );
+        assert_eq!(resolve_admin_peer_probe_timeout_secs(Some(7)), 7);
+    }
+
+    #[tokio::test]
+    async fn admin_peer_probe_health_fallback_respects_expired_deadline() {
+        let deadline = Instant::now();
+        assert!(peer_disk_health_with_deadline("peer-1", deadline).await.is_none());
+    }
+
+    #[test]
+    fn admin_peer_probe_deadline_is_shared_across_attempts() {
+        let start = Instant::now();
+        let deadline = start + Duration::from_secs(10);
+        assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(6)).is_some());
+        assert!(remaining_admin_peer_probe_timeout_at(deadline, start + Duration::from_secs(10)).is_none());
+    }
+
     #[tokio::test]
     async fn call_peer_with_timeout_returns_value_when_fast() {
         let result = call_peer_with_timeout(

crates/ecstore/src/store/init.rs

@@ -827,8 +827,8 @@ mod tests {
             },
             recovery_control::{
                 IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
-                IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, load_recovery_control, observe_recovery_source,
-                save_recovery_control_if_absent,
+                IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control,
+                observe_recovery_source, save_recovery_control_if_absent,
             },
             tier_delete_journal::{
                 DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
@@ -16760,6 +16760,141 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn legacy_tier_delete_journals_create_redacted_recovery_controls_without_remote_calls() {
+        let temp_dir = tempfile::tempdir().expect("create legacy journal recovery store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-tier-journal-recovery", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "LEGACY-RECOVERY";
+        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("legacy recovery tier lease should resolve")
+            .backend_identity();
+        let fixtures = [
+            serde_json::json!({
+                "version": 1,
+                "obj_name": "legacy/remote-v1",
+                "version_id": "opaque-v1",
+                "tier_name": tier_name,
+            }),
+            serde_json::json!({
+                "version": 2,
+                "obj_name": "legacy/remote-v2",
+                "version_id": "opaque-v2",
+                "tier_name": tier_name,
+                "backend_identity": backend_identity,
+            }),
+        ];
+        let mut journal_paths = Vec::new();
+        for fixture in &fixtures {
+            let data = serde_json::to_vec(&fixture).expect("legacy journal fixture should encode");
+            let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data)
+                .expect("legacy journal fixture should decode");
+            let path = tier_delete_journal_object_name(&entry);
+            com::save_config(store.clone(), &path, data)
+                .await
+                .expect("legacy journal fixture should persist");
+            journal_paths.push(path);
+        }
+        let corrupt_path = format!(
+            "{TIER_DELETE_JOURNAL_PREFIX}/{}.json",
+            rustfs_utils::crypto::hex_sha256(b"corrupt legacy tier journal", ToOwned::to_owned)
+        );
+        com::save_config(store.clone(), &corrupt_path, b"{corrupt".to_vec())
+            .await
+            .expect("corrupt legacy journal fixture should persist");
+
+        let (first, concurrent) = tokio::join!(
+            recover_tier_delete_journal_entries(store.clone(), 100, None),
+            recover_tier_delete_journal_entries(store.clone(), 100, None),
+        );
+        for stats in [first, concurrent] {
+            let stats = stats.expect("concurrent legacy journal recovery scan should finish");
+            assert_eq!((stats.scanned, stats.deleted, stats.failed), (3, 0, 0));
+        }
+        assert_eq!(tier_delete_journal_count(store.clone()).await, 3);
+        assert_eq!(backend.remove_count().await, 0, "legacy recovery must not call the remote tier");
+        assert_eq!(backend.exact_remove_count(), 0, "legacy recovery must not issue exact remote DELETE");
+        assert!(backend.op_log().await.is_empty(), "legacy recovery must not invoke any backend operation");
+
+        let mut first_controls = list_recovery_controls(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
+            .await
+            .expect("legacy recovery controls should be listable")
+            .records;
+        first_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
+        assert_eq!(first_controls.len(), 3);
+        assert_eq!(
+            first_controls
+                .iter()
+                .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous)
+                .count(),
+            2
+        );
+        assert_eq!(
+            first_controls
+                .iter()
+                .filter(|control| control.classification == IlmRecoveryClassification::Corrupt)
+                .count(),
+            1
+        );
+        for view in &first_controls {
+            assert_eq!(view.protocol, IlmRecoveryProtocol::TierDeleteJournal);
+            assert_eq!(view.revision, 1);
+            assert_eq!(view.attempt_count, 0);
+            let encoded = serde_json::to_string(view).expect("recovery control view should encode");
+            for secret in ["legacy/remote-v1", "legacy/remote-v2", "opaque-v1", "opaque-v2", tier_name] {
+                assert!(!encoded.contains(secret), "recovery control view must redact `{secret}`");
+            }
+            let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &view.control_id)
+                .await
+                .expect("legacy recovery control should load");
+            match view.source_schema.as_str() {
+                "rustfs-tier-delete-journal-v1" => {
+                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v1");
+                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
+                }
+                "rustfs-tier-delete-journal-v2" => {
+                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_v2");
+                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::RemoteVersionUnknown);
+                }
+                "rustfs-tier-delete-journal-unknown" => {
+                    assert_eq!(persisted.control.identity.record_class, "tier_delete_journal_corrupt");
+                    assert_eq!(view.last_error_code, IlmRecoveryErrorCode::SourceCorrupt);
+                }
+                schema => panic!("unexpected legacy recovery source schema: {schema}"),
+            }
+        }
+
+        com::save_config(
+            store.clone(),
+            &journal_paths[0],
+            serde_json::to_vec_pretty(&fixtures[0]).expect("rewritten legacy journal fixture should encode"),
+        )
+        .await
+        .expect("equivalent legacy journal rewrite should persist");
+        let second = recover_tier_delete_journal_entries(store.clone(), 100, None)
+            .await
+            .expect("repeated legacy journal recovery scan should finish");
+        assert_eq!((second.scanned, second.deleted, second.failed), (3, 0, 0));
+        let mut second_controls = list_recovery_controls(store, IlmRecoveryProtocol::TierDeleteJournal, None, 100, None)
+            .await
+            .expect("repeated legacy recovery controls should remain listable")
+            .records;
+        second_controls.sort_by(|left, right| left.control_id.cmp(&right.control_id));
+        assert_eq!(second_controls, first_controls, "repeated scans must not reset durable controls");
+        assert_eq!(backend.remove_count().await, 0, "repeated recovery must remain remote-call free");
+        assert_eq!(backend.exact_remove_count(), 0);
+        assert!(
+            backend.op_log().await.is_empty(),
+            "repeated recovery must not invoke any backend operation"
+        );
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]

crates/heal/src/heal/erasure_healer.rs

@@ -108,6 +108,144 @@ pub struct ErasureSetHealer {
     target_endpoints: Arc<[String]>,
     replacement_task_id: Option<String>,
     replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
+    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
+}
+
+async fn acquire_page_permit(
+    semaphore: Arc<Semaphore>,
+    pacer: Option<&super::pacing::MainlinePacer>,
+    cancel: &tokio_util::sync::CancellationToken,
+) -> Result<tokio::sync::OwnedSemaphorePermit> {
+    let acquire = || async {
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            permit = semaphore.clone().acquire_owned() => permit.map_err(|err| Error::other(format!("Failed to acquire page concurrency permit: {err}"))),
+        }
+    };
+    let mut paid_pause = false;
+    loop {
+        let permit = acquire().await?;
+        if let Some(pacer) = pacer {
+            // Keep the real permit on the low-pressure path. Every acquisition
+            // gets a fresh decision, including a waiter that queued a second
+            // time. One completed pause is a bounded minimum-progress grant.
+            match pacer.admission_decision() {
+                super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
+                    drop(permit);
+                    paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
+                    continue;
+                }
+                _ => {}
+            }
+        }
+        return Ok(permit);
+    }
+}
+
+#[cfg(test)]
+mod mainline_pacing_tests {
+    use super::*;
+    use crate::heal::pacing::{MainlinePacer, TestPressure};
+    use rustfs_concurrency::WorkloadClass;
+    use std::sync::atomic::Ordering;
+    use tokio_util::sync::CancellationToken;
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_page_waiters_resample_after_capacity_and_release_permits() {
+        let semaphore = Arc::new(Semaphore::new(1));
+        let occupied = semaphore
+            .clone()
+            .acquire_owned()
+            .await
+            .expect("existing object owns capacity");
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
+        let pacer = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250)).expect("pacer"));
+        let cancel = CancellationToken::new();
+        let waiting = tokio::spawn({
+            let semaphore = semaphore.clone();
+            let pacer = pacer.clone();
+            let cancel = cancel.clone();
+            async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
+        });
+        tokio::task::yield_now().await;
+        provider.active.store(100, Ordering::SeqCst);
+        drop(occupied);
+        provider.sampled.notified().await;
+        assert_eq!(semaphore.available_permits(), 1, "running pressure wait cannot retain the page permit");
+        cancel.cancel();
+        assert!(matches!(waiting.await.expect("page waiter"), Err(Error::TaskCancelled)));
+        assert_eq!(semaphore.available_permits(), 1);
+        let deadline = tokio::time::timeout(
+            Duration::from_millis(10),
+            acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
+        )
+        .await;
+        assert!(deadline.is_err());
+        assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
+        let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
+            .await
+            .expect("unpaced admission");
+        assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
+        drop(permit);
+        assert_eq!(semaphore.available_permits(), 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_two_page_waiters_check_pressure_at_final_admission() {
+        use std::task::Poll;
+        for raise_pressure in [false, true] {
+            let semaphore = Arc::new(Semaphore::new(1));
+            let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
+            let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
+            let pause = Duration::from_millis(250);
+            let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause).expect("pacer");
+            let cancel = CancellationToken::new();
+            let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
+            assert!(futures::poll!(first.as_mut()).is_pending());
+            assert!(futures::poll!(second.as_mut()).is_pending());
+            drop(occupied);
+            let first_ready = match futures::poll!(first.as_mut()) {
+                Poll::Ready(result) => Some(result.expect("first admission")),
+                Poll::Pending => None,
+            };
+            assert!(futures::poll!(second.as_mut()).is_pending());
+            let first_permit = match first_ready {
+                Some(permit) => permit,
+                None => tokio::time::timeout(Duration::from_millis(1), first)
+                    .await
+                    .expect("low-pressure waiters must not bounce capacity forever")
+                    .expect("first permit"),
+            };
+            // The first object owns real page capacity while its commit runs.
+            tokio::time::advance(Duration::from_millis(100)).await;
+            if raise_pressure {
+                provider.active.store(100, Ordering::SeqCst);
+            }
+            drop(first_permit);
+            let admitted = if raise_pressure {
+                assert!(
+                    futures::poll!(second.as_mut()).is_pending(),
+                    "a second acquisition cannot reuse the earlier low-pressure sample"
+                );
+                assert_eq!(semaphore.available_permits(), 1, "pressure wait must release page capacity");
+                tokio::time::advance(pause).await;
+                tokio::time::timeout(Duration::from_millis(1), second)
+                    .await
+                    .expect("sustained pressure must allow one unit after its bounded pause")
+                    .expect("second permit")
+            } else {
+                tokio::time::timeout(Duration::from_millis(1), second)
+                    .await
+                    .expect("low pressure must make progress")
+                    .expect("second permit")
+            };
+            assert_eq!(semaphore.available_permits(), 0);
+            drop(admitted);
+            assert_eq!(semaphore.available_permits(), 1);
+        }
+    }
 }
 
 pub(crate) fn target_outcomes_complete(result: &HealResultItem, target_endpoints: &[String]) -> bool {
@@ -219,9 +357,15 @@ impl ErasureSetHealer {
             target_endpoints: Vec::new().into(),
             replacement_task_id: None,
             replacement_target_identities: None,
+            mainline_pacer: None,
         }
     }
 
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+        self.mainline_pacer = pacer;
+        self
+    }
+
     pub(crate) fn with_replacement_targets(
         mut self,
         mut target_endpoints: Vec<String>,
@@ -856,6 +1000,9 @@ impl ErasureSetHealer {
         let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
 
         loop {
+            if let Some(pacer) = &self.mainline_pacer {
+                pacer.wait(&self.cancel_token).await?;
+            }
             self.verify_replacement_identity_fence("page scan").await?;
             // Get one page of object versions
             let (objects, next_token, is_truncated) = if use_disk_walk {
@@ -1034,13 +1181,10 @@ impl ErasureSetHealer {
                 let semaphore = semaphore.clone();
                 let target_endpoints = self.target_endpoints.clone();
                 let replacement_commit_evidence_required = self.replacement_task_id.is_some();
+                let mainline_pacer = self.mainline_pacer.clone();
 
                 page_tasks.push(async move {
-                    let permit = semaphore
-                        .clone()
-                        .acquire_owned()
-                        .await
-                        .map_err(|e| Error::other(format!("Failed to acquire page concurrency permit: {e}")));
+                    let permit = acquire_page_permit(semaphore, mainline_pacer.as_deref(), &cancel_token).await;
 
                     let _permit = match permit {
                         Ok(permit) => permit,

crates/heal/src/heal/manager.rs

@@ -602,13 +602,13 @@ pub struct HealConfig {
     pub set_bulkhead_enable: bool,
     /// Whether erasure-set page parallelism is enabled.
     pub page_parallel_enable: bool,
-    /// Whether foreground read pressure can delay best-effort heal task starts.
+    /// Whether foreground pressure delays best-effort starts and paces running admin work.
     pub mainline_throttle_enable: bool,
-    /// Foreground read permit utilization percentage that delays best-effort heal starts.
+    /// Foreground read utilization high watermark for start admission and admin pacing.
     pub mainline_read_utilization_high_percent: usize,
-    /// Foreground write utilization percentage that delays best-effort heal starts.
+    /// Foreground write utilization high watermark for start admission and admin pacing.
     pub mainline_write_utilization_high_percent: usize,
-    /// Delay before rechecking foreground pressure after delaying heal starts.
+    /// Start recheck interval; running admin pacing caps each holder's pause at one second.
     pub mainline_max_sleep: Duration,
 }
 

crates/heal/src/heal/manager/scheduler.rs

@@ -175,11 +175,23 @@ impl HealManager {
                     .unwrap_or_else(|poisoned| poisoned.into_inner())
                     .get(&request.id)
                     .cloned();
-                let task = Arc::new(HealTask::from_replacement_recovery_request(
-                    request,
-                    storage.clone(),
-                    replacement_resume_endpoint,
-                ));
+                let mainline_pacer = if request.source == HealRequestSource::Admin && config.mainline_throttle_enable {
+                    workload_provider.as_ref().and_then(|provider| {
+                        crate::heal::pacing::MainlinePacer::new(
+                            provider.clone(),
+                            config.mainline_read_utilization_high_percent,
+                            config.mainline_write_utilization_high_percent,
+                            config.mainline_max_sleep,
+                        )
+                        .map(Arc::new)
+                    })
+                } else {
+                    None
+                };
+                let task = Arc::new(
+                    HealTask::from_replacement_recovery_request(request, storage.clone(), replacement_resume_endpoint)
+                        .with_mainline_pacer(mainline_pacer),
+                );
                 let task_id = task.id.clone();
                 active_heals_guard.insert(task_id.clone(), task.clone());
                 publish_active_heal_count(&active_heals_guard);

crates/heal/src/heal/manager/tests.rs

@@ -14,6 +14,9 @@
 
 use super::*;
 use crate::heal::EcstoreError;
+use crate::heal::outcome::{
+    HealAbortReason, HealDeferredReason, HealExecutionOutcome, HealObjectDisposition, HealTraversalCoverage,
+};
 use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity};
 use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
 use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
@@ -23,6 +26,8 @@ use rustfs_madmin::heal_commands::HealResultItem;
 use std::sync::Mutex as StdMutex;
 use tempfile::TempDir;
 
+mod running_mainline;
+
 use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
 
 #[tokio::test]
@@ -515,10 +520,15 @@ impl HealStorageAPI for MockStorage {
     async fn heal_object(
         &self,
         bucket: &str,
-        _object: &str,
+        object: &str,
         _version_id: Option<&str>,
         _opts: &HealOpts,
     ) -> Result<(HealResultItem, Option<Error>)> {
+        if bucket.starts_with("heal-start-retry-deadline-object-") && object == "blocked" {
+            let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone();
+            hook.started.notify_one();
+            std::future::pending::<()>().await;
+        }
         if bucket == "completed-retention-failed" {
             return Err(Error::TaskExecutionFailed {
                 message: "retention fixture failure".to_string(),
@@ -580,11 +590,35 @@ impl HealStorageAPI for MockStorage {
 
     async fn list_objects_for_heal_page(
         &self,
-        _bucket: &str,
+        bucket: &str,
         _prefix: &str,
-        _continuation_token: Option<&str>,
+        continuation_token: Option<&str>,
         _include_lifecycle_object_info: bool,
     ) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
+        if bucket.starts_with("heal-start-retry-deadline-") {
+            if continuation_token.is_some() {
+                let hook = COMPLETED_RETENTION_HOOKS.lock().await[bucket].clone();
+                hook.started.notify_one();
+                std::future::pending::<()>().await;
+            }
+            let listing_timeout = bucket.starts_with("heal-start-retry-deadline-listing-");
+            let names = if listing_timeout {
+                vec!["completed"]
+            } else {
+                vec!["completed", "blocked"]
+            };
+            let objects = names
+                .into_iter()
+                .map(|name| crate::heal::storage::HealListItem {
+                    name: name.to_string(),
+                    version_id: None,
+                    mod_time_unix_nanos: None,
+                    lifecycle_object_info: None,
+                    is_delete_marker: false,
+                })
+                .collect();
+            return Ok((objects, listing_timeout.then(|| "next".to_string()), listing_timeout));
+        }
         Ok((Vec::new(), None, false))
     }
 
@@ -607,6 +641,161 @@ impl HealStorageAPI for MockStorage {
     }
 }
 
+async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) {
+    for phase in ["listing", "object"] {
+        let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}");
+        let manager = HealManager::new(Arc::new(MockStorage), None);
+        let mut request = HealRequest::new(
+            HealType::Prefix {
+                bucket: bucket.clone(),
+                prefix: String::new(),
+            },
+            HealOptions {
+                timeout: Some(if cancel {
+                    Duration::from_secs(60)
+                } else {
+                    Duration::from_millis(200)
+                }),
+                ..Default::default()
+            },
+            HealPriority::High,
+        );
+        request.source = HealRequestSource::Admin;
+        let task_id = request.id.clone();
+        let hook = Arc::new(CompletedRetentionHook::default());
+        {
+            let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
+            hooks.insert(bucket.clone(), Arc::clone(&hook));
+            hooks.insert(task_id.clone(), Arc::clone(&hook));
+        }
+        manager.submit_heal_request(request).await.expect("admit deadline task");
+        process_manager_queue_once(&manager).await;
+        tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
+            .await
+            .expect("executor reaches blocked storage");
+        let active = manager.get_task_report(&task_id).await.expect("active report");
+        assert_eq!(active.progress.expect("real completed object progress").objects_healed, 1);
+        if cancel {
+            manager.active_heals.lock().await[&task_id].cancel_token.cancel();
+        }
+        tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
+            .await
+            .expect("deadline archives task");
+        let report = manager.get_task_report(&task_id).await.expect("terminal report");
+        assert_eq!(
+            report.status,
+            if cancel {
+                HealTaskStatus::Cancelled
+            } else {
+                HealTaskStatus::Timeout
+            },
+            "blocked {phase}"
+        );
+        let progress = report.progress.expect("terminal progress retained");
+        assert_eq!(progress.objects_healed, 1);
+        assert_eq!(progress.objects_failed, 0, "interrupted object has no terminal storage result");
+        assert_eq!(report.result_items.len(), 1, "completed result retained");
+        let outcome = report.outcome.expect("canonical terminal outcome retained");
+        assert_eq!(
+            outcome.execution,
+            HealExecutionOutcome::Aborted(if cancel {
+                HealAbortReason::Cancelled
+            } else {
+                HealAbortReason::Deadline
+            })
+        );
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+        assert_eq!(outcome.counters.healed, 0, "legacy success supplies no authoritative repair proof");
+        let completed = outcome
+            .objects
+            .iter()
+            .find(|item| item.identity.object == "completed")
+            .expect("completed object diagnostic retained");
+        assert_eq!(completed.disposition, HealObjectDisposition::Unknown);
+        if phase == "object" {
+            let interrupted = outcome
+                .objects
+                .iter()
+                .find(|item| item.identity.object == "blocked")
+                .expect("interrupted object diagnostic retained");
+            assert_eq!(
+                interrupted.disposition,
+                if cancel {
+                    HealObjectDisposition::Cancelled
+                } else {
+                    HealObjectDisposition::Deferred {
+                        reason: HealDeferredReason::Deadline,
+                        retry_not_before: None,
+                    }
+                }
+            );
+        } else {
+            assert_eq!(outcome.objects.len(), 1, "an unread page cannot supply object identities");
+        }
+        assert!(!manager.active_heals.lock().await.contains_key(&task_id));
+        assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
+        assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
+        hook.finish.notify_one();
+        COMPLETED_RETENTION_HOOKS
+            .lock()
+            .await
+            .retain(|key, _| key != &bucket && key != &task_id);
+    }
+}
+
+#[tokio::test]
+async fn heal_start_retry_deadline_preserves_real_executor_progress() {
+    assert_heal_start_retry_control_preserves_real_executor_progress(false).await;
+}
+
+#[tokio::test]
+async fn heal_start_retry_cancellation_preserves_real_executor_progress() {
+    assert_heal_start_retry_control_preserves_real_executor_progress(true).await;
+}
+
+#[tokio::test]
+async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
+    let manager = HealManager::new(
+        Arc::new(MockStorage),
+        Some(HealConfig {
+            task_timeout: Duration::ZERO,
+            ..Default::default()
+        }),
+    );
+    let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    request.source = HealRequestSource::Admin;
+    request.options.timeout = Some(Duration::from_secs(60));
+    let task_id = request.id.clone();
+    let created_at = request.created_at;
+    let hook = Arc::new(CompletedRetentionHook::default());
+    COMPLETED_RETENTION_HOOKS
+        .lock()
+        .await
+        .insert(task_id.clone(), Arc::clone(&hook));
+    manager
+        .submit_heal_request(request)
+        .await
+        .expect("admit explicit-budget task");
+    process_manager_queue_once(&manager).await;
+    tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
+        .await
+        .expect("real read-quorum failure prepares retry");
+    let retry = manager.retrying_heals.lock().await[&task_id].request.clone();
+    assert_eq!(retry.id, task_id);
+    assert_eq!(retry.created_at, created_at);
+    assert_eq!(retry.source, HealRequestSource::Admin);
+    assert_eq!(retry.retry_attempts, 1);
+    let remaining = retry.options.timeout.expect("retry retains explicit budget");
+    assert!(remaining > Duration::ZERO && remaining < Duration::from_secs(60));
+    assert!(matches!(
+        manager.get_task_status(&task_id).await.expect("retry remains queryable"),
+        HealTaskStatus::Retrying { retry_attempt: 1, .. }
+    ));
+    manager.cancel_task(&task_id).await.expect("cancel held retry");
+    hook.finish.notify_one();
+    COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id);
+}
+
 struct ManagerRecoveryTestHook {
     replacement_resume_disk: DiskStore,
     listed: StdMutex<bool>,

crates/heal/src/heal/manager/tests/running_mainline.rs

@@ -0,0 +1,249 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use crate::heal::storage::HealListItem;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use tokio::sync::Semaphore;
+
+#[derive(Default)]
+struct PressureProbe {
+    active: AtomicUsize,
+    commit_open: AtomicBool,
+    high_sampled: Notify,
+}
+
+impl WorkloadAdmissionSnapshotProvider for PressureProbe {
+    fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
+        assert!(
+            !self.commit_open.load(Ordering::SeqCst),
+            "pressure must not be sampled inside an object commit"
+        );
+        let active = self.active.load(Ordering::SeqCst);
+        if active >= 80 {
+            self.high_sampled.notify_one();
+        }
+        WorkloadAdmissionRegistrySnapshot::new(vec![
+            WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
+                Some(active),
+                None,
+                Some(100),
+            ),
+        ])
+    }
+}
+
+struct RunningStorage {
+    provider: Arc<PressureProbe>,
+    namespace: Mutex<()>,
+    io: Arc<Semaphore>,
+    first_started: Notify,
+    release_first: Notify,
+    first_finished: Notify,
+    second_finished: Notify,
+    started: AtomicUsize,
+    committed: AtomicUsize,
+}
+
+#[async_trait::async_trait]
+impl HealStorageAPI for RunningStorage {
+    async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
+        Ok(None)
+    }
+    async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
+        Ok(Vec::new())
+    }
+    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
+        Ok(Some(BucketInfo {
+            name: bucket.into(),
+            ..Default::default()
+        }))
+    }
+    async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
+        Ok(Vec::new())
+    }
+    async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
+        Ok(true)
+    }
+    async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
+        Ok(HealResultItem::default())
+    }
+    async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
+        Ok((HealResultItem::default(), None))
+    }
+    async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
+        Err(Error::other("no resume disk in bucket fixture"))
+    }
+    async fn list_objects_for_heal_page(
+        &self,
+        _: &str,
+        _: &str,
+        _: Option<&str>,
+        _: bool,
+    ) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
+        Ok((
+            ["a", "b"]
+                .into_iter()
+                .map(|name| HealListItem {
+                    name: name.into(),
+                    version_id: None,
+                    mod_time_unix_nanos: None,
+                    lifecycle_object_info: None,
+                    is_delete_marker: false,
+                })
+                .collect(),
+            None,
+            false,
+        ))
+    }
+    async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
+        let permit = self.io.clone().acquire_owned().await.expect("fixture I/O permit");
+        let namespace = self.namespace.lock().await;
+        self.provider.commit_open.store(true, Ordering::SeqCst);
+        let index = self.started.fetch_add(1, Ordering::SeqCst);
+        if index == 0 {
+            self.first_started.notify_one();
+            self.release_first.notified().await;
+        }
+        self.committed.fetch_add(1, Ordering::SeqCst);
+        self.provider.commit_open.store(false, Ordering::SeqCst);
+        drop(namespace);
+        drop(permit);
+        if index == 0 {
+            self.first_finished.notify_one();
+        } else {
+            self.second_finished.notify_one();
+        }
+        Ok((
+            HealResultItem {
+                object_size: 1,
+                ..Default::default()
+            },
+            None,
+        ))
+    }
+}
+
+async fn start_fixture(
+    provider_enabled: bool,
+    pacing_enabled: bool,
+    timeout: Duration,
+) -> (HealManager, Arc<RunningStorage>, Arc<PressureProbe>, Arc<HealTask>) {
+    let provider = Arc::new(PressureProbe::default());
+    let storage = Arc::new(RunningStorage {
+        provider: provider.clone(),
+        namespace: Mutex::new(()),
+        io: Arc::new(Semaphore::new(1)),
+        first_started: Notify::new(),
+        release_first: Notify::new(),
+        first_finished: Notify::new(),
+        second_finished: Notify::new(),
+        started: AtomicUsize::new(0),
+        committed: AtomicUsize::new(0),
+    });
+    let manager = HealManager::new_with_workload_provider(
+        storage.clone(),
+        Some(HealConfig {
+            mainline_throttle_enable: pacing_enabled,
+            mainline_read_utilization_high_percent: 80,
+            mainline_write_utilization_high_percent: 80,
+            mainline_max_sleep: Duration::from_millis(250),
+            max_concurrent_heals: 1,
+            ..HealConfig::default()
+        }),
+        provider_enabled.then(|| provider.clone() as WorkloadSnapshotProviderRef),
+    );
+    let mut request = bucket_request("running-mainline", HealPriority::High, HealRequestSource::Admin);
+    request.options.recursive = true;
+    request.options.timeout = Some(timeout);
+    let task_id = request.id.clone();
+    manager.submit_heal_request(request).await.expect("queue admin heal");
+    process_manager_queue_once(&manager).await;
+    storage.first_started.notified().await;
+    let task = manager
+        .active_heals
+        .lock()
+        .await
+        .get(&task_id)
+        .cloned()
+        .expect("running task");
+    (manager, storage, provider, task)
+}
+
+#[tokio::test(start_paused = true)]
+async fn running_mainline_admin_resamples_after_commit_and_yields_without_io_guards() {
+    let (_manager, storage, provider, _task) = start_fixture(true, true, Duration::from_secs(60)).await;
+    provider.active.store(100, Ordering::SeqCst);
+    assert!(storage.provider.commit_open.load(Ordering::SeqCst));
+    assert_eq!(storage.committed.load(Ordering::SeqCst), 0);
+    storage.release_first.notify_one();
+    storage.first_finished.notified().await;
+    tokio::time::timeout(Duration::from_millis(1), provider.high_sampled.notified())
+        .await
+        .expect("running admin heal must re-sample rising pressure before its next object");
+    assert_eq!(storage.started.load(Ordering::SeqCst), 1);
+    assert_eq!(
+        storage.committed.load(Ordering::SeqCst),
+        1,
+        "in-flight commit must finish despite pressure"
+    );
+    assert_eq!(storage.io.available_permits(), 1, "pacing must release I/O permits");
+    assert!(storage.namespace.try_lock().is_ok(), "pacing must not hold the namespace lock");
+    tokio::time::advance(Duration::from_millis(250)).await;
+    storage.second_finished.notified().await;
+    assert_eq!(
+        storage.committed.load(Ordering::SeqCst),
+        2,
+        "sustained pressure must still allow bounded maintenance progress"
+    );
+}
+
+#[tokio::test(start_paused = true)]
+async fn running_mainline_missing_provider_or_disabled_pacing_preserves_progress() {
+    for (provider_enabled, pacing_enabled) in [(false, true), (true, false)] {
+        let (_manager, storage, provider, _task) = start_fixture(provider_enabled, pacing_enabled, Duration::from_secs(60)).await;
+        provider.active.store(100, Ordering::SeqCst);
+        let before = tokio::time::Instant::now();
+        storage.release_first.notify_one();
+        storage.second_finished.notified().await;
+        assert_eq!(storage.committed.load(Ordering::SeqCst), 2);
+        assert_eq!(tokio::time::Instant::now(), before);
+        assert_eq!(storage.io.available_permits(), 1);
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn running_mainline_cancellation_and_deadline_leave_next_object_unstarted() {
+    for cancelled in [true, false] {
+        let (_manager, storage, provider, task) = start_fixture(true, true, Duration::from_millis(100)).await;
+        provider.active.store(100, Ordering::SeqCst);
+        storage.release_first.notify_one();
+        provider.high_sampled.notified().await;
+        if cancelled {
+            task.cancel_token.cancel();
+        } else {
+            tokio::time::advance(Duration::from_millis(100)).await;
+        }
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while matches!(task.get_status().await, HealTaskStatus::Running) {
+                tokio::time::sleep(Duration::from_millis(1)).await;
+            }
+        })
+        .await
+        .expect("pacing must not mask cancellation or timeout");
+        assert_eq!(storage.started.load(Ordering::SeqCst), 1);
+        assert_eq!(storage.committed.load(Ordering::SeqCst), 1);
+        assert_eq!(storage.io.available_permits(), 1);
+        assert!(storage.namespace.try_lock().is_ok());
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.counters.processed, 1);
+        assert_eq!(
+            task.get_status().await,
+            if cancelled {
+                HealTaskStatus::Cancelled
+            } else {
+                HealTaskStatus::Timeout
+            }
+        );
+    }
+}

crates/heal/src/heal/mod.rs

@@ -17,6 +17,7 @@ pub mod erasure_healer;
 pub mod manager;
 pub mod mrf_queue;
 pub mod outcome;
+pub(crate) mod pacing;
 pub mod progress;
 pub(crate) mod replacement_readiness;
 pub mod resume;

crates/heal/src/heal/pacing.rs

@@ -0,0 +1,238 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use crate::{Error, Result};
+use rustfs_concurrency::{
+    WorkloadAdmissionSnapshotProvider,
+    workload::{ForegroundPressure, foreground_pressure},
+};
+use std::{sync::Arc, time::Duration};
+use tokio::{sync::Mutex, time::Instant};
+use tokio_util::sync::CancellationToken;
+
+#[derive(Default)]
+struct PacingState {
+    throttled: bool,
+    low_since: Option<Instant>,
+}
+
+pub(crate) enum PacingDecision {
+    Ready,
+    Wait(Option<ForegroundPressure>),
+}
+
+/// Cooperative pacing for one admin execution, not a storage admission permit.
+pub(crate) struct MainlinePacer {
+    provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
+    read_high: usize,
+    write_high: usize,
+    pause: Duration,
+    state: Mutex<PacingState>,
+}
+
+impl MainlinePacer {
+    pub(crate) fn new(
+        provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
+        read_high: usize,
+        write_high: usize,
+        pause: Duration,
+    ) -> Option<Self> {
+        if (read_high == 0 && write_high == 0) || pause.is_zero() {
+            return None;
+        }
+        Some(Self {
+            provider,
+            read_high: read_high.min(100),
+            write_high: write_high.min(100),
+            pause: pause.min(Duration::from_secs(1)),
+            state: Mutex::new(PacingState::default()),
+        })
+    }
+
+    /// Fresh, nonblocking decision while the caller owns actual page capacity.
+    /// A contended pacing latch is conservative, but never awaited here.
+    pub(crate) fn admission_decision(&self) -> PacingDecision {
+        let snapshot = self.provider.workload_admission_snapshot();
+        let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high);
+        if pressure.is_none() && self.state.try_lock().is_ok_and(|state| !state.throttled) {
+            PacingDecision::Ready
+        } else {
+            PacingDecision::Wait(pressure)
+        }
+    }
+
+    /// Call only between storage operations, with no namespace lock or I/O
+    /// permit held. The pacing-only mutex serializes starts within this task;
+    /// each holder waits at most one pause so persistent pressure cannot stop
+    /// all maintenance progress. Cancellation also interrupts queued waiters.
+    pub(crate) async fn wait(&self, cancel: &CancellationToken) -> Result<()> {
+        self.wait_after_admission(cancel, None).await.map(|_| ())
+    }
+
+    /// Returns whether this unit paid a bounded pause. That grant permits one
+    /// unit even if pressure persists when page capacity becomes available.
+    pub(crate) async fn wait_after_admission(
+        &self,
+        cancel: &CancellationToken,
+        observed: Option<ForegroundPressure>,
+    ) -> Result<bool> {
+        let mut state = tokio::select! {
+            biased;
+            _ = cancel.cancelled() => return Err(Error::TaskCancelled),
+            state = self.state.lock() => state,
+        };
+        if observed.is_some() {
+            state.throttled = true;
+            state.low_since = None;
+        }
+        let snapshot = self.provider.workload_admission_snapshot();
+        let pressure = foreground_pressure(&snapshot, self.read_high, self.write_high);
+        if pressure.is_some() {
+            state.throttled = true;
+            state.low_since = None;
+        } else if state.throttled {
+            let low = |high: usize| if high == 0 { 0 } else { (high * 3 / 4).max(1) };
+            if foreground_pressure(&snapshot, low(self.read_high), low(self.write_high)).is_none() {
+                let now = Instant::now();
+                let since = state.low_since.get_or_insert(now);
+                if now.duration_since(*since) >= self.pause.saturating_mul(4) {
+                    state.throttled = false;
+                    state.low_since = None;
+                }
+            } else {
+                state.low_since = None;
+            }
+        }
+        if !state.throttled {
+            return Ok(false);
+        }
+        metrics::counter!(
+            "rustfs_heal_mainline_throttle_total",
+            "source" => "admin",
+            "result" => "delayed",
+            "reason" => pressure.or(observed).map_or("recovery_window", |pressure| pressure.reason())
+        )
+        .increment(1);
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::TaskCancelled),
+            _ = tokio::time::sleep(self.pause) => Ok(true),
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) struct TestPressure {
+    pub(crate) active: std::sync::atomic::AtomicUsize,
+    pub(crate) sampled: tokio::sync::Notify,
+    class: rustfs_concurrency::WorkloadClass,
+}
+
+#[cfg(test)]
+impl TestPressure {
+    pub(crate) fn new(class: rustfs_concurrency::WorkloadClass, active: usize) -> Self {
+        Self {
+            active: std::sync::atomic::AtomicUsize::new(active),
+            sampled: tokio::sync::Notify::new(),
+            class,
+        }
+    }
+}
+
+#[cfg(test)]
+impl WorkloadAdmissionSnapshotProvider for TestPressure {
+    fn workload_admission_snapshot(&self) -> rustfs_concurrency::WorkloadAdmissionRegistrySnapshot {
+        let active = self.active.load(std::sync::atomic::Ordering::SeqCst);
+        self.sampled.notify_one();
+        rustfs_concurrency::WorkloadAdmissionRegistrySnapshot::new(vec![
+            rustfs_concurrency::WorkloadAdmissionSnapshot::new(self.class, rustfs_concurrency::AdmissionState::Open).with_counts(
+                Some(active),
+                None,
+                Some(100),
+            ),
+        ])
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_concurrency::WorkloadClass;
+    use std::sync::atomic::Ordering;
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
+        for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
+            let provider = Arc::new(TestPressure::new(class, 0));
+            let pause = Duration::from_millis(250);
+            let pacer = MainlinePacer::new(
+                provider.clone(),
+                if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
+                if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
+                pause,
+            )
+            .expect("enabled pacer");
+            let cancel = CancellationToken::new();
+            let now = Instant::now();
+            pacer.wait(&cancel).await.expect("quiet work");
+            assert_eq!(Instant::now(), now);
+            // Low watermark is 30 for the configured high watermark 40.
+            for utilization in [40, 29, 35, 29, 29, 29, 29] {
+                provider.active.store(utilization, Ordering::SeqCst);
+                let before = Instant::now();
+                pacer.wait(&cancel).await.expect("bounded maintenance progress");
+                assert_eq!(Instant::now() - before, pause);
+            }
+            let before = Instant::now();
+            pacer.wait(&cancel).await.expect("stable low pressure restores unpaced work");
+            assert_eq!(Instant::now(), before);
+        }
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
+        assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1)).is_none());
+        assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO).is_none());
+        let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600)).expect("pacer");
+        let before = Instant::now();
+        pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
+        assert_eq!(Instant::now() - before, Duration::from_secs(1));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
+        let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
+        let paced = Arc::new(MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("pacer"));
+        let cancel_first = CancellationToken::new();
+        let first = tokio::spawn({
+            let paced = paced.clone();
+            let cancel = cancel_first.clone();
+            async move { paced.wait(&cancel).await }
+        });
+        provider.sampled.notified().await;
+        let cancel_second = CancellationToken::new();
+        let second = tokio::spawn({
+            let paced = paced.clone();
+            let cancel = cancel_second.clone();
+            async move { paced.wait(&cancel).await }
+        });
+        tokio::task::yield_now().await;
+        cancel_second.cancel();
+        assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
+        provider.active.store(0, Ordering::SeqCst);
+        let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1)).expect("independent task");
+        let before = Instant::now();
+        other_task
+            .wait(&CancellationToken::new())
+            .await
+            .expect("another task has no inherited latch");
+        assert_eq!(Instant::now(), before, "task/set pacing state must not be global");
+        cancel_first.cancel();
+        assert!(matches!(first.await.expect("sleeping waiter"), Err(Error::TaskCancelled)));
+        tokio::time::timeout(Duration::from_secs(2), paced.wait(&CancellationToken::new()))
+            .await
+            .expect("pacing lock released")
+            .expect("bounded work after cancellation");
+    }
+}

crates/heal/src/heal/task.rs

@@ -446,6 +446,7 @@ pub struct HealTask {
     pub cancel_token: tokio_util::sync::CancellationToken,
     /// Storage layer interface
     pub storage: Arc<dyn HealStorageAPI>,
+    mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
 }
 
 impl HealTask {
@@ -493,6 +494,7 @@ impl HealTask {
             task_start_instant: Arc::new(RwLock::new(None)),
             cancel_token: tokio_util::sync::CancellationToken::new(),
             storage,
+            mainline_pacer: None,
         }
     }
 
@@ -529,6 +531,18 @@ impl HealTask {
         task
     }
 
+    pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
+        self.mainline_pacer = pacer;
+        self
+    }
+
+    async fn pace_mainline(&self) -> Result<()> {
+        if let Some(pacer) = &self.mainline_pacer {
+            self.await_with_control(pacer.wait(&self.cancel_token)).await?;
+        }
+        Ok(())
+    }
+
     pub fn metric_type_label(&self) -> &'static str {
         self.heal_type.kind_label()
     }
@@ -933,24 +947,32 @@ impl HealTask {
         });
         self.emit_trace_task_state("started", Duration::ZERO, None);
 
-        let result = match &self.heal_type {
-            HealType::Cluster => self.heal_cluster().await,
-            HealType::Object {
-                bucket,
-                object,
-                version_id,
-            } => self.heal_object(bucket, object, version_id.as_deref()).await,
-            HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
-            HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await,
+        let result = async {
+            if self.heal_type.is_per_object() {
+                self.pace_mainline().await?;
+            }
+            match &self.heal_type {
+                HealType::Cluster => self.heal_cluster().await,
+                HealType::Object {
+                    bucket,
+                    object,
+                    version_id,
+                } => self.heal_object(bucket, object, version_id.as_deref()).await,
+                HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
+                HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await,
 
-            HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
-            HealType::ECDecode {
-                bucket,
-                object,
-                version_id,
-            } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
-            HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await,
-        };
+                HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
+                HealType::ECDecode {
+                    bucket,
+                    object,
+                    version_id,
+                } => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
+                HealType::ErasureSet { buckets, set_disk_id } => {
+                    self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await
+                }
+            }
+        }
+        .await;
 
         #[cfg(test)]
         pause_outcome_finish(&self.id).await;

crates/heal/src/heal/task/heal_bucket.rs

@@ -34,6 +34,7 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
 
 impl HealTask {
     pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
+        self.pace_mainline().await?;
         debug!(
             target: "rustfs::heal::task",
             event = EVENT_HEAL_BUCKET_STAGE,
@@ -308,6 +309,7 @@ impl HealTask {
                 self.check_control_flags().await?;
                 let mut listing_attempt = 0;
                 let (objects, next_token, is_truncated) = loop {
+                    self.pace_mainline().await?;
                     let page = if let Some(set_disk_id) = set_disk_id.as_deref() {
                         self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
                             set_disk_id,
@@ -362,6 +364,7 @@ impl HealTask {
                     let mut retry = Vec::with_capacity(pending.len());
                     for item in pending {
                         self.check_control_flags().await?;
+                        self.pace_mainline().await?;
                         let mut telemetry_unknown = false;
                         let object = item.name.as_str();
                         let identity =

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -422,7 +422,8 @@ impl HealTask {
             self.source,
         )
         .with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
-        .with_replacement_identity_fence(replacement_target_identities.clone());
+        .with_replacement_identity_fence(replacement_target_identities.clone())
+        .with_mainline_pacer(self.mainline_pacer.clone());
 
         {
             let mut progress = self.progress.write().await;

crates/iam/src/manager.rs

@@ -429,6 +429,27 @@ where
         }
     }
 
+    /// The cached mapping record for one user or group, looked up in the same
+    /// cache partition `policy_db_set` writes it to (group / STS / regular+service
+    /// user). `None` when no mapping is stored.
+    pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option<MappedPolicy> {
+        let cache = self.cache.snapshot();
+        if is_group {
+            cache.group_policies.get(name).cloned()
+        } else if user_type == UserType::Sts {
+            cache.sts_policies.get(name).cloned()
+        } else {
+            cache.user_policies.get(name).cloned()
+        }
+    }
+
+    /// The cached group record (members, status, own timestamp) without the
+    /// mapped-policy overlay `get_group_description` applies. `None` when the
+    /// group does not exist.
+    pub async fn get_group_info(&self, name: &str) -> Option<GroupInfo> {
+        self.cache.snapshot().groups.get(name).cloned()
+    }
+
     pub async fn get_policy(&self, name: &str) -> Result<Policy> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
@@ -534,6 +555,17 @@ where
     }
 
     pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
+        self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_policy`] stamping the document with `updated_at` instead
+    /// of the local clock.
+    ///
+    /// A site-replication receiver passes the edit's source time: the next
+    /// incoming revision is judged against the stored `UpdateDate`, so a
+    /// local stamp would reject a newer source edit that was merely delivered
+    /// later (backlog#2291). The returned stamp is the one persisted.
+    pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
         if name.is_empty() || policy.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -544,18 +576,17 @@ where
             .get(name)
             .map(|v| {
                 let mut p = v.clone();
-                p.update(policy.clone());
+                p.update_at(policy.clone(), updated_at);
                 p
             })
-            .unwrap_or_else(|| PolicyDoc::new(policy));
+            .unwrap_or_else(|| PolicyDoc::new_at(policy, updated_at));
 
         self.api.save_policy_doc(name, policy_doc.clone()).await?;
 
-        let now = OffsetDateTime::now_utc();
+        self.cache
+            .add_or_update_policy_doc(name, &policy_doc, OffsetDateTime::now_utc());
 
-        self.cache.add_or_update_policy_doc(name, &policy_doc, now);
-
-        Ok(now)
+        Ok(updated_at)
     }
 
     pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
@@ -789,6 +820,12 @@ where
 
     /// create a service account and update cache
     pub async fn add_service_account(&self, cred: Credentials) -> Result<OffsetDateTime> {
+        self.add_service_account_at(cred, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_service_account`] stamping the identity with `updated_at`
+    /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn add_service_account_at(&self, cred: Credentials, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
         if cred.access_key.is_empty() || cred.parent_user.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -800,7 +837,8 @@ where
         }
         drop(cache);
 
-        let u = UserIdentity::new(cred);
+        let mut u = UserIdentity::new(cred);
+        u.update_at = Some(updated_at);
 
         self.api
             .save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None)
@@ -808,10 +846,22 @@ where
 
         self.update_user_with_claims(&u.credentials.access_key, u.clone())?;
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
+        self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::update_service_account`] stamping the identity with
+    /// `updated_at` instead of the local clock; see [`Self::set_policy_at`]
+    /// (backlog#2291).
+    pub async fn update_service_account_at(
+        &self,
+        name: &str,
+        opts: UpdateServiceAccountOpts,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         let _mutation_guard = self.cache.service_account_mutation_lock().lock().await;
         let cache = self.cache.snapshot();
         let Some(ui) = cache.users.get(name).cloned() else {
@@ -858,13 +908,7 @@ where
         }
 
         if let Some(status) = opts.status {
-            match status.as_str() {
-                val if val == AccountStatus::Enabled.as_ref() => cr.status = auth::ACCOUNT_ON.to_owned(),
-                val if val == AccountStatus::Disabled.as_ref() => cr.status = auth::ACCOUNT_OFF.to_owned(),
-                auth::ACCOUNT_ON => cr.status = auth::ACCOUNT_ON.to_owned(),
-                auth::ACCOUNT_OFF => cr.status = auth::ACCOUNT_OFF.to_owned(),
-                _ => cr.status = auth::ACCOUNT_OFF.to_owned(),
-            }
+            cr.status = account_status_flag(&status).to_owned();
         }
 
         let mut m: HashMap<String, Value> = if token_without_expiration {
@@ -916,8 +960,8 @@ where
 
         cr.session_token = jwt_sign(&m, &cr.secret_key)?;
 
-        let u = UserIdentity::new(cr);
-        let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc);
+        let mut u = UserIdentity::new(cr);
+        u.update_at = Some(updated_at);
         self.api
             .save_user_identity(&u.credentials.access_key, UserType::Svc, u.clone(), None)
             .await?;
@@ -1149,6 +1193,20 @@ where
         Ok((policies.into_iter().collect(), update_at))
     }
     pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
+        self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::policy_db_set`] stamping the mapping with `updated_at` instead
+    /// of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn policy_db_set_at(
+        &self,
+        name: &str,
+        user_type: UserType,
+        is_group: bool,
+        policy: &str,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1168,10 +1226,11 @@ where
                 self.cache.delete_user_policy(name, OffsetDateTime::now_utc());
             }
 
-            return Ok(OffsetDateTime::now_utc());
+            return Ok(updated_at);
         }
 
-        let mp = MappedPolicy::new(policy);
+        let mut mp = MappedPolicy::new(policy);
+        mp.update_at = updated_at;
 
         let cache = self.cache.snapshot();
         let policy_docs_cache = Arc::clone(&cache.policy_docs);
@@ -1194,7 +1253,7 @@ where
             self.cache.add_or_update_user_policy(name, &mp, OffsetDateTime::now_utc());
         }
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn set_temp_user(&self, access_key: &str, cred: &Credentials, policy_name: Option<&str>) -> Result<OffsetDateTime> {
@@ -1391,6 +1450,17 @@ where
     }
 
     pub async fn add_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
+        self.add_user_at(access_key, args, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_user`] stamping the identity with `updated_at` instead of
+    /// the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn add_user_at(
+        &self,
+        access_key: &str,
+        args: &AddOrUpdateUserReq,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         let cache = self.cache.snapshot();
         let users = Arc::clone(&cache.users);
         if let Some(x) = users.get(access_key) {
@@ -1408,12 +1478,13 @@ where
                 _ => auth::ACCOUNT_OFF,
             }
         };
-        let user_entry = UserIdentity::from(Credentials {
+        let mut user_entry = UserIdentity::from(Credentials {
             access_key: access_key.to_string(),
             secret_key: args.secret_key.to_string(),
             status: status.to_owned(),
             ..Default::default()
         });
+        user_entry.update_at = Some(updated_at);
 
         self.api
             .save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
@@ -1421,7 +1492,7 @@ where
 
         self.update_user_with_claims(access_key, user_entry)?;
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn delete_user(&self, access_key: &str, utype: UserType) -> Result<()> {
@@ -1599,6 +1670,17 @@ where
     }
 
     pub async fn set_user_status(&self, access_key: &str, status: AccountStatus) -> Result<OffsetDateTime> {
+        self.set_user_status_at(access_key, status, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_user_status`] stamping the identity with `updated_at`
+    /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn set_user_status_at(
+        &self,
+        access_key: &str,
+        status: AccountStatus,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if access_key.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1625,12 +1707,13 @@ where
             }
         };
 
-        let user_entry = UserIdentity::from(Credentials {
+        let mut user_entry = UserIdentity::from(Credentials {
             access_key: access_key.to_string(),
             secret_key: u.credentials.secret_key.clone(),
             status: status.to_owned(),
             ..Default::default()
         });
+        user_entry.update_at = Some(updated_at);
         drop(cache);
         drop(users);
 
@@ -1640,7 +1723,7 @@ where
 
         self.update_user_with_claims(access_key, user_entry)?;
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     fn update_user_with_claims(&self, k: &str, u: UserIdentity) -> Result<()> {
@@ -1676,6 +1759,17 @@ where
     }
 
     pub async fn add_users_to_group(&self, group: &str, members: Vec<String>) -> Result<OffsetDateTime> {
+        self.add_users_to_group_at(group, members, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_users_to_group`] stamping the group with `updated_at`
+    /// instead of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn add_users_to_group_at(
+        &self,
+        group: &str,
+        members: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if group.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1693,6 +1787,14 @@ where
             }
         }
 
+        // The group's own timestamp moves with every membership or status
+        // change: site replication judges an incoming group item against it
+        // (backlog#2291), so it must reflect the last change, not creation.
+        // `updated_at` is the record's stamp only; the cache is published
+        // with the local clock, because `LockedCache::exec` drops a write
+        // whose time predates the entity's load time — a replicated edit
+        // whose source time is older than this node's startup would
+        // otherwise never reach the cache.
         let gi = match cache.groups.get(group) {
             Some(res) => {
                 let mut gi = res.clone();
@@ -1701,15 +1803,20 @@ where
                 uniq_set.extend(members.iter().cloned());
 
                 gi.members = uniq_set.into_iter().collect();
+                gi.update_at = Some(updated_at);
+                gi
+            }
+            None => {
+                let mut gi = GroupInfo::new(members.clone());
+                gi.update_at = Some(updated_at);
                 gi
             }
-            None => GroupInfo::new(members.clone()),
         };
         drop(cache);
 
         self.api.save_group_info(group, gi.clone()).await?;
 
-        let now = self.cache.with_write_lock(|cache| {
+        self.cache.with_write_lock(|cache| {
             let now = OffsetDateTime::now_utc();
             cache.add_or_update_group(group, &gi, now);
 
@@ -1719,13 +1826,18 @@ where
                 m.insert(group.to_string());
                 cache.add_or_update_user_group_membership(member, &m, now);
             });
-            now
         });
 
-        Ok(now)
+        Ok(updated_at)
     }
 
     pub async fn set_group_status(&self, name: &str, enable: bool) -> Result<OffsetDateTime> {
+        self.set_group_status_at(name, enable, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_group_status`] stamping the group with `updated_at` instead
+    /// of the local clock; see [`Self::set_policy_at`] (backlog#2291).
+    pub async fn set_group_status_at(&self, name: &str, enable: bool, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
         if name.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1743,12 +1855,15 @@ where
         } else {
             gi.status = STATUS_DISABLED.to_owned();
         }
+        gi.update_at = Some(updated_at);
 
         self.api.save_group_info(name, gi.clone()).await?;
 
+        // Cache publication time is the local clock, not the record stamp
+        // (see `add_users_to_group_at`).
         self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
 
-        Ok(OffsetDateTime::now_utc())
+        Ok(updated_at)
     }
 
     pub async fn get_group_description(&self, name: &str) -> Result<GroupDesc> {
@@ -1818,6 +1933,20 @@ where
         name: &str,
         members: Vec<String>,
         update_cache_only: bool,
+    ) -> Result<OffsetDateTime> {
+        self.remove_members_from_group_at(name, members, update_cache_only, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::remove_members_from_group`] stamping the group with
+    /// `updated_at` instead of the local clock; see [`Self::set_policy_at`]
+    /// (backlog#2291).
+    pub async fn remove_members_from_group_at(
+        &self,
+        name: &str,
+        members: Vec<String>,
+        update_cache_only: bool,
+        updated_at: OffsetDateTime,
     ) -> Result<OffsetDateTime> {
         let cache = self.cache.snapshot();
         let mut gi = cache
@@ -1830,12 +1959,14 @@ where
         let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
         let d: HashSet<&String> = HashSet::from_iter(members.iter());
         gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
-
+        gi.update_at = Some(updated_at);
         if !update_cache_only {
             self.api.save_group_info(name, gi.clone()).await?;
         }
 
-        let now = self.cache.with_write_lock(|cache| {
+        self.cache.with_write_lock(|cache| {
+            // Sample after storage completes so a concurrent reload cannot
+            // make this publication older than the cache it must update.
             let now = OffsetDateTime::now_utc();
             cache.add_or_update_group(name, &gi, now);
 
@@ -1847,13 +1978,25 @@ where
                     cache.add_or_update_user_group_membership(member, &m, now);
                 }
             });
-            now
         });
 
-        Ok(now)
+        Ok(updated_at)
     }
 
     pub async fn remove_users_from_group(&self, group: &str, members: Vec<String>) -> Result<OffsetDateTime> {
+        self.remove_users_from_group_at(group, members, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::remove_users_from_group`] stamping the group with `updated_at`
+    /// instead of the local clock; a group delete (no members) leaves no
+    /// record and returns the stamp unchanged (backlog#2291).
+    pub async fn remove_users_from_group_at(
+        &self,
+        group: &str,
+        members: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if group.is_empty() {
             return Err(Error::InvalidArgument);
         }
@@ -1902,18 +2045,17 @@ where
                 return Err(err);
             }
 
-            let now = self.cache.with_write_lock(|cache| {
+            self.cache.with_write_lock(|cache| {
                 let now = OffsetDateTime::now_utc();
                 self.remove_group_from_memberships_map_unlocked(cache, group, now);
                 cache.delete_group(group, now);
                 cache.delete_group_policy(group, now);
-                now
             });
 
-            return Ok(now);
+            return Ok(updated_at);
         }
 
-        self.remove_members_from_group(group, members, false).await
+        self.remove_members_from_group_at(group, members, false, updated_at).await
     }
 
     fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
@@ -2235,6 +2377,19 @@ where
     }
 }
 
+/// The stored `status` flag for a service-account status given on the admin
+/// or replication wire: the madmin `enabled` / `disabled` words and the stored
+/// `on` / `off` flags are both accepted; anything else disables the account.
+pub(crate) fn account_status_flag(status: &str) -> &'static str {
+    match status {
+        val if val == AccountStatus::Enabled.as_ref() => auth::ACCOUNT_ON,
+        val if val == AccountStatus::Disabled.as_ref() => auth::ACCOUNT_OFF,
+        auth::ACCOUNT_ON => auth::ACCOUNT_ON,
+        auth::ACCOUNT_OFF => auth::ACCOUNT_OFF,
+        _ => auth::ACCOUNT_OFF,
+    }
+}
+
 pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
     let default_policies = &DEFAULT_POLICIES;
     default_policies

crates/iam/src/sys.rs

@@ -385,7 +385,14 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
-        let updated_at = self.store.set_policy(name, policy).await?;
+        self.set_policy_at(name, policy, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_policy`] stamping the document with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock; see
+    /// `IamCache::set_policy_at` (backlog#2291).
+    pub async fn set_policy_at(&self, name: &str, policy: Policy, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
+        let updated_at = self.store.set_policy_at(name, policy, updated_at).await?;
 
         if !self.has_watcher() {
             for r in notify_iam_load_policy(name).await {
@@ -643,7 +650,18 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_user_status(&self, name: &str, status: rustfs_madmin::AccountStatus) -> Result<OffsetDateTime> {
-        let updated_at = self.store.set_user_status(name, status).await?;
+        self.set_user_status_at(name, status, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_user_status`] stamping the identity with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn set_user_status_at(
+        &self,
+        name: &str,
+        status: rustfs_madmin::AccountStatus,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        let updated_at = self.store.set_user_status_at(name, status, updated_at).await?;
 
         self.notify_for_user(name, false).await;
 
@@ -655,6 +673,20 @@ impl<T: Store> IamSys<T> {
         parent_user: &str,
         groups: Option<Vec<String>>,
         opts: NewServiceAccountOpts,
+    ) -> Result<(Credentials, OffsetDateTime)> {
+        self.new_service_account_at(parent_user, groups, opts, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::new_service_account`] stamping the identity with `updated_at`
+    /// (a replicated edit's source time) instead of the local clock
+    /// (backlog#2291).
+    pub async fn new_service_account_at(
+        &self,
+        parent_user: &str,
+        groups: Option<Vec<String>>,
+        opts: NewServiceAccountOpts,
+        updated_at: OffsetDateTime,
     ) -> Result<(Credentials, OffsetDateTime)> {
         if parent_user.is_empty() {
             return Err(IamError::InvalidArgument);
@@ -724,11 +756,18 @@ impl<T: Store> IamSys<T> {
         let mut cred = create_new_credentials_with_metadata(&access_key, &secret_key, &m, &secret_key)?;
         cred.parent_user = parent_user.to_owned();
         cred.groups = groups;
-        cred.status = ACCOUNT_ON.to_owned();
+        // The status is part of the created identity: a replicated disabled
+        // account must never exist enabled, not even between a create and a
+        // follow-up status write (backlog#2289).
+        cred.status = opts
+            .status
+            .as_deref()
+            .map_or(ACCOUNT_ON, crate::manager::account_status_flag)
+            .to_owned();
         cred.name = opts.name;
         cred.description = opts.description;
 
-        let create_at = self.store.add_service_account(cred.clone()).await?;
+        let create_at = self.store.add_service_account_at(cred.clone(), updated_at).await?;
 
         self.notify_for_service_account(&cred.access_key).await;
 
@@ -736,11 +775,23 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn update_service_account(&self, name: &str, opts: UpdateServiceAccountOpts) -> Result<OffsetDateTime> {
+        self.update_service_account_at(name, opts, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::update_service_account`] stamping the identity with
+    /// `updated_at` (a replicated edit's source time) instead of the local
+    /// clock (backlog#2291).
+    pub async fn update_service_account_at(
+        &self,
+        name: &str,
+        opts: UpdateServiceAccountOpts,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if name == SITE_REPLICATOR_SERVICE_ACCOUNT && !opts.allow_site_replicator_account {
             return Err(IamError::IAMActionNotAllowed);
         }
 
-        let updated_at = self.store.update_service_account(name, opts).await?;
+        let updated_at = self.store.update_service_account_at(name, opts, updated_at).await?;
 
         self.notify_for_service_account(name).await;
 
@@ -940,6 +991,17 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn create_user(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
+        self.create_user_at(access_key, args, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::create_user`] stamping the identity with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn create_user_at(
+        &self,
+        access_key: &str,
+        args: &AddOrUpdateUserReq,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if !is_access_key_valid(access_key) {
             return Err(IamError::InvalidAccessKeyLength);
         }
@@ -952,7 +1014,7 @@ impl<T: Store> IamSys<T> {
             return Err(IamError::InvalidSecretKeyLength);
         }
 
-        let updated_at = self.store.add_user(access_key, args).await?;
+        let updated_at = self.store.add_user_at(access_key, args, updated_at).await?;
         self.load_user(access_key, UserType::Reg).await?;
 
         self.notify_for_user(access_key, false).await;
@@ -1026,10 +1088,21 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn add_users_to_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
+        self.add_users_to_group_at(group, users, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::add_users_to_group`] stamping the group with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn add_users_to_group_at(
+        &self,
+        group: &str,
+        users: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
         if contains_reserved_chars(group) {
             return Err(IamError::GroupNameContainsReservedChars);
         }
-        let updated_at = self.store.add_users_to_group(group, users).await?;
+        let updated_at = self.store.add_users_to_group_at(group, users, updated_at).await?;
 
         self.notify_for_group(group).await;
 
@@ -1037,7 +1110,19 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn remove_users_from_group(&self, group: &str, users: Vec<String>) -> Result<OffsetDateTime> {
-        let updated_at = self.store.remove_users_from_group(group, users).await?;
+        self.remove_users_from_group_at(group, users, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::remove_users_from_group`] stamping the group with `updated_at`
+    /// (a replicated edit's source time) instead of the local clock
+    /// (backlog#2291).
+    pub async fn remove_users_from_group_at(
+        &self,
+        group: &str,
+        users: Vec<String>,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        let updated_at = self.store.remove_users_from_group_at(group, users, updated_at).await?;
 
         self.notify_for_group(group).await;
 
@@ -1045,7 +1130,13 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_group_status(&self, group: &str, enable: bool) -> Result<OffsetDateTime> {
-        let updated_at = self.store.set_group_status(group, enable).await?;
+        self.set_group_status_at(group, enable, OffsetDateTime::now_utc()).await
+    }
+
+    /// [`Self::set_group_status`] stamping the group with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn set_group_status_at(&self, group: &str, enable: bool, updated_at: OffsetDateTime) -> Result<OffsetDateTime> {
+        let updated_at = self.store.set_group_status_at(group, enable, updated_at).await?;
 
         self.notify_for_group(group).await;
 
@@ -1055,6 +1146,22 @@ impl<T: Store> IamSys<T> {
         self.store.get_group_description(group).await
     }
 
+    /// The stored group record itself (see `IamCache::get_group_info`).
+    pub async fn get_group_info(&self, group: &str) -> Option<GroupInfo> {
+        self.store.get_group_info(group).await
+    }
+
+    /// The stored policy document, `Error::NoSuchPolicy` when absent.
+    pub async fn get_policy_doc(&self, name: &str) -> Result<PolicyDoc> {
+        self.store.get_policy_doc(name).await
+    }
+
+    /// The stored mapping record for one user or group (see
+    /// `IamCache::get_mapped_policy_record`).
+    pub async fn get_mapped_policy_record(&self, name: &str, user_type: UserType, is_group: bool) -> Option<MappedPolicy> {
+        self.store.get_mapped_policy_record(name, user_type, is_group).await
+    }
+
     pub async fn list_groups_load(&self) -> Result<Vec<String>> {
         self.store.update_groups().await
     }
@@ -1064,7 +1171,24 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
-        let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
+        self.policy_db_set_at(name, user_type, is_group, policy, OffsetDateTime::now_utc())
+            .await
+    }
+
+    /// [`Self::policy_db_set`] stamping the mapping with `updated_at` (a
+    /// replicated edit's source time) instead of the local clock (backlog#2291).
+    pub async fn policy_db_set_at(
+        &self,
+        name: &str,
+        user_type: UserType,
+        is_group: bool,
+        policy: &str,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        let updated_at = self
+            .store
+            .policy_db_set_at(name, user_type, is_group, policy, updated_at)
+            .await?;
 
         if !self.has_watcher() {
             for r in notify_iam_load_policy_mapping(name, user_type.to_u64(), is_group).await {
@@ -1846,6 +1970,11 @@ pub struct NewServiceAccountOpts {
     pub expiration: Option<OffsetDateTime>,
     pub allow_site_replicator_account: bool,
     pub claims: Option<HashMap<String, Value>>,
+    /// Status the account is created with (`enabled` / `disabled` or the
+    /// stored `on` / `off` flags); `None` creates it enabled. Site
+    /// replication passes the source account's status so a disabled account
+    /// is never enabled on the peer, not even transiently (backlog#2289).
+    pub status: Option<String>,
 }
 
 pub struct UpdateServiceAccountOpts {
@@ -2081,6 +2210,9 @@ mod tests {
         block_delete: Arc<std::sync::atomic::AtomicBool>,
         delete_started: Arc<tokio::sync::Notify>,
         release_delete: Arc<tokio::sync::Notify>,
+        block_group_save: Arc<std::sync::atomic::AtomicBool>,
+        group_save_started: Arc<tokio::sync::Notify>,
+        group_save_release: Arc<tokio::sync::Notify>,
     }
 
     impl StsTestMockStore {
@@ -2094,6 +2226,9 @@ mod tests {
                 block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
                 delete_started: Arc::new(tokio::sync::Notify::new()),
                 release_delete: Arc::new(tokio::sync::Notify::new()),
+                block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                group_save_started: Arc::new(tokio::sync::Notify::new()),
+                group_save_release: Arc::new(tokio::sync::Notify::new()),
             }
         }
 
@@ -2197,11 +2332,15 @@ mod tests {
         }
 
         async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
-            Err(Error::InvalidArgument)
+            if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
+                self.group_save_started.notify_one();
+                self.group_save_release.notified().await;
+            }
+            Ok(())
         }
 
         async fn delete_group_info(&self, _name: &str) -> Result<()> {
-            Err(Error::InvalidArgument)
+            Ok(())
         }
 
         async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()> {
@@ -2378,6 +2517,140 @@ mod tests {
         IamSys::new(cache)
     }
 
+    async fn assert_group_write_during_reload_is_published(remove: bool) {
+        let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
+        let member = "sts-fallback-test-parent";
+        let group = if remove { "testgroup" } else { "new-published-group" };
+        let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        iam_sys
+            .store
+            .api
+            .block_group_save
+            .store(true, std::sync::atomic::Ordering::SeqCst);
+        let before = iam_sys.store.cache.snapshot();
+        let writer_iam = iam_sys.clone();
+        let writer = tokio::spawn(async move {
+            if remove {
+                writer_iam
+                    .remove_users_from_group_at(group, vec![member.to_string()], source_time)
+                    .await
+            } else {
+                writer_iam
+                    .add_users_to_group_at(group, vec![member.to_string()], source_time)
+                    .await
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(5), iam_sys.store.api.group_save_started.notified())
+            .await
+            .expect("group save should reach the barrier");
+        // The pending store write has not changed the cache, so the production
+        // full-reload snapshot guard permits this replacement.
+        assert!(iam_sys.store.cache.with_write_lock(|cache| cache.matches_snapshot(&before)));
+        iam_sys
+            .store
+            .api
+            .load_all(&iam_sys.store.cache)
+            .await
+            .expect("reload while group save is pending");
+        iam_sys.store.api.group_save_release.notify_one();
+        assert_eq!(writer.await.expect("join group writer").expect("group write should succeed"), source_time);
+        let info = iam_sys
+            .get_group_info(group)
+            .await
+            .expect("successful group write must remain readable after reload");
+        assert_eq!(info.update_at, Some(source_time), "source timestamp must remain on the record");
+        assert_eq!(info.members, if remove { Vec::new() } else { vec![member.to_string()] });
+        let groups = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
+        assert_eq!(
+            groups.is_some_and(|groups| groups.contains(group)),
+            !remove,
+            "membership index must reflect the write"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn add_group_write_during_reload_publishes_after_store_save() {
+        assert_group_write_during_reload_is_published(false).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn remove_group_write_during_reload_publishes_after_store_save() {
+        assert_group_write_during_reload_is_published(true).await;
+    }
+
+    /// Review finding on rustfs#7195: a replicated group edit carries a source
+    /// stamp that may predate this node's cache load time. The stamp belongs on
+    /// the record only; publishing the cache with it makes `LockedCache::exec`
+    /// drop the write, so the group is written to the store but unreadable
+    /// here and the receiver's next `set_group_status_at` fails with
+    /// `NoSuchGroup`. Add, status and removal must all publish with the local
+    /// clock while keeping the source stamp on `GroupInfo::update_at`.
+    #[tokio::test]
+    async fn group_writes_stamped_before_the_cache_load_time_still_publish() {
+        let iam_sys = test_iam_sys().await;
+        let member = "group-stamp-member";
+        let identity = UserIdentity {
+            version: 1,
+            credentials: Credentials {
+                access_key: member.to_string(),
+                secret_key: "longenoughsecret".to_string(),
+                status: "on".to_string(),
+                ..Default::default()
+            },
+            update_at: Some(OffsetDateTime::now_utc()),
+        };
+        iam_sys.store.cache.with_write_lock(|cache| {
+            cache.add_or_update_user(member, &identity, OffsetDateTime::now_utc());
+            // The startup load publishes every entity with the load time.
+            cache.replace_groups(CacheEntity::new(HashMap::new()));
+            cache.replace_user_group_memberships(CacheEntity::new(HashMap::new()));
+        });
+
+        let group = "group-stamp";
+        let source_time = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        let stamped = iam_sys
+            .add_users_to_group_at(group, vec![member.to_string()], source_time)
+            .await
+            .expect("add members with a source stamp older than the cache load");
+        assert_eq!(stamped, source_time, "the returned stamp is the source time");
+        let info = iam_sys
+            .get_group_info(group)
+            .await
+            .expect("the group must be readable right after the add");
+        assert_eq!(info.members, vec![member.to_string()]);
+        assert_eq!(info.update_at, Some(source_time), "the record keeps the source stamp");
+        let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
+        assert!(
+            memberships.is_some_and(|groups| groups.contains(group)),
+            "the membership index is published too"
+        );
+
+        let disabled_at = source_time + time::Duration::seconds(1);
+        iam_sys
+            .set_group_status_at(group, false, disabled_at)
+            .await
+            .expect("status change with a source stamp older than the cache load");
+        let info = iam_sys.get_group_info(group).await.expect("group after status change");
+        assert_eq!(info.status, "disabled");
+        assert_eq!(info.update_at, Some(disabled_at));
+
+        let removed_at = source_time + time::Duration::seconds(2);
+        iam_sys
+            .remove_users_from_group_at(group, vec![member.to_string()], removed_at)
+            .await
+            .expect("removal with a source stamp older than the cache load");
+        let info = iam_sys.get_group_info(group).await.expect("group after removal");
+        assert!(info.members.is_empty(), "the removal must be visible in the cache");
+        assert_eq!(info.update_at, Some(removed_at));
+        let memberships = iam_sys.store.cache.snapshot().user_group_memberships.get(member).cloned();
+        assert!(
+            !memberships.is_some_and(|groups| groups.contains(group)),
+            "the membership index follows the removal"
+        );
+    }
+
     fn service_account_opts(access_key: &str, secret_key: &str) -> NewServiceAccountOpts {
         NewServiceAccountOpts {
             access_key: access_key.to_string(),

crates/madmin/fixtures/on_demand_migration/get_response.json

@@ -1 +1 @@
-{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
+{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}

crates/madmin/fixtures/on_demand_migration/set_request.json

@@ -1 +1 @@
-{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
+{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}

crates/madmin/fixtures/on_demand_migration/set_response.json

@@ -1 +1 @@
-{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
+{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}

crates/madmin/fixtures/on_demand_migration/source_config_e2a.rs

@@ -0,0 +1,80 @@
+// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01.
+// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted.
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+const REDACTED: &str = "REDACTED";
+
+/// The external S3-compatible source bucket.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SourceConfig {
+    pub provider: Provider,
+    /// `http(s)://host[:port]` with no path or query. Optional only for
+    /// [`Provider::Aws`], where it is derived from `region`.
+    #[serde(default)]
+    pub endpoint: Option<String>,
+    pub region: String,
+    pub bucket: String,
+    #[serde(default)]
+    pub path_style: PathStyle,
+    /// `None` means anonymous access to a public source bucket.
+    #[serde(default)]
+    pub credentials: Option<SourceCredentials>,
+    #[serde(default)]
+    pub tls: TlsConfig,
+}
+
+/// Source vendor family. `azure` is deliberately absent from this version.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Provider {
+    /// Generic S3-compatible endpoint.
+    S3,
+    Aws,
+    Minio,
+    Rustfs,
+    R2,
+    /// GCS XML interoperability API with HMAC keys.
+    Gcs,
+}
+
+/// Bucket addressing style. `auto` is resolved by the source client builder.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum PathStyle {
+    #[default]
+    Auto,
+    Path,
+    Virtual,
+}
+
+/// Static credentials for the source. `Debug` never prints the secret or
+/// the session token.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SourceCredentials {
+    pub access_key: String,
+    pub secret_key: String,
+    #[serde(default)]
+    pub session_token: Option<String>,
+}
+
+impl fmt::Debug for SourceCredentials {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SourceCredentials")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &REDACTED)
+            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED))
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TlsConfig {
+    #[serde(default)]
+    pub skip_verify: bool,
+    #[serde(default)]
+    pub ca_cert_pem: Option<String>,
+}

crates/madmin/src/on_demand_migration.rs

@@ -85,10 +85,10 @@ pub struct OnDemandMigrationSource {
     #[serde(default)]
     pub tls: OnDemandMigrationTls,
     /// Required for `azure` and rejected for every other provider.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub azure: Option<OnDemandMigrationAzure>,
     /// Required for `gcs_native` and rejected for every other provider.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub gcs: Option<OnDemandMigrationGcs>,
 }
 
@@ -651,6 +651,10 @@ mod tests {
     use super::*;
     use crate::test_support::TestServer;
 
+    mod before_native_sources {
+        include!("../fixtures/on_demand_migration/source_config_e2a.rs");
+    }
+
     const SET_REQUEST_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_request.json");
     const SET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_response.json");
     const GET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/get_response.json");
@@ -684,6 +688,20 @@ mod tests {
         assert_eq!(config.source.tls, OnDemandMigrationTls::default());
     }
 
+    #[test]
+    fn s3_admin_writes_remain_readable_by_the_strict_pre_native_server() {
+        for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] {
+            let historical = SET_REQUEST_FIXTURE.replace("\"provider\":\"minio\"", &format!("\"provider\":\"{provider}\""));
+            let config: OnDemandMigrationConfig = serde_json::from_str(&historical).expect("historical set request");
+            let wire = serde_json::to_string(&config).expect("current admin set request");
+            let actual: serde_json::Value = serde_json::from_str(&wire).expect("admin request JSON");
+            let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone())
+                .expect("the strict e2a server must accept an ordinary S3 source from the new admin client");
+            assert_eq!(serde_json::to_value(old_source).expect("old source wire"), actual["source"]);
+            assert_eq!(wire, historical.trim(), "provider={provider}: preserve the historical request bytes");
+        }
+    }
+
     #[test]
     fn set_response_fixture_round_trips_and_is_redacted() {
         let response: OnDemandMigrationSetResponse = round_trip(SET_RESPONSE_FIXTURE);
@@ -878,11 +896,11 @@ mod tests {
         for (label, json) in [
             (
                 "azure",
-                r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#,
+                r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"}}"#,
             ),
             (
                 "gcs_native",
-                r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
+                r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
             ),
         ] {
             let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}"));
@@ -891,6 +909,16 @@ mod tests {
                 json,
                 "{label} must reproduce the server wire shape byte for byte"
             );
+            let mut wire: serde_json::Value = serde_json::from_str(json).expect("native wire fixture");
+            assert!(
+                serde_json::from_value::<before_native_sources::SourceConfig>(wire.clone()).is_err(),
+                "native provider names and fields still require an upgraded server"
+            );
+            wire[if label == "azure" { "gcs" } else { "azure" }] = serde_json::Value::Null;
+            assert_eq!(
+                serde_json::from_value::<OnDemandMigrationSource>(wire).expect("the prior explicit-null wire still decodes"),
+                source
+            );
         }
 
         let azure = OnDemandMigrationAzure {
@@ -945,6 +973,10 @@ mod tests {
                 .is_some_and(|auth| auth.starts_with("AWS4-HMAC-SHA256"))
         );
         assert_eq!(request.body, SET_REQUEST_FIXTURE.trim(), "the body is the canonical config document");
+        let body: serde_json::Value = serde_json::from_str(&request.body).expect("signed admin request JSON");
+        let old_source: before_native_sources::SourceConfig = serde_json::from_value(body["source"].clone())
+            .expect("the strict pre-native server must accept the actual signed PUT source");
+        assert_eq!(old_source.provider, before_native_sources::Provider::Minio);
     }
 
     #[tokio::test]

crates/policy/src/policy/doc.rs

@@ -45,18 +45,33 @@ pub struct PolicyDoc {
 
 impl PolicyDoc {
     pub fn new(policy: Policy) -> Self {
+        Self::new_at(policy, OffsetDateTime::now_utc())
+    }
+
+    /// [`Self::new`] with an explicit `UpdateDate` (and `CreateDate`).
+    ///
+    /// A replicated document keeps the edit's source time: the receiver
+    /// judges the next incoming revision against the stored stamp, so a
+    /// local stamp would reject a newer source edit that was merely
+    /// delivered later.
+    pub fn new_at(policy: Policy, at: OffsetDateTime) -> Self {
         Self {
             version: 1,
             policy,
-            create_date: Some(OffsetDateTime::now_utc()),
-            update_date: Some(OffsetDateTime::now_utc()),
+            create_date: Some(at),
+            update_date: Some(at),
         }
     }
 
     pub fn update(&mut self, policy: Policy) {
+        self.update_at(policy, OffsetDateTime::now_utc());
+    }
+
+    /// [`Self::update`] with an explicit `UpdateDate`; see [`Self::new_at`].
+    pub fn update_at(&mut self, policy: Policy, at: OffsetDateTime) {
         self.version += 1;
         self.policy = policy;
-        self.update_date = Some(OffsetDateTime::now_utc());
+        self.update_date = Some(at);
 
         if self.create_date.is_none() {
             self.create_date = self.update_date;

crates/replication/src/delete.rs

@@ -76,6 +76,21 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
                 .delete_object
                 .delete_marker_mtime
                 .and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
+            // Carry the target-assigned marker version ids (and the fail-closed corrupt
+            // flag) into the journal so a purge intent replayed after a restart addresses
+            // the same version the live path did (backlog#2290). Only delete-marker state
+            // ever records these; other deletes serialize an empty map.
+            target_delete_marker_version_ids: self
+                .delete_object
+                .replication_state
+                .as_ref()
+                .map(|state| state.target_delete_marker_version_ids.clone())
+                .unwrap_or_default(),
+            target_delete_marker_version_ids_corrupt: self
+                .delete_object
+                .replication_state
+                .as_ref()
+                .is_some_and(|state| state.target_delete_marker_version_ids_corrupt),
             target_arns: self.admitted_target_arns(),
             force_delete_id: self.delete_object.force_delete_id,
             force_delete_generation: self.delete_object.force_delete_generation,
@@ -238,6 +253,28 @@ pub fn delete_marker_purge_version_id(
     })
 }
 
+/// The version a delete replication addresses on `arn`, or `None` to refuse.
+///
+/// A version purge whose purged version is a delete marker must address the
+/// marker version the TARGET assigned — the recorded mapping, exactly as the
+/// delayed-purge watcher does. The source-side `DELETE ?versionId=<marker>`
+/// replicates as such a purge, and a generic S3 target answers a DELETE of an
+/// unknown versionId with 204 while keeping its marker, so addressing it by
+/// the source id reported success and left the marker behind (backlog#2290,
+/// R6.1 on the VMs). Nothing recorded falls back to the source-derived id
+/// (id-mirroring peers); a corrupt record refuses, as the watcher does.
+pub fn delete_replication_target_version_id(dobj: &DeletedObject, arn: &str) -> Option<Option<String>> {
+    let is_version_purge = is_version_delete_replication(dobj);
+    if is_version_purge
+        && !dobj.delete_marker
+        && let Some(marker) = dobj.delete_marker_version_id
+    {
+        return delete_marker_purge_version_id(dobj.replication_state.as_ref(), arn, marker);
+    }
+    let source_version = dobj.delete_marker_version_id.or(dobj.version_id).unwrap_or_default();
+    Some(target_delete_version_id(source_version, is_version_purge))
+}
+
 /// Shape an exhausted purge intent as a marker-creation delete entry. Replay
 /// reconstructs it with `delete_marker: true`, finds the source marker gone,
 /// and funnels into the stale-marker branch of `replicate_delete_with_outcome`
@@ -258,9 +295,9 @@ mod tests {
 
     use super::{
         DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-        delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error,
-        is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info,
-        should_retry_delete_marker_purge, target_delete_version_id,
+        delete_replication_creates_marker, delete_replication_target_version_id, is_object_lock_denied_delete,
+        is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
+        resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id,
     };
     use crate::storage_api::DeletedObject;
     use crate::{
@@ -595,6 +632,76 @@ mod tests {
         assert_eq!(entry.retry_count, 0);
         assert_eq!(entry.bucket, "bucket-a");
         assert_eq!(entry.object, "doc.txt");
+        assert!(
+            entry.target_delete_marker_version_ids.is_empty(),
+            "no recorded target marker ids means the journal carries none"
+        );
+        assert!(!entry.target_delete_marker_version_ids_corrupt);
+    }
+
+    /// backlog#2290: a purge intent journaled to MRF must carry the marker
+    /// version ids the targets assigned, plus the fail-closed corrupt flag,
+    /// so a replay after restart addresses the same version the live path did.
+    #[test]
+    fn delete_marker_purge_mrf_entry_carries_target_assigned_marker_versions() {
+        let delete_marker_version_id = Uuid::new_v4();
+        let mut state = ReplicationState::default();
+        state
+            .target_delete_marker_version_ids
+            .insert("arn:a".to_string(), "remote-marker-a".to_string());
+        state
+            .target_delete_marker_version_ids
+            .insert("arn:b".to_string(), "remote-marker-b".to_string());
+        let mut dobj = DeletedObjectReplicationInfo {
+            delete_object: DeletedObject {
+                object_name: "doc.txt".to_string(),
+                delete_marker: false,
+                version_id: Some(Uuid::new_v4()),
+                delete_marker_version_id: Some(delete_marker_version_id),
+                replication_state: Some(state),
+                ..Default::default()
+            },
+            bucket: "bucket-a".to_string(),
+            ..Default::default()
+        };
+
+        let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
+        assert_eq!(
+            entry.target_delete_marker_version_ids,
+            HashMap::from([
+                ("arn:a".to_string(), "remote-marker-a".to_string()),
+                ("arn:b".to_string(), "remote-marker-b".to_string()),
+            ]),
+            "every recorded target marker id survives the journal, regardless of the retried ARN subset"
+        );
+        assert!(!entry.target_delete_marker_version_ids_corrupt);
+        assert_eq!(
+            delete_marker_purge_version_id(
+                Some(&ReplicationState {
+                    target_delete_marker_version_ids: entry.target_delete_marker_version_ids,
+                    ..Default::default()
+                }),
+                "arn:a",
+                delete_marker_version_id
+            ),
+            Some(Some("remote-marker-a".to_string()))
+        );
+
+        // The live path refuses to purge on inconsistent metadata and reports the target
+        // as failed; the journaled intent must keep refusing after a restart.
+        dobj.delete_object
+            .replication_state
+            .as_mut()
+            .expect("state was set above")
+            .target_delete_marker_version_ids_corrupt = true;
+        let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
+        assert!(entry.target_delete_marker_version_ids_corrupt);
+
+        // A delete without replication state journals an empty map.
+        dobj.delete_object.replication_state = None;
+        let entry = dobj.to_mrf_entry();
+        assert!(entry.target_delete_marker_version_ids.is_empty());
+        assert!(!entry.target_delete_marker_version_ids_corrupt);
     }
 
     #[test]
@@ -656,4 +763,57 @@ mod tests {
         assert!(!is_object_lock_denied_delete(Some("InternalError"), Some("retention lookup failed")));
         assert!(!is_object_lock_denied_delete(None, Some("legal hold")));
     }
+
+    fn purge_of_marker(marker: Uuid, state: Option<ReplicationState>) -> DeletedObject {
+        DeletedObject {
+            object_name: "obj".to_string(),
+            delete_marker: false,
+            delete_marker_version_id: Some(marker),
+            version_id: None,
+            replication_state: state,
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn delete_replication_target_version_id_addresses_recorded_marker_for_purges() {
+        let arn = "arn:minio:replication::generic:photos";
+        let marker = Uuid::new_v4();
+        let mut state = ReplicationState::default();
+        state
+            .target_delete_marker_version_ids
+            .insert(arn.to_string(), "remote-marker".to_string());
+
+        // purge of a replicated marker: the target's own version
+        assert_eq!(
+            delete_replication_target_version_id(&purge_of_marker(marker, Some(state.clone())), arn),
+            Some(Some("remote-marker".to_string()))
+        );
+        // nothing recorded for this arn: the source-derived id (id-mirroring peers)
+        assert_eq!(
+            delete_replication_target_version_id(&purge_of_marker(marker, None), arn),
+            Some(Some(marker.to_string()))
+        );
+        // corrupt record: refuse instead of guessing
+        state.target_delete_marker_version_ids_corrupt = true;
+        assert_eq!(delete_replication_target_version_id(&purge_of_marker(marker, Some(state)), arn), None);
+
+        // marker creation keeps the source id (the target mints its own on a
+        // versionless DELETE; the id only travels in the source header)
+        let creation = DeletedObject {
+            object_name: "obj".to_string(),
+            delete_marker: true,
+            delete_marker_version_id: Some(marker),
+            ..Default::default()
+        };
+        assert_eq!(delete_replication_target_version_id(&creation, arn), Some(Some(marker.to_string())));
+        // plain version purge: the source version id
+        let version = Uuid::new_v4();
+        let purge = DeletedObject {
+            object_name: "obj".to_string(),
+            version_id: Some(version),
+            ..Default::default()
+        };
+        assert_eq!(delete_replication_target_version_id(&purge, arn), Some(Some(version.to_string())));
+    }
 }

crates/replication/src/filemeta.rs

@@ -641,6 +641,26 @@ pub struct MrfReplicateEntry {
     #[serde(rename = "deleteMarkerMtime", skip_serializing_if = "Option::is_none", default)]
     pub delete_marker_mtime: Option<i64>,
 
+    // For delete-marker purge intents: the exact version id each target assigned to the
+    // replicated marker, keyed by target ARN. A generic S3 target mints its own version ids
+    // and answers a DELETE of an unknown id with 204, so a replay that fell back to the source
+    // marker id would be acknowledged while the real marker stayed behind (backlog#2290).
+    // Old files lack this key; default=empty means "unknown" and replay keeps the source-id
+    // fallback it always had.
+    #[serde(rename = "targetDeleteMarkerVersionIDs", skip_serializing_if = "HashMap::is_empty", default)]
+    pub target_delete_marker_version_ids: HashMap<String, String>,
+
+    // Companion to the map above: the source metadata disagreed about the recorded ids when
+    // the intent was journaled, so the live path refused to guess and reported the target as
+    // failed. Replay must keep refusing instead of falling back to the source id. Old files
+    // lack this key; default=false.
+    #[serde(
+        rename = "targetDeleteMarkerVersionIDsCorrupt",
+        skip_serializing_if = "std::ops::Not::not",
+        default
+    )]
+    pub target_delete_marker_version_ids_corrupt: bool,
+
     #[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)]
     pub target_arns: Vec<String>,
 

crates/replication/src/lib.rs

@@ -41,9 +41,9 @@ pub use config::{
 };
 pub use delete::{
     DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-    delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error,
-    is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info,
-    should_retry_delete_marker_purge, target_delete_version_id,
+    delete_replication_creates_marker, delete_replication_target_version_id, is_object_lock_denied_delete,
+    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
+    resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id,
 };
 pub use filemeta::{
     NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,

crates/replication/src/mrf.rs

@@ -31,8 +31,13 @@ const CAPABILITY_OPERATION_KIND: u64 = 1 << 0;
 const CAPABILITY_TARGET_ARNS: u64 = 1 << 1;
 const CAPABILITY_FORCE_DELETE: u64 = 1 << 2;
 const CAPABILITY_DELETE_MARKER_MTIME: u64 = 1 << 3;
-const MRF_KNOWN_CAPABILITIES: u64 =
-    CAPABILITY_OPERATION_KIND | CAPABILITY_TARGET_ARNS | CAPABILITY_FORCE_DELETE | CAPABILITY_DELETE_MARKER_MTIME;
+// Per-ARN target-assigned delete-marker version ids on purge intents (backlog#2290).
+const CAPABILITY_TARGET_DELETE_MARKER_VERSION_IDS: u64 = 1 << 4;
+const MRF_KNOWN_CAPABILITIES: u64 = CAPABILITY_OPERATION_KIND
+    | CAPABILITY_TARGET_ARNS
+    | CAPABILITY_FORCE_DELETE
+    | CAPABILITY_DELETE_MARKER_MTIME
+    | CAPABILITY_TARGET_DELETE_MARKER_VERSION_IDS;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum MrfCapability {
@@ -40,6 +45,7 @@ pub enum MrfCapability {
     TargetArns,
     ForceDelete,
     DeleteMarkerMtime,
+    TargetDeleteMarkerVersionIds,
 }
 
 impl MrfCapability {
@@ -49,6 +55,7 @@ impl MrfCapability {
             Self::TargetArns => CAPABILITY_TARGET_ARNS,
             Self::ForceDelete => CAPABILITY_FORCE_DELETE,
             Self::DeleteMarkerMtime => CAPABILITY_DELETE_MARKER_MTIME,
+            Self::TargetDeleteMarkerVersionIds => CAPABILITY_TARGET_DELETE_MARKER_VERSION_IDS,
         }
     }
 }
@@ -601,9 +608,17 @@ pub fn decode_mrf_file(data: &[u8]) -> Result<Vec<MrfReplicateEntry>> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::collections::HashMap;
     use uuid::Uuid;
 
+    // Capability word 31 = OperationKind | TargetArns | ForceDelete | DeleteMarkerMtime |
+    // TargetDeleteMarkerVersionIds (backlog#2290).
     const ENVELOPE_FIXTURE: &[u8] = &[
+        b'M', b'R', b'F', b'E', 1, 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 2, 3,
+    ];
+
+    // The envelope a binary from before backlog#2290 writes: same header, capability word 15.
+    const PRE_TARGET_MARKER_IDS_ENVELOPE_FIXTURE: &[u8] = &[
         b'M', b'R', b'F', b'E', 1, 0, 1, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 2, 3,
     ];
 
@@ -626,6 +641,8 @@ mod tests {
                 delete_marker_version_id: None,
                 delete_marker: false,
                 delete_marker_mtime: None,
+                target_delete_marker_version_ids: HashMap::new(),
+                target_delete_marker_version_ids_corrupt: false,
                 target_arns: vec!["arn:target-a".to_string()],
             },
             MrfReplicateEntry {
@@ -642,6 +659,8 @@ mod tests {
                 delete_marker_version_id: None,
                 delete_marker: false,
                 delete_marker_mtime: None,
+                target_delete_marker_version_ids: HashMap::new(),
+                target_delete_marker_version_ids_corrupt: false,
                 target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
             },
             MrfReplicateEntry {
@@ -658,6 +677,11 @@ mod tests {
                 delete_marker_version_id: Some(del_vid),
                 delete_marker: true,
                 delete_marker_mtime: Some(1_705_312_200_123_456_789),
+                target_delete_marker_version_ids: HashMap::from([
+                    ("arn:target-a".to_string(), "remote-marker-a".to_string()),
+                    ("arn:target-b".to_string(), "remote-marker-b".to_string()),
+                ]),
+                target_delete_marker_version_ids_corrupt: false,
                 target_arns: vec!["arn:target-a".to_string()],
             },
         ];
@@ -685,6 +709,54 @@ mod tests {
             Some(1_705_312_200_123_456_789),
             "delete-marker mtime must survive the MRF disk round-trip"
         );
+        assert!(decoded[0].target_delete_marker_version_ids.is_empty());
+        assert!(decoded[1].target_delete_marker_version_ids.is_empty());
+        assert_eq!(
+            decoded[2].target_delete_marker_version_ids,
+            HashMap::from([
+                ("arn:target-a".to_string(), "remote-marker-a".to_string()),
+                ("arn:target-b".to_string(), "remote-marker-b".to_string()),
+            ]),
+            "target-assigned marker version ids must survive the MRF disk round-trip (backlog#2290)"
+        );
+        assert!(!decoded[2].target_delete_marker_version_ids_corrupt);
+    }
+
+    /// backlog#2290: the corrupt flag rides the same journal round trip, and an
+    /// entry that carries neither field encodes exactly as it did before the
+    /// field existed (both keys are skipped when empty/false).
+    #[test]
+    fn mrf_file_round_trips_target_marker_ids_corrupt_flag_and_skips_empty_keys() {
+        let corrupt = MrfReplicateEntry {
+            bucket: "bucket-a".to_string(),
+            object: "delete-a".to_string(),
+            op: MrfOpKind::Delete,
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            target_delete_marker_version_ids_corrupt: true,
+            target_arns: vec!["arn:target-a".to_string()],
+            ..Default::default()
+        };
+        let decoded = decode_mrf_file(&encode_mrf_file(std::slice::from_ref(&corrupt)).expect("mrf file should encode"))
+            .expect("mrf file should decode");
+        assert_eq!(decoded, vec![corrupt]);
+        assert!(decoded[0].target_delete_marker_version_ids_corrupt);
+
+        let plain = MrfReplicateEntry {
+            bucket: "bucket-a".to_string(),
+            object: "delete-a".to_string(),
+            op: MrfOpKind::Delete,
+            delete_marker: true,
+            target_arns: vec!["arn:target-a".to_string()],
+            ..Default::default()
+        };
+        let encoded = encode_mrf_file(std::slice::from_ref(&plain)).expect("mrf file should encode");
+        let payload = String::from_utf8_lossy(&encoded);
+        assert!(
+            !payload.contains("targetDeleteMarkerVersionIDs"),
+            "an entry without recorded ids must not grow the new keys: {payload}"
+        );
+        assert_eq!(decode_mrf_file(&encoded).expect("mrf file should decode"), vec![plain]);
     }
 
     #[test]
@@ -719,6 +791,99 @@ mod tests {
         // Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the
         // pre-#867 fallback to the current time.
         assert_eq!(decoded[0].delete_marker_mtime, None);
+        // Old files also lack the target marker id keys; they must default to an empty map
+        // and a clear corrupt flag so replay keeps the pre-#2290 source-id fallback.
+        assert!(decoded[0].target_delete_marker_version_ids.is_empty());
+        assert!(!decoded[0].target_delete_marker_version_ids_corrupt);
+    }
+
+    /// backlog#2290: a delete-marker entry written by a binary that predates the
+    /// `targetDeleteMarkerVersionIDs` key decodes with an empty map and a clear
+    /// corrupt flag — the exact shape replay handled before the field existed.
+    #[test]
+    fn mrf_pre_target_marker_ids_delete_entry_decodes_with_empty_map() {
+        let marker_version_id = Uuid::new_v4();
+        let mut payload = Vec::new();
+        rmp::encode::write_array_len(&mut payload, 1).expect("array len should encode");
+        rmp::encode::write_map_len(&mut payload, 9).expect("map len should encode");
+        rmp::encode::write_str(&mut payload, "bucket").expect("bucket key should encode");
+        rmp::encode::write_str(&mut payload, "old-bucket").expect("bucket value should encode");
+        rmp::encode::write_str(&mut payload, "object").expect("object key should encode");
+        rmp::encode::write_str(&mut payload, "old-key").expect("object value should encode");
+        rmp::encode::write_str(&mut payload, "retryCount").expect("retry key should encode");
+        rmp::encode::write_i32(&mut payload, 0).expect("retry value should encode");
+        rmp::encode::write_str(&mut payload, "size").expect("size key should encode");
+        rmp::encode::write_i64(&mut payload, 0).expect("size value should encode");
+        rmp::encode::write_str(&mut payload, "op").expect("op key should encode");
+        rmp::encode::write_str(&mut payload, "delete").expect("op value should encode");
+        rmp::encode::write_str(&mut payload, "forceDelete").expect("forceDelete key should encode");
+        rmp::encode::write_bool(&mut payload, false).expect("forceDelete value should encode");
+        rmp::encode::write_str(&mut payload, "deleteMarkerVersionID").expect("marker id key should encode");
+        // Uuid serializes as a 16-byte bin in the MessagePack journal.
+        rmp::encode::write_bin(&mut payload, marker_version_id.as_bytes()).expect("marker id value should encode");
+        rmp::encode::write_str(&mut payload, "deleteMarker").expect("deleteMarker key should encode");
+        rmp::encode::write_bool(&mut payload, true).expect("deleteMarker value should encode");
+        rmp::encode::write_str(&mut payload, "targetARNs").expect("targetARNs key should encode");
+        rmp::encode::write_array_len(&mut payload, 1).expect("targetARNs len should encode");
+        rmp::encode::write_str(&mut payload, "arn:target-a").expect("targetARNs value should encode");
+
+        let mut data = Vec::with_capacity(4 + payload.len());
+        data.extend_from_slice(&MRF_META_FORMAT.to_le_bytes());
+        data.extend_from_slice(&MRF_META_VERSION.to_le_bytes());
+        data.extend_from_slice(&payload);
+
+        let decoded = decode_mrf_file(&data).expect("pre-#2290 delete-marker entry should decode");
+
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded[0].op, MrfOpKind::Delete);
+        assert!(decoded[0].delete_marker);
+        assert_eq!(decoded[0].delete_marker_version_id, Some(marker_version_id));
+        assert_eq!(decoded[0].target_arns, vec!["arn:target-a".to_string()]);
+        assert!(decoded[0].target_delete_marker_version_ids.is_empty());
+        assert!(!decoded[0].target_delete_marker_version_ids_corrupt);
+    }
+
+    /// backlog#2290: the new field is fenced by its own capability bit exactly
+    /// like the earlier optional fields — a reader without the bit refuses an
+    /// envelope that advertises it, while the current reader still accepts the
+    /// pre-#2290 envelope.
+    #[test]
+    fn envelope_target_marker_ids_capability_is_fenced_and_backward_compatible() {
+        assert!(MrfCapabilities::current().contains(MrfCapability::TargetDeleteMarkerVersionIds));
+        assert_eq!(MrfCapabilities::with(MrfCapability::TargetDeleteMarkerVersionIds).bits(), 1 << 4);
+
+        // Old envelope, current reader: accepted, and the negotiated set lacks the new bit.
+        let legacy = MrfEnvelope::decode(PRE_TARGET_MARKER_IDS_ENVELOPE_FIXTURE, MrfProtocolCapabilities::current())
+            .expect("pre-#2290 envelope should decode");
+        assert_eq!(legacy.protocol().capabilities().bits(), 15);
+        assert!(
+            !legacy
+                .protocol()
+                .capabilities()
+                .contains(MrfCapability::TargetDeleteMarkerVersionIds)
+        );
+        assert_eq!(legacy.payload(), &[1, 2, 3]);
+
+        // Current envelope, reader that only knows the pre-#2290 bits: refused.
+        let pre_2290_reader = MrfProtocolCapabilities::new(1, 1, MrfCapabilities::from_bits(15).expect("known bits"));
+        assert_eq!(
+            MrfEnvelope::decode(ENVELOPE_FIXTURE, pre_2290_reader),
+            Err(MrfEnvelopeError::MissingCapabilities {
+                required: 31,
+                available: 15,
+            })
+        );
+
+        // Negotiation with such a peer drops the bit instead of failing.
+        let negotiated = MrfProtocolCapabilities::current()
+            .negotiate(pre_2290_reader)
+            .expect("negotiation with a pre-#2290 peer should succeed");
+        assert!(
+            !negotiated
+                .capabilities()
+                .contains(MrfCapability::TargetDeleteMarkerVersionIds)
+        );
+        assert!(negotiated.capabilities().contains(MrfCapability::DeleteMarkerMtime));
     }
 
     #[test]

crates/scanner/src/data_usage_define/tests.rs

@@ -29,6 +29,8 @@ use tokio::sync::Mutex;
 
 const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([3; 32]);
 
+mod cache_cost;
+
 #[test]
 fn scoped_scan_coverage_metadata_preserves_map_compatibility() {
     #[derive(serde::Deserialize)]

crates/scanner/src/data_usage_define/tests/cache_cost.rs

@@ -0,0 +1,346 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use std::hint::black_box;
+use std::sync::atomic::AtomicU64;
+use std::time::Instant as WallInstant;
+
+const MAX_WIRE_BYTES: u64 = 32 * 1024 * 1024;
+const CACHE_NAME: &str = "bucket/cache-cost.bin";
+
+/// Two bounded memory slots model revision preconditions and count the bytes
+/// consumed by the real save entry point, not disk writes or fsync latency.
+#[derive(Debug, Default)]
+struct CountingStore {
+    slots: Mutex<[(u64, Vec<u8>); 2]>,
+    puts: AtomicU64,
+    bytes: AtomicU64,
+    ingest_ns: AtomicU64,
+}
+
+impl CountingStore {
+    fn slot(object: &str) -> usize {
+        let main = path_join_buf(&[BUCKET_META_PREFIX, CACHE_NAME]);
+        if object == main {
+            0
+        } else {
+            assert_eq!(object, format!("{main}.bkp"), "only two fixture cache paths are permitted");
+            1
+        }
+    }
+
+    fn reset_counts(&self) {
+        self.puts.store(0, Ordering::Relaxed);
+        self.bytes.store(0, Ordering::Relaxed);
+        self.ingest_ns.store(0, Ordering::Relaxed);
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectIO for CountingStore {
+    type Error = Error;
+    type RangeSpec = HTTPRangeSpec;
+    type HeaderMap = HeaderMap;
+    type ObjectOptions = ObjectOptions;
+    type ObjectInfo = ObjectInfo;
+    type GetObjectReader = ScannerGetObjectReader;
+    type PutObjectReader = ScannerPutObjReader;
+
+    async fn get_object_reader(
+        &self,
+        bucket: &str,
+        object: &str,
+        _range: Option<Self::RangeSpec>,
+        _headers: Self::HeaderMap,
+        _options: &Self::ObjectOptions,
+    ) -> StorageResult<Self::GetObjectReader> {
+        // The real loader may probe the legacy metadata bucket on a miss.
+        if bucket != RUSTFS_META_BUCKET {
+            return Err(Error::FileNotFound);
+        }
+        let slots = self.slots.lock().await;
+        let (revision, bytes) = &slots[Self::slot(object)];
+        if *revision == 0 {
+            return Err(Error::FileNotFound);
+        }
+        Ok(CacheReadStore::reader(CacheReadBody::Bytes(bytes.clone()), &revision.to_string()))
+    }
+
+    async fn put_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut Self::PutObjectReader,
+        options: &Self::ObjectOptions,
+    ) -> StorageResult<Self::ObjectInfo> {
+        assert_eq!(bucket, RUSTFS_META_BUCKET);
+        let started = WallInstant::now();
+        let mut bytes = Vec::new();
+        (&mut data.stream).take(MAX_WIRE_BYTES + 1).read_to_end(&mut bytes).await?;
+        assert!(u64::try_from(bytes.len()).expect("wire length") <= MAX_WIRE_BYTES);
+        let mut slots = self.slots.lock().await;
+        let (revision, stored) = &mut slots[Self::slot(object)];
+        let preconditions = options.http_preconditions.as_ref().expect("profile saves must use CAS");
+        let expected = revision.to_string();
+        if (*revision == 0 && preconditions.if_none_match_value() != Some("*"))
+            || (*revision != 0 && preconditions.if_match_value() != Some(expected.as_str()))
+        {
+            return Err(Error::PreconditionFailed);
+        }
+        self.bytes
+            .fetch_add(u64::try_from(bytes.len()).expect("save length"), Ordering::Relaxed);
+        self.puts.fetch_add(1, Ordering::Relaxed);
+        *stored = bytes;
+        *revision += 1;
+        self.ingest_ns.fetch_add(elapsed_ns(started), Ordering::Relaxed);
+        Ok(ObjectInfo {
+            etag: Some(revision.to_string()),
+            ..Default::default()
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl crate::ScannerConfigObjectDelete for CountingStore {
+    async fn delete_config_object(
+        &self,
+        _bucket: &str,
+        _object: &str,
+        _options: crate::ScannerObjectOptions,
+    ) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
+        Err(Error::NotImplemented)
+    }
+
+    async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
+        Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
+    }
+}
+
+fn elapsed_ns(started: WallInstant) -> u64 {
+    u64::try_from(started.elapsed().as_nanos()).expect("bounded profile duration")
+}
+
+fn fixture(objects: usize) -> DataUsageCache {
+    assert!((1..=16384).contains(&objects));
+    let mut cache = DataUsageCache::default();
+    cache.info.name = "bucket".to_string();
+    cache.info.snapshot_complete = true;
+    cache.replace("bucket", "", DataUsageEntry::default());
+    for index in 0..objects {
+        cache.replace(
+            &format!("bucket/object-{index:05}"),
+            "bucket",
+            DataUsageEntry {
+                objects: 1,
+                versions: 2,
+                size: 4096,
+                ..Default::default()
+            },
+        );
+    }
+    cache
+}
+
+fn canonical_cache_value(mut value: Value) -> Value {
+    for entry in value["cache"].as_object_mut().expect("cache entry map").values_mut() {
+        let children = entry["children"].as_array_mut().expect("entry children set");
+        // Sort only the set representation. Do not deduplicate or reorder
+        // histograms and other arrays whose element positions carry meaning.
+        children.sort_unstable_by(|left, right| {
+            left.as_str()
+                .expect("child key string")
+                .cmp(right.as_str().expect("child key string"))
+        });
+    }
+    value
+}
+
+fn same_cache(actual: &DataUsageCache, expected: &DataUsageCache) {
+    assert_eq!(
+        canonical_cache_value(serde_json::to_value(actual).expect("actual cache structure")),
+        canonical_cache_value(serde_json::to_value(expected).expect("expected cache structure")),
+        "every cache field and map entry must be retained"
+    );
+}
+
+#[test]
+fn cache_cost_comparison_preserves_set_and_ordered_field_semantics() {
+    let forward = fixture(2);
+    let mut reverse = forward.clone();
+    let children = &mut reverse.cache.get_mut(&hash_path("bucket").key()).expect("root").children;
+    children.clear();
+    for index in (0..2).rev() {
+        children.insert(hash_path(&format!("bucket/object-{index:05}")).key());
+    }
+    same_cache(&forward, &reverse);
+
+    let original = serde_json::json!({"cache": {"root": {"children": ["a", "b"], "size": 1, "histogram": [1, 2]}}});
+    let mut reordered = original.clone();
+    reordered["cache"]["root"]["children"] = serde_json::json!(["b", "a"]);
+    assert_eq!(canonical_cache_value(original.clone()), canonical_cache_value(reordered));
+    for children in [serde_json::json!(["a"]), serde_json::json!(["a", "b", "b"])] {
+        let mut changed = original.clone();
+        changed["cache"]["root"]["children"] = children;
+        assert_ne!(canonical_cache_value(original.clone()), canonical_cache_value(changed));
+    }
+    for (field, replacement) in [("size", serde_json::json!(2)), ("histogram", serde_json::json!([2, 1]))] {
+        let mut changed = original.clone();
+        changed["cache"]["root"][field] = replacement;
+        assert_ne!(canonical_cache_value(original.clone()), canonical_cache_value(changed));
+    }
+}
+
+fn quantiles(mut samples: Vec<u64>) -> Value {
+    assert!(!samples.is_empty() && samples.len() <= 5);
+    samples.sort_unstable();
+    serde_json::json!({"p50_ns": samples[samples.len() / 2], "max_ns": samples[samples.len() - 1]})
+}
+
+async fn profile_case(objects: usize, scenario: &str, samples: usize) {
+    let baseline = fixture(objects);
+    let mut cache = baseline.clone();
+    let dirty = match scenario {
+        "unchanged" => 0,
+        "small_dirty" => (objects / 100).max(1),
+        "all_dirty" => objects,
+        _ => panic!("unknown fixed scenario"),
+    };
+    let mut changed_entry_wire_bytes = 0;
+    for index in 0..dirty {
+        let entry = cache
+            .cache
+            .get_mut(&hash_path(&format!("bucket/object-{index:05}")).key())
+            .expect("dirty leaf");
+        entry.size += 1;
+        entry.versions += 1;
+        changed_entry_wire_bytes += rmp_serde::to_vec(entry).expect("changed entry wire bytes").len();
+    }
+    if scenario == "small_dirty" {
+        cache.info.snapshot_complete = false;
+        cache.info.scan_resume_after = Some("bucket/object-00000".to_string());
+    }
+    let expected_wire = cache.marshal_msg().expect("fixture encoding");
+    assert!(u64::try_from(expected_wire.len()).expect("fixture bytes") <= MAX_WIRE_BYTES);
+    let store = Arc::new(CountingStore::default());
+    let mut loaded = DataUsageCache::default();
+    let initial = loaded
+        .load_with_revisions(store.clone(), CACHE_NAME)
+        .await
+        .expect("initial revisions");
+    baseline
+        .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0)
+        .await
+        .expect("baseline save");
+
+    let mut clone_ns = Vec::new();
+    let mut copy_ns = Vec::new();
+    let mut flatten_ns = Vec::new();
+    let mut encode_ns = Vec::new();
+    let mut save_ns = Vec::new();
+    let mut ingest_ns = Vec::new();
+    for _ in 0..samples {
+        let started = WallInstant::now();
+        let cloned = black_box(cache.clone());
+        clone_ns.push(elapsed_ns(started));
+        same_cache(&cloned, &cache);
+        drop(cloned);
+
+        let mut copied = DataUsageCache {
+            info: cache.info.clone(),
+            ..Default::default()
+        };
+        let started = WallInstant::now();
+        copied.copy_with_children(black_box(&cache), &hash_path("bucket"), &None);
+        copy_ns.push(elapsed_ns(started));
+        same_cache(&copied, &cache);
+        drop(copied);
+
+        let started = WallInstant::now();
+        let aggregate = black_box(cache.checked_flatten("bucket").expect("valid fixture tree"));
+        flatten_ns.push(elapsed_ns(started));
+        assert_eq!(
+            (aggregate.objects, aggregate.versions, aggregate.size),
+            (objects, objects * 2 + dirty, objects * 4096 + dirty)
+        );
+
+        let started = WallInstant::now();
+        let encoded = black_box(cache.marshal_msg().expect("measured encoding"));
+        encode_ns.push(elapsed_ns(started));
+        assert_eq!(encoded, expected_wire);
+        same_cache(&DataUsageCache::unmarshal(&encoded).expect("measured wire reload"), &cache);
+
+        let revisions = loaded
+            .load_with_revisions(store.clone(), CACHE_NAME)
+            .await
+            .expect("current revisions");
+        store.reset_counts();
+        let started = WallInstant::now();
+        cache
+            .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
+            .await
+            .expect("measured save");
+        save_ns.push(elapsed_ns(started));
+        ingest_ns.push(store.ingest_ns.load(Ordering::Relaxed));
+        assert_eq!(store.puts.load(Ordering::Relaxed), 2, "main and backup writes must both occur");
+        assert_eq!(
+            store.bytes.load(Ordering::Relaxed),
+            u64::try_from(expected_wire.len() * 2).expect("two saved bodies")
+        );
+        loaded
+            .load_with_revisions(store.clone(), CACHE_NAME)
+            .await
+            .expect("saved cache reload");
+        same_cache(&loaded, &cache);
+    }
+
+    let before_rejected = store.slots.lock().await[0].1.clone();
+    let mut conflicting = cache.clone();
+    conflicting.info.next_cycle += 1;
+    assert!(matches!(
+        conflicting
+            .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0)
+            .await,
+        Err(Error::PreconditionFailed)
+    ));
+    assert_eq!(
+        store.slots.lock().await[0].1,
+        before_rejected,
+        "stale CAS must not replace the retained checkpoint"
+    );
+    println!(
+        "CACHE_COST {}",
+        serde_json::json!({
+            "schema": 1, "scenario": scenario, "objects": objects, "dirty_objects": dirty, "samples": samples,
+            "build": {
+                "debug_assertions": cfg!(debug_assertions),
+                "test_opt_level_override": option_env!("CARGO_PROFILE_TEST_OPT_LEVEL"),
+                "dev_opt_level_override": option_env!("CARGO_PROFILE_DEV_OPT_LEVEL"),
+                "rustflags_visible_to_rustc": option_env!("RUSTFLAGS"),
+                "encoded_rustflags_visible_to_rustc": option_env!("CARGO_ENCODED_RUSTFLAGS"),
+                "source_revision": option_env!("RUSTFS_CACHE_COST_SOURCE"),
+                "source_tree": option_env!("RUSTFS_CACHE_COST_TREE"),
+            },
+            "retained_cache_entries": cache.cache.len(), "cache_wire_bytes": expected_wire.len(),
+            "changed_entry_wire_bytes": changed_entry_wire_bytes, "save_body_bytes_per_sample": expected_wire.len() * 2,
+            "snapshot_complete": cache.info.snapshot_complete,
+            "clone": quantiles(clone_ns), "copy_with_children": quantiles(copy_ns), "checked_flatten": quantiles(flatten_ns),
+            "encode": quantiles(encode_ns), "save_inclusive": quantiles(save_ns), "memory_backend_ingest": quantiles(ingest_ns),
+        })
+    );
+}
+
+#[tokio::test]
+async fn cache_cost_profile_preserves_checkpoint_and_counts() {
+    let profile = match std::env::var("RUSTFS_CACHE_COST_PROFILE") {
+        Err(std::env::VarError::NotPresent) => false,
+        Ok(value) if value == "1" => true,
+        _ => panic!("RUSTFS_CACHE_COST_PROFILE must be absent or 1"),
+    };
+    let (sizes, samples): (&[usize], usize) = if profile { (&[1024, 4096, 16384], 5) } else { (&[64], 1) };
+    for &objects in sizes {
+        for scenario in ["unchanged", "small_dirty", "all_dirty"] {
+            profile_case(objects, scenario, samples).await;
+        }
+    }
+}

crates/scanner/src/scanner_io/tests.rs

@@ -39,6 +39,8 @@ use temp_env::with_var;
 use time::OffsetDateTime;
 use uuid::Uuid;
 
+mod scoped_entry_fallback;
+
 #[derive(Clone)]
 struct FixedWorkloadProvider {
     snapshot: WorkloadAdmissionRegistrySnapshot,

crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs

@@ -0,0 +1,289 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
+use crate::storage_api::owner::EcstoreDiskAPI;
+
+type DriveIdentities = HashMap<String, (Uuid, DataUsageCacheSource)>;
+type WalkCounts = HashMap<(String, String, String), u64>;
+
+async fn drive_identities(store: &ECStore) -> DriveIdentities {
+    let mut identities = HashMap::new();
+    let mut ids = HashSet::new();
+    for set in store.all_set_disks() {
+        let source = DataUsageCacheSource::new(set.pool_index, set.set_index);
+        for disk in scanner_set_disk_inventory(set.as_ref()).await {
+            let id = EcstoreDiskAPI::get_disk_id(disk.as_ref())
+                .await
+                .expect("fixture disk identity should be readable")
+                .expect("fixture disk must have a durable identity");
+            assert!(!id.is_nil());
+            assert!(ids.insert(id), "fixture disk identities must be unique");
+            let path = crate::ScannerDiskExt::path(disk.as_ref()).to_string_lossy().into_owned();
+            assert!(identities.insert(path, (id, source)).is_none());
+        }
+    }
+    assert_eq!(identities.len(), 8);
+    identities
+}
+
+fn walk_counts(drives: &DriveIdentities) -> WalkCounts {
+    rustfs_scanner_metrics::metrics::global_metrics()
+        .scanner_runtime_details_report()
+        .bucket_drive_results
+        .into_iter()
+        .filter(|result| drives.contains_key(&result.drive))
+        .map(|result| ((result.bucket, result.drive, result.result), result.count))
+        .collect()
+}
+
+async fn put_and_settle(store: &ECStore, bucket: &str, object: &str) {
+    let set = &store.pools[0].disk_set[0];
+    let mut reader = ScannerPutObjReader::from_vec(b"object".to_vec());
+    set.put_object(bucket, object, &mut reader, &ScannerObjectOptions::default())
+        .await
+        .expect("fixture object should persist");
+    let lock = set.new_ns_lock(bucket, object).await.expect("fixture namespace lock");
+    let _settled = lock
+        .get_write_lock(Duration::from_secs(30))
+        .await
+        .expect("quorum-ACK rename tail must settle before taking the activity baseline");
+}
+
+async fn create_bucket(store: &ECStore, bucket: &str) {
+    store
+        .make_bucket(bucket, &MakeBucketOptions::default())
+        .await
+        .expect("fixture bucket should be created");
+    put_and_settle(store, bucket, "initial").await;
+}
+
+async fn persist_baseline(store: &Arc<ECStore>, baseline: &DataUsageInfo) {
+    let mut baseline = baseline.clone();
+    baseline.usage_snapshot_converged = Some(true);
+    crate::save_config(
+        store.clone(),
+        DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        serde_json::to_vec(&baseline).expect("baseline should encode"),
+    )
+    .await
+    .expect("fixture baseline should persist");
+}
+
+// Every invocation uses the production default scope. The expected walker set
+// comes from storage's per-source inventory, not the resolver's selected names.
+async fn run_entry(store: &Arc<ECStore>, cycle: u64, selected: Option<&str>, expect_walks: bool) -> DataUsageInfo {
+    let drives = drive_identities(store).await;
+    let inventory = store
+        .list_bucket_for_scanner(&BucketOptions::default())
+        .await
+        .expect("fixture inventory should be complete");
+    assert!(inventory.topology_complete);
+    let expected_walks = if expect_walks {
+        inventory
+            .set_buckets
+            .into_iter()
+            .flat_map(|set| {
+                let source = DataUsageCacheSource::new(set.pool_index, set.set_index);
+                set.buckets.into_iter().map(move |bucket| ((source, bucket.name), 1_u64))
+            })
+            .collect::<HashMap<_, _>>()
+    } else {
+        HashMap::new()
+    };
+    let root_before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("root baseline should be readable");
+    let dirty_before = dirty_usage_buckets_for_tests();
+    let generation_before = dirty_usage_generation();
+    let before = walk_counts(&drives);
+    let ctx = CancellationToken::new();
+    let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
+    let (updates, mut receiver) = mpsc::channel(1);
+    let (observer, observed) = tokio::sync::oneshot::channel();
+    let result = tokio::time::timeout(
+        Duration::from_secs(30),
+        nsscanner_with_storage_status_scoped(
+            store.as_ref(),
+            ScannerCycleRequest {
+                ctx,
+                budget,
+                updates,
+                want_cycle: cycle,
+                leader_epoch: 11,
+                scan_mode: HealScanMode::Normal,
+                scan_scope: ScannerBucketScanScope::default(),
+                persisted_usage_baseline: root_before.0.clone().map(Bytes::from),
+                requires_full_scan: false,
+                resolved_scope_observer: Some(observer),
+            },
+        ),
+    )
+    .await
+    .expect("entry cycle should finish within the fixture deadline")
+    .expect("entry cycle should succeed");
+    assert_eq!(result.status, ScannerCycleStatus::Complete);
+    let scope = observed.await.expect("production resolver should report its decision");
+    assert_eq!(
+        scope.selected_buckets.as_deref(),
+        selected.map(|name| HashSet::from([name.to_string()])).as_ref()
+    );
+    let usage = receiver.recv().await.expect("one candidate should be delivered");
+    assert!(receiver.recv().await.is_none(), "there must be exactly one terminal candidate");
+    assert!(usage.usage_snapshot_complete);
+    assert!(!usage.usage_snapshot_partial);
+    assert_eq!(usage.scanner_cycle, Some(cycle));
+    assert_eq!(
+        drive_identities(store).await,
+        drives,
+        "drive identities must not change during the oracle"
+    );
+
+    let after = walk_counts(&drives);
+    let mut actual = HashMap::new();
+    for key in before.keys() {
+        assert!(after.contains_key(key), "metrics eviction would invalidate this exact-delta oracle");
+    }
+    for ((bucket, drive, outcome), count) in after {
+        let previous = before
+            .get(&(bucket.clone(), drive.clone(), outcome.clone()))
+            .copied()
+            .unwrap_or(0);
+        let delta = count.checked_sub(previous).expect("fixture counters must not reset");
+        if delta > 0 {
+            assert_eq!(outcome, "success", "no error or partial walker is expected");
+            *actual.entry((drives[&drive].1, bucket)).or_insert(0_u64) += delta;
+        }
+    }
+    assert_eq!(
+        actual, expected_walks,
+        "each listed source/bucket must have exactly the expected real walks"
+    );
+    assert_eq!(
+        read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+            .await
+            .expect("root after scan"),
+        root_before,
+        "producing a candidate must not replace the coordinator-owned root baseline"
+    );
+    assert_eq!(dirty_usage_generation(), generation_before);
+    assert!(
+        dirty_usage_buckets_for_tests() == dirty_before,
+        "candidate delivery must not ACK pending dirty buckets"
+    );
+    usage
+}
+
+#[tokio::test]
+#[serial]
+async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks() {
+    let (_dir, store) = setup_two_pool_scanner_store().await;
+    clear_dirty_usage_buckets_for_tests();
+    let hot = format!("hot-{}", Uuid::new_v4().simple());
+    let cold = format!("cold-{}", Uuid::new_v4().simple());
+    create_bucket(&store, &hot).await;
+    create_bucket(&store, &cold).await;
+    record_dirty_usage_bucket(&hot);
+    let baseline = run_entry(&store, 1, None, true).await;
+    persist_baseline(&store, &baseline).await;
+
+    // A same-intent, same-cycle Current cache is a retry, not proof that a
+    // later cycle may reuse unselected buckets without durable incarnation.
+    run_entry(&store, 1, Some(&hot), false).await;
+    let usage = run_entry(&store, 2, Some(&hot), true).await;
+    assert_eq!(usage.buckets_usage[&hot].objects_count, 1);
+    assert_eq!(usage.buckets_usage[&cold].objects_count, 1);
+    assert_eq!(usage.objects_total_count, 2);
+    clear_dirty_usage_buckets_for_tests();
+}
+
+#[tokio::test]
+#[serial]
+async fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker() {
+    let (_dir, store) = setup_two_pool_scanner_store().await;
+    clear_dirty_usage_buckets_for_tests();
+    let hot = format!("hot-{}", Uuid::new_v4().simple());
+    let cold = format!("cold-{}", Uuid::new_v4().simple());
+    create_bucket(&store, &hot).await;
+    create_bucket(&store, &cold).await;
+    record_dirty_usage_bucket(&hot);
+    // The first real scan is also the missing persisted-baseline case.
+    let baseline = run_entry(&store, 1, None, true).await;
+    for (index, kind) in [
+        "malformed",
+        "unconverged",
+        "missing-set",
+        "wrong-source",
+        "mixed-plan",
+        "wrong-epoch",
+    ]
+    .into_iter()
+    .enumerate()
+    {
+        let mut candidate = baseline.clone();
+        candidate.usage_snapshot_converged = Some(true);
+        match kind {
+            "unconverged" => candidate.usage_snapshot_converged = Some(false),
+            "missing-set" => {
+                candidate.usage_snapshot_set_states.pop();
+            }
+            "wrong-source" => candidate.usage_snapshot_set_states[0].set_index = 99,
+            "mixed-plan" => candidate.usage_snapshot_set_states[1].scan_plan_digest = Some([0xA5; 32]),
+            "wrong-epoch" => candidate.usage_snapshot_set_states[0].scanner_epoch = Some(10),
+            "malformed" => {}
+            _ => unreachable!(),
+        }
+        let bytes = if kind == "malformed" {
+            b"{broken".to_vec()
+        } else {
+            serde_json::to_vec(&candidate).expect("candidate JSON")
+        };
+        crate::save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes)
+            .await
+            .expect("negative baseline should persist");
+        let usage = run_entry(&store, u64::try_from(index).expect("fixture cycle index should fit") + 2, None, true).await;
+        assert_eq!(usage.objects_total_count, 2, "{kind}");
+        assert_eq!(usage.buckets_usage[&cold].objects_count, 1, "{kind}");
+    }
+    clear_dirty_usage_buckets_for_tests();
+}
+
+#[tokio::test]
+#[serial]
+async fn scoped_entry_fallback_covers_overflow_and_new_bucket_inventory() {
+    let (_dir, store) = setup_two_pool_scanner_store().await;
+    clear_dirty_usage_buckets_for_tests();
+    let hot = format!("hot-{}", Uuid::new_v4().simple());
+    create_bucket(&store, &hot).await;
+    record_dirty_usage_bucket(&hot);
+    let baseline = run_entry(&store, 1, None, true).await;
+    persist_baseline(&store, &baseline).await;
+    for index in 0..=crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
+        record_dirty_usage_bucket(&format!("overflow-{index}"));
+    }
+    assert!(dirty_usage_buckets_for_tests().len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES);
+    let usage = run_entry(&store, 2, None, true).await;
+    assert_eq!(usage.objects_total_count, 1);
+
+    clear_dirty_usage_buckets_for_tests();
+    record_dirty_usage_bucket(&hot);
+    let new_bucket = format!("new-{}", Uuid::new_v4().simple());
+    create_bucket(&store, &new_bucket).await;
+    // Even a previously valid baseline cannot cover the changed inventory.
+    let usage = run_entry(&store, 3, None, true).await;
+    assert_eq!(usage.objects_total_count, 2);
+    assert_eq!(usage.buckets_usage[&new_bucket].objects_count, 1);
+    clear_dirty_usage_buckets_for_tests();
+}

docs/architecture/compat-cleanup-register.md

@@ -11,6 +11,7 @@
 
 ## Open Items
 
+- `odm-list-bare-envelope` historical ODM continuation tokens: preserve complete bare v1/v2 envelopes. Framed issuance defaults on for the deployed framed-only generation; upgrades from older bare-only readers must explicitly disable it before starting new nodes and keep it off until reader convergence. Remove the legacy classifier and framing issuance override only after every supported reader accepts framing and outstanding bare listings have drained or clients explicitly restarted them; tokens have no automatic expiry. Exact full-envelope object keys remain intrinsically ambiguous during this compatibility period.
 - `backlog-2263` legacy heal MRF inspection: retained per-record journals remain readable while committed-snapshot ownership and writer activation are staged. Remove legacy import only after all supported direct-upgrade and rollback readers understand committed snapshots and migration tooling confirms that no retained or restorable legacy journal requires it. This does not enable a new writer or change the automatic legacy consumer.
 - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
 - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.

docs/operations/admin-peer-probe-timeout.md

@@ -0,0 +1,27 @@
+# Admin peer probe timeout
+
+RustFS admin server information and storage information aggregate read-only
+state from remote peers. A peer may answer the RPC while its local disk
+diagnostic is still recovering after a restart or outage, so these probes use a
+bounded per-peer round budget.
+
+## Configuration
+
+| Environment variable | Default | Accepted range | Behavior |
+| --- | ---: | ---: | --- |
+| `RUSTFS_ADMIN_PEER_PROBE_TIMEOUT_SECS` | `10` seconds | `1..=60` seconds | Total budget for one peer probe round; `server_info` may reconnect once, while `storage_info` remains a single attempt. |
+
+`0` and invalid values fall back to the default. Values above `60` are clamped
+to `60`. The timeout is read by the node aggregating the admin response; it is
+not a wire or mixed-version protocol setting.
+
+Any retry shares the same per-peer deadline. A fast transport failure can still
+trigger the existing reconnect retry, but a slow first attempt consumes the
+remaining budget and cannot add another full timeout. Configure this value
+with margin below any external health-check deadline (for example, a
+keepalived script timeout); the default preserves the previous two-attempt
+worst-case budget and may need to be lowered for a tighter watchdog.
+
+This setting does not change `RUSTFS_INTERNODE_RPC_TIMEOUT_SECS` or the drive
+health policy. A disk probe timeout can still update drive health according to
+`RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION`.

docs/operations/on-demand-migration.md

@@ -23,13 +23,27 @@ Both builds can read, redact and preserve GCS configuration. A build without `gc
 
 ## List continuation token rollout
 
-`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape.
+The two rollout switches have different defaults. Unset or invalid boolean values use the stated default. Both are node environment variables, not bucket settings:
 
-Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled.
+- `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false` and allows a v1 listing to first issue a v2 token after an empty truncated merged page. Existing v2 tokens keep their budget even on reader-only nodes. Consuming an object/common prefix or reaching a new EOF resets the budget to v1 without changing the chain's framing.
+- `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS` defaults to `true`, preserving the framed output of the #7187 / `e1608fbd9` generation. It allows a bare/new merged listing to first issue a NUL-prefixed JSON envelope inside the existing base64 encoding. Existing framed chains stay framed even with this switch off, including a reset to v1 and local continuation after list-through is disabled. With framing issuance off, new bare v1 output keeps its historical bytes; ordinary local listings remain unchanged. This switch does not enable the v2 budget.
 
-An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.
+When list-through or the global module is disabled, existing merged continuations retain their last emitted key and original bare/framed format, filter already returned objects and common prefixes, and make no source requests. If the local-only response remains truncated, its continuation can resume both sides after re-enabling list-through.
 
-For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear.
+Choose the upgrade path from the binaries currently serving LIST requests, including every load-balancer route. This build reads complete historical bare envelopes and framed v1/v2 envelopes with the same strict version/count validation. There is no single writer format understood by both bare-only and framed-only readers. A bare-v1-only binary rejects bare v2 with `400 InvalidArgument`; a bare-only reader mistakes framed input for a local marker, while a framed-only reader mistakes bare input for one. These framing mismatches can restart a merged scan and lose its budget without returning an error. For example, with local keys `b,d` and source keys `a,c`, a new node issuing bare after `a` followed by an `e1608fbd9` reader can return `a` again. That old reader then emits framing, so the symptom need not be an infinite loop.
+
+- **Upgrading from the framed-only #7187 / `e1608fbd9` generation:** before deploying the first new node, set `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=true` in the new nodes' deployment environment, or leave it unset to use this build's `true` default. Remove any previous explicit `false` override. Existing framed-only nodes ignore this new variable and already emit framing. Keep framing enabled while any of those nodes serves continuations. While list-through remains active, new and existing framed v1/v2 chains then retain their cursors and any active budget in both directions. Only after all serving nodes are dual readers may you choose `false`; existing framed chains still stay framed, while newly started bare chains must stay on dual readers.
+- **Upgrading from older, pre-#7187 bare-only binaries:** explicitly set `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=false` on every new node **before its first start**. Keep it false until all serving readers accept both formats; this preserves bare v1 bytes and active bare cursors during that rollout. Keep `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS=false` until every reader also supports v2. After reader convergence, you may enable framing and then the v2 budget; enabling framing also frames the next nonzero merged continuation of an existing bare chain, so do not do this while bare-only readers remain.
+
+Restart nodes after changing their environment. Do not directly mix bare-only and framed-only binaries on the same continuation routes. Existing bare tokens must be routed to dual readers while any framed-only nodes remain. The v2 issuance switch is independent: keep it off until all readers support v2, but turning it off never removes an existing v2 budget.
+
+Partial JSON-shaped object keys remain local markers. To retain already issued cursors, a bare JSON object with the ODM tag and every historical writer field (`v`, `local`, `local_done`, `source`, `source_done`, `last_key`) is treated as an envelope, then strictly validated. A valid object key can be identical to that complete envelope: the two byte strings are indistinguishable, so legacy compatibility necessarily gives the envelope interpretation precedence. Framing identifies new merged tokens unambiguously, but dual-format readers do not eliminate this old full-envelope key collision. There is no signature, session store, or automatic format negotiation.
+
+An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. A zero-sized request does not spend an existing budget. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64-encoded JSON, optionally framed, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.
+
+With budget issuance off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. Default rollout does not fix that chain until the v2 switch is enabled. Framing alone does not impose the budget.
+
+For rollback, turn v2 issuance off, but choose framing for the readers being restored. When returning to the framed-only generation, keep framing `true` and route any outstanding bare tokens only to dual readers. Before restoring bare-only readers, set framing `false` on the remaining dual readers and deal with all outstanding framed tokens; v1-only readers also cannot resume v2 tokens. Neither switch rewrites existing framed or v2 chains, and tokens have no expiration that proves they have drained. Retain compatible readers for those continuations or have clients explicitly restart their listings before restoring incompatible binaries. Restarting a listing is a new scan and can repeat entries. Switching issuance off alone does not make outstanding tokens safe for older readers.
 
 ## Positioning
 
@@ -188,9 +202,9 @@ No write, delete, ACL or versioning permission is required or used. Scope the po
 
 Behaviour a client can observe. The "Test" column names the case that pins it: `*_test.rs` files live under `crates/e2e_test/src/on_demand_migration/`, and the unit tests live next to the code in `rustfs/src/app/object/get.rs`, `head.rs` and `shared.rs`.
 
-ODM merged continuation tokens use a NUL-prefixed JSON envelope inside the existing base64 encoding. NUL is not valid in a local object key, so a legitimate JSON-shaped key can never be mistaken for a merged cursor. Upgrade every node before using list-through, and restart any in-progress ODM listing issued by an older build: its unframed JSON tokens cannot be distinguished from legitimate local keys. Ordinary local listing tokens remain unchanged. Tokens issued by this build can still resume the local side after list-through is disabled.
+ODM merged continuation tokens use bare or NUL-prefixed JSON inside the existing base64 encoding. The default writer preserves framed output; an explicit `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS=false` keeps historical bare output during older-reader rollouts. Compatible readers accept both formats and retain existing budgets. See [List continuation token rollout](#list-continuation-token-rollout) for the independent issuance switches, rolling-upgrade requirements, and the unavoidable ambiguity between a complete historical envelope and an identically named local key.
 
-Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket, a missing source version, or an ambiguous GET 404 is not proof that the requested key is absent. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object.
+Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket or a missing source version is not proof that the requested key is absent. Native GCS verifies the bucket after either HEAD or GET returns 404 and preserves a failed probe as a source error. Azure accepts explicit `BlobNotFound` only on an unversioned object read with status 404; an ambiguous HEAD may make one container probe, while an ambiguous GET remains a source error. Native probes add at most one request and retain the existing per-request timeouts, rather than a single deadline for the pair. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object.
 
 Write-back currently requires namespace locking enabled and exactly one pool with one erasure set. Other topologies fail write-back explicitly as `unsupported`: source reads remain available, but backfill cannot complete successfully or certify cutover. This restriction avoids relying on a set-local condition across distinct pool or lock domains; it does not restrict ordinary S3 writes. Full cross-pool migration requires a globally fenced commit protocol.
 
@@ -358,8 +372,8 @@ sum by (bucket, reason) (rate(rustfs_on_demand_migration_pull_failures_total[5m]
 - **Source updates do not propagate.** Once an object is pulled, the local copy is authoritative; a later change on the source is never noticed. Plan the cutover so the source stops taking writes.
 - **Unversioned buckets re-pull deleted keys.** An unversioned bucket keeps nothing after a delete, so the key looks like an ordinary miss and is migrated again. Only a versioned bucket can shadow the source with a delete marker (`respect_local_delete_marker`).
 - **SSE-C source objects are not supported.** They are rejected with 424 `unsupported`; migrate them by another route.
-- **Anonymous (credential-less) sources are not supported yet.** `source.credentials: null` parses and passes structural validation, but the client builder has no anonymous mode, so the admin `PUT` refuses it and the runtime would treat such a bucket as unavailable. A public source still needs a key pair.
-- **Azure Blob is not a supported source** (rustfs/backlog#2166). GCS is supported only through its XML interoperability API with HMAC keys.
+- **Native Azure/GCS keys containing a standalone `.` or `..` path segment are unsupported.** The URL transport would remove that segment and address a different object. These keys fail before any source request; ordinary dotted names, repeated slashes and literal percent escapes keep their identity.
+- **Anonymous S3 sources are not supported yet.** `source.credentials: null` parses and passes structural validation, but the S3 client builder has no anonymous mode, so the admin `PUT` refuses it and the runtime would treat such a bucket as unavailable. A public S3 source still needs a key pair; native Azure/GCS credentials belong in their provider blocks.
 - **LIST merges the source only when asked, and only for v2.** With the default `policy.list_through = false` a client that lists before reading will not see un-migrated keys. Turning it on merges `ListObjectsV2` alone; `ListObjects` (v1) and `ListObjectVersions` stay local.
 - **A merged listing costs up to two local listings and two source listings per page** (one per side, plus a refill when the previous page consumed most of what that side had buffered). Walking N merged keys at `max-keys=K` therefore costs ceil(N/K) requests and between ceil(N/K) and 2*ceil(N/K) source listings. Source listings are capped at 10 per second per bucket (a compile-time constant); a listing that cannot get a slot inside one second is treated like a source failure and follows `policy.source_error`.
 - **A degraded merged page loses the source keys in its window.** Under `source_error = not_found` the page is answered locally and the source cursor is left where it was, so the keys the source would have contributed between the previous page's last key and this one are not shown again once pagination moves on. The `x-rustfs-on-demand-migration-list: local_only` header marks every page this happened on.

docs/operations/scanner-runtime-controls.md

@@ -280,10 +280,10 @@ Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/
 | `RUSTFS_HEAL_SET_BULKHEAD_ENABLE` | `true` (`DEFAULT_HEAL_SET_BULKHEAD_ENABLE`) | Per-set bulkhead scheduling. |
 | `RUSTFS_HEAL_PAGE_PARALLEL_ENABLE` | `true` (`DEFAULT_HEAL_PAGE_PARALLEL_ENABLE`) | Page-level parallel object healing during erasure-set repair. |
 | `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY` | `8` (`DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY`) | Concurrent object heals within one erasure-set page. Forced to `1` when page parallelism is off, for `Deep` scan mode, and for `AutoHeal`-sourced requests (`ErasureSetHealer::effective_heal_page_object_concurrency_for_source`). |
-| `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE` | `true` (`DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE`) | Pause best-effort heal task starts while foreground I/O is saturated. |
-| `RUSTFS_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT`, capped at 100) | Foreground read-permit utilization at which heal starts pause. |
-| `RUSTFS_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT`, capped at 100) | Foreground write utilization at which heal starts pause. |
-| `RUSTFS_HEAL_MAINLINE_MAX_SLEEP_MS` | `250` (`DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS`) | Recheck delay after deferring heal starts for foreground pressure. |
+| `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE` | `true` (`DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE`) | Defer best-effort starts and cooperatively pace running admin heal at safe work boundaries. |
+| `RUSTFS_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT`, capped at 100) | Read-utilization high watermark for start admission and running admin pacing; zero disables this class. |
+| `RUSTFS_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT`, capped at 100) | Write-utilization high watermark for start admission and running admin pacing; zero disables this class. |
+| `RUSTFS_HEAL_MAINLINE_MAX_SLEEP_MS` | `250` (`DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS`) | Start recheck interval; running admin waits cap each pacing-gate holder at 1000 ms. Zero disables running pacing. |
 | `RUSTFS_HEAL_OVERLAP_POLICY` | `merge` (`DEFAULT_HEAL_OVERLAP_POLICY`) | `merge` dedups an admin heal start that overlaps a running or queued heal; `minio_error` returns a typed already-running / overlapping-paths rejection like madmin. |
 | `RUSTFS_HEAL_MRF_ENABLE` | `true` (`DEFAULT_HEAL_MRF_ENABLE`) | MRF intent pipeline: error paths deliver repair intents to the heal runtime and unconsumed intents replay from the durable journal after restart. |
 | `RUSTFS_HEAL_MRF_QUEUE_SIZE` | `100000` (`DEFAULT_HEAL_MRF_QUEUE_SIZE`) | MRF in-memory queue capacity. |
@@ -291,6 +291,16 @@ Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/
 | `RUSTFS_HEAL_MRF_REPLAY_BATCH` | `256` (`DEFAULT_HEAL_MRF_REPLAY_BATCH`) | Intents per replay push round. |
 | `RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS` | `3600` (`DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS`, `crates/ecstore/src/set_disk/core/io_primitives.rs`) | A recently modified object is never deleted as dangling inside this window; `0` disables the grace window. |
 
+### Running admin heal pacing
+
+The manager passes its existing workload provider and a configuration snapshot into each admin execution. Bucket/prefix listing and object boundaries resample foreground pressure; erasure-set page workers also resample after earlier work releases page capacity. `High`, `Urgent`, and `force_start` do not exempt ordinary admin execution from this runtime pacing. The existing start-time bypass and overlap-control meanings are unchanged.
+
+Each execution has its own pacing latch, with no new global manager or cross-set pacing lock. The low watermark for each enabled class is `max(1, floor(high * 3 / 4))`: the default high watermark 80 therefore recovers below 60. High pressure latches pacing, and intermediate pressure resets the recovery window. Unpaced starts resume after sampled pressure remains below the low watermarks for four pause intervals, normally one second. While pressure persists, a pacing-gate holder waits only one interval, at most one second, then permits maintenance to continue. Concurrent page waiters serialize through this task-local gate; queue waiting still counts against the existing task execution timeout.
+
+The pacing gate holds neither namespace locks nor I/O/page permits while sleeping. At final page admission, each real permit acquisition gets a fresh, nonblocking pressure decision. Low-pressure work keeps that permit; only a unit that needs a pause releases capacity to wait. A unit that has completed one bounded pause may proceed despite persistent pressure, which supplies minimum maintenance progress without an endless acquire/pause loop. Existing object operations and commit tails are not interrupted because pressure rose. Cancellation and deadlines remain interruptible, and disabling pacing cannot bypass the global, per-set or page-concurrency hard caps. The existing `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE=false` setting is the operational opt-out for newly created executions; no additional request override is introduced.
+
+A missing provider, zero pause, or both class thresholds set to zero preserves unpaced execution. Missing counts follow the existing shared pressure interpreter; they are observations, not health, quorum or resource-ownership proof. The current provider exposes node-level workload classes, so this does not claim independent per-set foreground measurements or a hard global resource budget. Runtime waits increment `rustfs_heal_mainline_throttle_total` with `source=admin`, `result=delayed`, and a foreground-pressure or `recovery_window` reason. Real p99/throughput protection requires the separate W20 fixed-load ABBA measurements.
+
 ## Deliberate non-parity with MinIO
 
 These differences from MinIO are design decisions, recorded so they are not re-filed as gaps.

docs/testing/README.md

@@ -23,6 +23,8 @@ Every script named above is indexed with status and wiring in [`scripts/README.m
 
 The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation.
 
+The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree copy, encoding, and counted save costs without changing production cache behavior.
+
 ## Naming conventions
 
 ### Reserved test-name substrings (migration gate)

docs/testing/distributed-e2e.md

@@ -17,7 +17,7 @@ A multi-pool layout in which any pool spans several localhost ports is not expre
 
 Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran.
 
-The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata.
+The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job runs on GitHub-hosted `ubuntu-latest` and mounts four isolated 1 GiB tmpfs filesystems, then exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use the self-hosted `sm-standard-4` ARC pods: those cannot create filesystems (`mount -o loop` fails with `No such file or directory`, and `mount -t tmpfs` fails with `cannot mount tmpfs read-only`). Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata.
 
 The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing.
 

docs/testing/scanner-cache-cost.md

@@ -0,0 +1,36 @@
+# Scanner Cache Cost Profile
+
+The `cache_cost_profile_preserves_checkpoint_and_counts` test isolates the real cache operations used by the scanner: full clone, `copy_with_children`, checked flattening, MessagePack encoding, and `save_with_revisions_for_epoch`. It does not run a namespace walker or the scanner scheduler. An unchanged-cache save is deliberately requested to measure its cost, not to claim that production always saves cold buckets.
+
+```sh
+cargo test -p rustfs-scanner --lib cache_cost_profile -- --list
+RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib cache_cost_profile -- --nocapture
+env -u RUSTFLAGS -u CARGO_ENCODED_RUSTFLAGS \
+  CARGO_PROFILE_TEST_OPT_LEVEL=0 CARGO_PROFILE_DEV_OPT_LEVEL=0 \
+  RUSTFS_CACHE_COST_SOURCE="$(git rev-parse HEAD)" \
+  RUSTFS_CACHE_COST_TREE="$(git rev-parse HEAD^{tree})" \
+  RUST_MIN_STACK=4194304 RUSTFS_CACHE_COST_PROFILE=1 \
+  cargo test -p rustfs-scanner --lib cache_cost_profile -- --nocapture
+```
+
+The default positive control has 64 object entries and one sample for each of unchanged, small-dirty, and all-dirty caches. Explicit profiling uses 1,024, 4,096, and 16,384 object entries, each with five samples in all three scenarios. Small-dirty updates one percent of leaves, with a minimum of one; all-dirty updates every leaf. Each synthetic object initially accounts for two versions and 4,096 logical bytes. These are cache metadata fixtures, not uploaded S3 bodies. Small-dirty snapshots also carry a partial flag and resume marker. No test is ignored, and wall-time thresholds do not determine correctness.
+
+Every measured result is checked outside its timing interval: clone and subtree copy retain every field and entry; flattening yields exact object/version/byte counts; encoding reloads the same structure; saves write both main and backup and reload the same checkpoint. A stale revision with conflicting content must fail without replacing the preceding main cache. This checks the fixture's revision contract, not distributed CAS or publication-authority behavior.
+
+`CACHE_COST` JSON rows contain:
+
+The explicit profile command is an unoptimized Cargo test/debug run (`opt-level=0`), not a release build. Run from a clean worktree and retain the command, source SHA/tree, compiler version and relevant Cargo configuration with the raw rows. Each row records compile-time assertion mode, visible optimization/flag overrides and supplied source identifiers. Null build fields mean unrecorded, not inferred defaults; these fields alone do not discover every Cargo configuration source. Debug phase ratios are not production hotspot evidence and cannot justify a runtime optimization or close the performance task. No release rebuild is required for this bounded diagnostic.
+
+| Field | Meaning |
+|---|---|
+| `clone`, `copy_with_children`, `checked_flatten`, `encode` | Phase wall-clock p50 and maximum nanoseconds; setup, validation, and disposal are excluded. |
+| `save_inclusive` | Actual save entry-point wall time, including its own encoding, buffer copies, admission checks, and both backend calls. This overlaps the independently measured encode operation. |
+| `memory_backend_ingest` | Sum of time inside the two counted in-memory backend puts, including stream consumption and revision checking. It is part of `save_inclusive`, not an additional cost. |
+| `cache_wire_bytes` | Full snapshot's actual MessagePack size; not heap allocation, cloned bytes, or retained S3 payload bytes. |
+| `changed_entry_wire_bytes` | Sum of serialized changed leaf entries, excluding keys, ancestors and metadata; a diagnostic denominator, not a durable-progress proof. Zero in the unchanged scenario. |
+| `save_body_bytes_per_sample` | Bytes consumed by both successful main/backup put streams. It does not include network framing, erasure shards or retries. |
+| `retained_cache_entries` | Structurally verified cache entries including the root, not newly proven namespace coverage. |
+
+The fixture has two memory slots capped at 32 MiB each, at most 16,384 leaves, at most five samples per case, and nine profile rows. Oversized wire data and unknown configuration fail. No sample history grows with runtime and no permanent service starts. Profile runs must be exclusive of builds and other benchmarks; otherwise label the measurements exploratory/noisy. The default debug build is a diagnostic, not release throughput evidence. Repeated identical phases can benefit from warm allocator and CPU caches; the test does not establish absence of quadratic growth or bounded production RSS.
+
+Use the existing [scanner ABBA harness](../../scripts/scanner_abba.py) and [benchmark runbook](../operations/scanner-benchmark-runbook.md) for deployment comparisons. This microprofile does not supply deployment ABBA, a flamegraph, allocation attribution, syscall/fsync latency, remote RPC, erasure persistence, process-crash recovery, or a performance improvement. Only measured evidence can justify a separately reviewed runtime optimization; serialization, partial/complete proof, and persistence boundaries remain unchanged here.

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1787964612,
-        "narHash": "sha256-0N9nghg3nwzX6b6qc77EzjR9cu/Z+UR66FlfsCqiURs=",
+        "lastModified": 1788549839,
+        "narHash": "sha256-kOrCcSIA6w9J1hX5DqHy2k9pDTJymExTsbV74U9UtCA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e8be7818e19ada32105a8af937a6a473b38167ca",
+        "rev": "17de0b976395537756f30a3e78f2f06e5cec89ed",
         "type": "github"
       },
       "original": {
@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1787993548,
-        "narHash": "sha256-+IAEnmmx5YIhUWo0lp15jLLHchnXo5yKgWsi6C6Cf+0=",
+        "lastModified": 1788591095,
+        "narHash": "sha256-Vh+BeLWfbTT9AecazIsQ/Tkg/RzJeX3lEduANf256WA=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "996e9b0b019a4a9eb9e9a5641aefa06d801b5895",
+        "rev": "c361047d3a538f547f1617bb6b410411929ac9cc",
         "type": "github"
       },
       "original": {

rustfs/fixtures/on_demand_migration/source_config_e2a.rs

@@ -0,0 +1,80 @@
+// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01.
+// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted.
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+const REDACTED: &str = "REDACTED";
+
+/// The external S3-compatible source bucket.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SourceConfig {
+    pub provider: Provider,
+    /// `http(s)://host[:port]` with no path or query. Optional only for
+    /// [`Provider::Aws`], where it is derived from `region`.
+    #[serde(default)]
+    pub endpoint: Option<String>,
+    pub region: String,
+    pub bucket: String,
+    #[serde(default)]
+    pub path_style: PathStyle,
+    /// `None` means anonymous access to a public source bucket.
+    #[serde(default)]
+    pub credentials: Option<SourceCredentials>,
+    #[serde(default)]
+    pub tls: TlsConfig,
+}
+
+/// Source vendor family. `azure` is deliberately absent from this version.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Provider {
+    /// Generic S3-compatible endpoint.
+    S3,
+    Aws,
+    Minio,
+    Rustfs,
+    R2,
+    /// GCS XML interoperability API with HMAC keys.
+    Gcs,
+}
+
+/// Bucket addressing style. `auto` is resolved by the source client builder.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum PathStyle {
+    #[default]
+    Auto,
+    Path,
+    Virtual,
+}
+
+/// Static credentials for the source. `Debug` never prints the secret or
+/// the session token.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SourceCredentials {
+    pub access_key: String,
+    pub secret_key: String,
+    #[serde(default)]
+    pub session_token: Option<String>,
+}
+
+impl fmt::Debug for SourceCredentials {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SourceCredentials")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &REDACTED)
+            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED))
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TlsConfig {
+    #[serde(default)]
+    pub skip_verify: bool,
+    #[serde(default)]
+    pub ca_cert_pem: Option<String>,
+}

rustfs/src/admin/handlers/heal.rs

@@ -1636,6 +1636,44 @@ mod tests {
         assert!(executed.load(Ordering::SeqCst));
     }
 
+    #[tokio::test]
+    async fn heal_start_retry_preflight_failures_do_not_create_request_identities() {
+        let hip = HealInitParams {
+            bucket: "bucket".to_string(),
+            ..Default::default()
+        };
+        let mut request_ids = Vec::new();
+        for attempt in 0..3 {
+            let executed_ids = &mut request_ids;
+            let request_params = &hip;
+            let result = execute_after_heal_control_capability(
+                || async {
+                    if attempt < 2 {
+                        Err(super::cluster_heal_control_unavailable("test_capability_failure"))
+                    } else {
+                        Ok(())
+                    }
+                },
+                || async move {
+                    let request = build_heal_channel_request(request_params);
+                    executed_ids.push(request.id);
+                    Ok(())
+                },
+            )
+            .await;
+            if attempt < 2 {
+                assert!(result.is_err(), "failed capability checks must not start a heal");
+                assert!(
+                    request_ids.is_empty(),
+                    "preflight failure must precede request construction and admission"
+                );
+            } else {
+                result.expect("restored capabilities allow the first execution");
+                assert_eq!(request_ids.len(), 1);
+            }
+        }
+    }
+
     #[test]
     fn replacement_recovery_status_response_reports_cluster_proof() {
         let local = replacement_snapshot("11111111-1111-4111-8111-111111111111");
@@ -1743,6 +1781,21 @@ mod tests {
         assert!(decoded.is_none());
     }
 
+    #[test]
+    fn heal_start_retry_conflicts_keep_actionable_public_reasons() {
+        for (reason, label) in [
+            (HealAdmissionDropReason::AlreadyRunning, "already_running"),
+            (HealAdmissionDropReason::OverlappingPaths, "overlapping_paths"),
+        ] {
+            let error = reject_heal_admission(HealAdmissionResult::Dropped(reason));
+            assert_eq!(error.code(), &S3ErrorCode::OperationAborted);
+            assert!(
+                error.to_string().contains(label),
+                "the caller must distinguish conflicts from transient coordination failure"
+            );
+        }
+    }
+
     #[test]
     fn test_reject_heal_admission_preserves_retry_semantics() {
         for admission in [

rustfs/src/admin/handlers/user.rs

@@ -1131,6 +1131,7 @@ impl Operation for ImportIam {
                         expiration: req.expiration,
                         allow_site_replicator_account: false,
                         claims: Some(req.claims),
+                        status: None,
                     };
 
                     let groups = if req.groups.is_empty() { None } else { Some(req.groups) };

rustfs/src/admin/storage_api.rs

@@ -295,6 +295,8 @@ pub(crate) mod remote_s3_client {
 }
 
 pub(crate) mod metadata_sys {
+    #[cfg(test)]
+    pub(crate) use super::ecstore_bucket::metadata_sys::ConfigWriteLockProbe;
     use std::sync::Arc;
 
     use rustfs_policy::policy::BucketPolicy;
@@ -312,6 +314,7 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::get(bucket).await
     }
 
+    #[cfg(test)]
     pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
         crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await
     }
@@ -332,6 +335,25 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await
     }
 
+    /// [`update_if_incarnation`] stamping the config with a replicated edit's
+    /// source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn update_if_incarnation_at(
+        bucket: &str,
+        config_file: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: uuid::Uuid,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::update_if_incarnation_at(
+            bucket,
+            config_file,
+            data,
+            expected_incarnation_id,
+            updated_at,
+        )
+        .await
+    }
+
     pub(crate) async fn update_quota_if_incarnation(
         bucket: &str,
         data: Vec<u8>,
@@ -341,6 +363,25 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::update_quota_if_incarnation(bucket, data, expected_incarnation_id, proof).await
     }
 
+    /// [`update_quota_if_incarnation`] stamping the quota with a replicated
+    /// edit's source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn update_quota_if_incarnation_at(
+        bucket: &str,
+        data: Vec<u8>,
+        expected_incarnation_id: uuid::Uuid,
+        proof: &super::ecstore_notification::CrossPoolFenceFleetProofToken,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::update_quota_if_incarnation_at(
+            bucket,
+            data,
+            expected_incarnation_id,
+            proof,
+            updated_at,
+        )
+        .await
+    }
+
     pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result<uuid::Uuid> {
         super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await
     }
@@ -395,6 +436,18 @@ pub(crate) mod metadata_sys {
         super::ecstore_bucket::metadata_sys::delete_if_incarnation(bucket, config_file, expected_incarnation_id).await
     }
 
+    /// [`delete_if_incarnation`] stamping the cleared config with a replicated
+    /// deletion's source `updated_at` instead of the local clock (backlog#2292).
+    pub(crate) async fn delete_if_incarnation_at(
+        bucket: &str,
+        config_file: &str,
+        expected_incarnation_id: uuid::Uuid,
+        updated_at: OffsetDateTime,
+    ) -> Result<OffsetDateTime> {
+        super::ecstore_bucket::metadata_sys::delete_if_incarnation_at(bucket, config_file, expected_incarnation_id, updated_at)
+            .await
+    }
+
     pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
         super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await
     }
@@ -667,7 +720,7 @@ pub(crate) mod replication {
 }
 
 pub(crate) mod target {
-    pub(crate) use super::ecstore_bucket::target::duration_from_secs_or_nanos;
+    pub(crate) use super::ecstore_bucket::target::{ARN, duration_from_secs_or_nanos};
     pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget;
     pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType;
     pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets;

rustfs/src/app/bucket_usecase.rs

@@ -2783,42 +2783,49 @@ impl DefaultBucketUsecase {
         } else {
             (None, None)
         };
-        let (object_infos, degraded) = match source_state {
-            Some(state) => {
+        let (object_infos, degraded) = match (source_state, merged_token.as_ref()) {
+            (None, Some(token)) if params.max_keys == 0 => {
+                // No source was consulted, so retain every unconsumed side and
+                // the original wire format without spending its progress budget.
+                let is_truncated = !token.local_done || !token.source_done;
+                (
+                    StorageListObjectsV2Info {
+                        is_truncated,
+                        next_continuation_token: params.decoded_continuation_token.clone().filter(|_| is_truncated),
+                        ..Default::default()
+                    },
+                    false,
+                )
+            }
+            (None, None) => {
+                let infos = store
+                    .list_objects_v2(
+                        &bucket,
+                        &params.prefix,
+                        params.decoded_continuation_token.clone(),
+                        params.delimiter.clone(),
+                        params.max_keys,
+                        fetch_owner.unwrap_or_default(),
+                        params.start_after_for_query.clone(),
+                        incl_deleted,
+                    )
+                    .await
+                    .map_err(ApiError::from)?;
+                (infos, false)
+            }
+            (state, token) => {
                 let outcome = list_through::merged_list_objects_v2(
                     &store,
-                    &state,
+                    state.as_ref(),
                     &bucket,
                     &params,
                     fetch_owner.unwrap_or_default(),
                     incl_deleted,
-                    merged_token.as_ref(),
+                    token,
                 )
                 .await?;
                 (outcome.info, outcome.degraded)
             }
-            None => {
-                let cursor = list_through::local_cursor(params.decoded_continuation_token.as_deref(), merged_token.as_ref());
-                match cursor {
-                    list_through::LocalListCursor::Exhausted => (StorageListObjectsV2Info::default(), false),
-                    list_through::LocalListCursor::Token(token) => {
-                        let infos = store
-                            .list_objects_v2(
-                                &bucket,
-                                &params.prefix,
-                                token,
-                                params.delimiter.clone(),
-                                params.max_keys,
-                                fetch_owner.unwrap_or_default(),
-                                params.start_after_for_query.clone(),
-                                incl_deleted,
-                            )
-                            .await
-                            .map_err(ApiError::from)?;
-                        (infos, false)
-                    }
-                }
-            }
         };
 
         let output = build_list_objects_v2_output(

rustfs/src/app/storage_api.rs

@@ -718,6 +718,8 @@ pub(crate) mod bucket {
                     delete_marker_version_id: None,
                     delete_marker: false,
                     delete_marker_mtime: None,
+                    target_delete_marker_version_ids: Default::default(),
+                    target_delete_marker_version_ids_corrupt: false,
                     target_arns,
                     force_delete_id: Some(operation_id),
                     force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)),

rustfs/src/on_demand_migration/azure.rs

@@ -163,6 +163,32 @@ impl AzureSourceBackend {
         Ok(request)
     }
 
+    /// A missing blob is distinct from a missing container or version. Only
+    /// object reads may use BlobNotFound as positive evidence of absence.
+    async fn send_object_request(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
+        let is_head = request.method() == Method::HEAD;
+        let versioned = request
+            .url()
+            .query_pairs()
+            .any(|(name, _)| name.eq_ignore_ascii_case("versionid") || name.eq_ignore_ascii_case("snapshot"));
+        let response = self.http.execute(request).await?;
+        if response.status() == http::StatusCode::NOT_FOUND && !versioned {
+            match header(response.headers(), HEADER_ERROR_CODE) {
+                Some("BlobNotFound") => return Err(SourceError::NotFound),
+                None | Some("ResourceNotFound") if is_head => {
+                    // HEAD may omit an error code. One successful container
+                    // probe proves key absence; a failed probe keeps its error.
+                    // These are two independently timed requests, not one deadline.
+                    drop(response);
+                    self.probe().await?;
+                    return Err(SourceError::NotFound);
+                }
+                _ => {}
+            }
+        }
+        NativeHttp::check_response(response, Some(HEADER_ERROR_CODE))
+    }
+
     /// Shared mapping for Get Blob and Get Blob Properties.
     fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
         // A customer-provided key means the service holds ciphertext it cannot
@@ -189,7 +215,7 @@ impl AzureSourceBackend {
 impl SourceBackend for AzureSourceBackend {
     async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
         let request = self.request(Method::HEAD, self.blob_url(key)?, HeaderMap::new())?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.send_object_request(request).await?;
         Self::head_from_response(response.headers())
     }
 
@@ -202,7 +228,7 @@ impl SourceBackend for AzureSourceBackend {
             );
         }
         let request = self.request(Method::GET, self.blob_url(key)?, headers)?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.send_object_request(request).await?;
         let head = Self::head_from_response(response.headers())?;
         let content_range = header(response.headers(), "content-range").map(str::to_string);
         Ok(SourceGet {
@@ -240,7 +266,7 @@ impl SourceBackend for AzureSourceBackend {
         }
 
         let request = self.request(Method::GET, url, HeaderMap::new())?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
         let body = read_text(response, MAX_XML_BYTES).await?;
         let listing = parse_list_blobs(&body)?;
 
@@ -256,7 +282,7 @@ impl SourceBackend for AzureSourceBackend {
         let mut url = self.blob_url(key)?;
         url.query_pairs_mut().append_pair("comp", "tags");
         let request = self.request(Method::GET, url, HeaderMap::new())?;
-        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
+        let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
         let body = read_text(response, MAX_XML_BYTES).await?;
         parse_blob_tags(&body)
     }
@@ -265,7 +291,7 @@ impl SourceBackend for AzureSourceBackend {
         let mut url = self.container_url()?;
         url.query_pairs_mut().append_pair("restype", "container");
         let request = self.request(Method::HEAD, url, HeaderMap::new())?;
-        self.http.send(request, HEADER_ERROR_CODE).await?;
+        self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
         Ok(())
     }
 }
@@ -343,9 +369,9 @@ struct AzureListing {
 
 #[derive(Default)]
 struct BlobEntry {
-    name: String,
+    name: Option<String>,
     etag: Option<String>,
-    size: u64,
+    size: Option<u64>,
     last_modified: Option<std::time::SystemTime>,
     access_tier: Option<String>,
 }
@@ -358,6 +384,7 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
     let mut next_marker = None;
     let mut blob: Option<BlobEntry> = None;
     let mut in_blob_prefix = false;
+    let mut blob_prefix: Option<String> = None;
     // Open container elements. quick-xml reports a truncated document as a
     // plain end of input, so a non-zero depth at EOF is the only signal that
     // the page was cut short and must not be read as a complete listing.
@@ -367,6 +394,9 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
         match reader.read_event() {
             Ok(Event::Start(start)) => {
                 let name = local_name(start.name().as_ref());
+                if matches!(name.as_str(), "blob" | "blobprefix") && (blob.is_some() || in_blob_prefix) {
+                    return Err(SourceError::Other("source listing entries must not be nested".to_string()));
+                }
                 match name.as_str() {
                     "blob" => {
                         depth += 1;
@@ -385,27 +415,35 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
                         } else {
                             text
                         };
-                        apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix);
+                        apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?;
                     }
                 }
             }
             Ok(Event::Empty(empty)) => {
                 let name = local_name(empty.name().as_ref());
+                if matches!(name.as_str(), "blob" | "blobprefix") {
+                    return Err(SourceError::Other("source listing entry has no name".to_string()));
+                }
                 let text = if name == "name" {
                     decode_list_name(&empty, String::new())?
                 } else {
                     String::new()
                 };
-                apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix);
+                apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?;
             }
             Ok(Event::End(end)) => match local_name(end.name().as_ref()).as_str() {
                 "blob" => {
                     depth = depth.saturating_sub(1);
                     if let Some(entry) = blob.take() {
                         objects.push(SourceObject {
-                            key: entry.name,
+                            key: entry
+                                .name
+                                .filter(|name| !name.is_empty())
+                                .ok_or_else(|| SourceError::Other("source listing object has no name".to_string()))?,
                             etag: entry.etag,
-                            size: entry.size,
+                            size: entry
+                                .size
+                                .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?,
                             last_modified: entry.last_modified,
                             storage_class: entry.access_tier,
                             // Azure ETags carry no part count; the listing
@@ -417,6 +455,12 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
                 "blobprefix" => {
                     depth = depth.saturating_sub(1);
                     in_blob_prefix = false;
+                    prefixes.push(
+                        blob_prefix
+                            .take()
+                            .filter(|name| !name.is_empty())
+                            .ok_or_else(|| SourceError::Other("source listing prefix has no name".to_string()))?,
+                    );
                 }
                 "properties" | "blobs" | "enumerationresults" => depth = depth.saturating_sub(1),
                 _ => {}
@@ -478,16 +522,22 @@ fn apply_list_field(
     name: &str,
     text: String,
     blob: &mut Option<BlobEntry>,
-    prefixes: &mut Vec<String>,
+    blob_prefix: &mut Option<String>,
     next_marker: &mut Option<String>,
     in_blob_prefix: bool,
-) {
+) -> Result<(), SourceError> {
     match name {
         "name" => {
             if in_blob_prefix {
-                prefixes.push(text);
+                if blob_prefix.is_some() {
+                    return Err(SourceError::Other("source listing prefix has duplicate names".to_string()));
+                }
+                *blob_prefix = Some(text);
             } else if let Some(entry) = blob.as_mut() {
-                entry.name = text;
+                if entry.name.is_some() {
+                    return Err(SourceError::Other("source listing object has duplicate names".to_string()));
+                }
+                entry.name = Some(text);
             }
         }
         "nextmarker" => *next_marker = Some(text),
@@ -498,7 +548,14 @@ fn apply_list_field(
         }
         "content-length" => {
             if let Some(entry) = blob.as_mut() {
-                entry.size = text.trim().parse().unwrap_or(0);
+                if entry.size.is_some() {
+                    return Err(SourceError::Other("source listing object has duplicate sizes".to_string()));
+                }
+                entry.size = Some(
+                    text.trim()
+                        .parse()
+                        .map_err(|_| SourceError::Other("source listing object has no valid size".to_string()))?,
+                );
             }
         }
         "last-modified" => {
@@ -513,6 +570,7 @@ fn apply_list_field(
         }
         _ => {}
     }
+    Ok(())
 }
 
 /// Parses a `Get Blob Tags` response.
@@ -599,7 +657,7 @@ mod tests {
     use super::*;
     use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
     use crate::on_demand_migration::source_client::SourceError;
-    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
+    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server};
 
     const LIST_PAGE: &str = r#"<?xml version="1.0" encoding="utf-8"?>
 <EnumerationResults ServiceEndpoint="https://acct.blob.core.windows.net/" ContainerName="legacy">
@@ -769,6 +827,168 @@ mod tests {
         assert!(parse_blob_tags("<Tags><TagSet>").is_err(), "a truncated tag set must fail");
     }
 
+    #[tokio::test]
+    async fn native_listing_rejects_missing_or_invalid_required_object_fields() {
+        for entry in [
+            "<Blob />",
+            "<Blob><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name /><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name>broken</Name></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length /></Properties></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length>-1</Content-Length></Properties></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length>18446744073709551616</Content-Length></Properties></Blob>",
+            "<Blob><Name>broken</Name><Properties><Content-Length>not-a-size</Content-Length></Properties></Blob>",
+            "<BlobPrefix />",
+            "<BlobPrefix><Name /></BlobPrefix>",
+            "<BlobPrefix></BlobPrefix>",
+        ] {
+            // Reject the entire page even if a valid object precedes the bad
+            // entry, so callers cannot expose partial data or advance its cursor.
+            let body = format!(
+                "<EnumerationResults><Blobs><Blob><Name>valid</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker>next</NextMarker></EnumerationResults>"
+            );
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .list(&SourceListRequest {
+                    prefix: Some("dir/"),
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await
+                .expect_err("malformed object must reject the complete native page");
+            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
+            assert!(!err.is_retryable());
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2",
+                )],
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn native_listing_rejects_duplicate_fields_and_nested_entries() {
+        for entry in [
+            "<Blob><Name>a</Name><Name>b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name /><Name>b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>",
+            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length><Content-Length>2</Content-Length></Properties></Blob>",
+            "<BlobPrefix><Name>a/</Name><Name>b/</Name></BlobPrefix>",
+            "<BlobPrefix><Name /><Name>b/</Name></BlobPrefix>",
+            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length></Properties><Blob><Name>b</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></Blob>",
+            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length></Properties><BlobPrefix><Name>b/</Name></BlobPrefix></Blob>",
+            "<BlobPrefix><Name>a/</Name><Blob><Name>b</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></BlobPrefix>",
+            "<BlobPrefix><Name>a/</Name><BlobPrefix><Name>b/</Name></BlobPrefix></BlobPrefix>",
+        ] {
+            let body = format!(
+                "<EnumerationResults><Blobs><Blob><Name>valid</Name><Properties><Content-Length>0</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker>next</NextMarker></EnumerationResults>"
+            );
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let result = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .list(&SourceListRequest {
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await;
+            let err = result.expect_err("ambiguous entries must reject the entire page and its cursor");
+            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
+            assert!(!err.is_retryable(), "{entry}: {err:?}");
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2",
+                )],
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() {
+        let body = "<EnumerationResults><Blobs><Blob><Name>目录/空 &amp; file</Name><Properties><Content-Length>0</Content-Length></Properties></Blob><BlobPrefix><Name>目录/子/</Name></BlobPrefix></Blobs><NextMarker>opaque+/=</NextMarker></EnumerationResults>";
+        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+        let page = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+            .list(&SourceListRequest {
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("valid native page");
+        assert_eq!(page.objects.len(), 1);
+        assert_eq!(page.objects[0].key, "目录/空 & file");
+        assert_eq!(page.objects[0].size, 0);
+        assert_eq!(page.common_prefixes, ["目录/子/"]);
+        assert!(page.is_truncated);
+        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
+        assert_requests(&recorded, &[("GET", "/legacy?restype=container&comp=list&maxresults=2")]);
+    }
+
+    #[tokio::test]
+    async fn encoded_listing_preserves_required_field_and_entry_validation() {
+        for (entry, expected_error) in [
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name></Blob>"#,
+                "source listing object has no valid size",
+            ),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Properties><Content-Length>-1</Content-Length></Properties></Blob>"#,
+                "source listing object has no valid size",
+            ),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Name>a/b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>"#,
+                "source listing object has duplicate names",
+            ),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Properties><Content-Length>1</Content-Length><Content-Length>2</Content-Length></Properties></Blob>"#,
+                "source listing object has duplicate sizes",
+            ),
+            (
+                r#"<BlobPrefix><Name Encoded="true">a%2F</Name><Name>a/</Name></BlobPrefix>"#,
+                "source listing prefix has duplicate names",
+            ),
+            (r#"<BlobPrefix><Name Encoded="true" /></BlobPrefix>"#, "source listing prefix has no name"),
+            (
+                r#"<Blob><Name Encoded="true">a%2Fb</Name><Properties><Content-Length>1</Content-Length></Properties><Blob><Name Encoded="true">c%2Fd</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></Blob>"#,
+                "source listing entries must not be nested",
+            ),
+            (
+                r#"<BlobPrefix><Name Encoded="true">a%2F</Name><BlobPrefix><Name Encoded="true">b%2F</Name></BlobPrefix></BlobPrefix>"#,
+                "source listing entries must not be nested",
+            ),
+        ] {
+            let body = format!(
+                r#"<EnumerationResults><Blobs><Blob><Name Encoded="true">valid%252F</Name><Properties><Content-Length>0</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker Encoded="true">opaque%2B+marker</NextMarker></EnumerationResults>"#
+            );
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .list(&SourceListRequest {
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque%2B+marker"),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await
+                .expect_err("encoded names cannot bypass whole-page validation");
+            assert!(!err.is_retryable(), "{entry}: {err:?}");
+            let SourceError::Other(message) = err else {
+                panic!("wrong error class for {entry}: {err:?}");
+            };
+            assert_eq!(message, expected_error, "{entry}");
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%252B%2Bmarker&maxresults=2",
+                )],
+            );
+        }
+    }
+
     #[test]
     fn blob_tags_parse_into_the_shared_tag_map() {
         let tags = parse_blob_tags(TAGS).expect("tags should parse");
@@ -917,6 +1137,18 @@ mod tests {
         assert!(head.sse.is_none());
     }
 
+    #[tokio::test]
+    async fn dot_segment_keys_fail_before_any_source_request() {
+        let (endpoint, recorded) = scripted_server(Vec::new()).await;
+        let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
+        for key in [".", "..", "dir/./key", "dir/../key", "\u{fffe}/../key"] {
+            assert!(matches!(backend.head(key).await, Err(SourceError::Unsupported(_))), "HEAD {key:?}");
+            assert!(matches!(backend.get(key, None).await, Err(SourceError::Unsupported(_))), "GET {key:?}");
+            assert!(matches!(backend.tagging(key).await, Err(SourceError::Unsupported(_))), "tags {key:?}");
+        }
+        assert!(recorded.lock().expect("recorder lock").is_empty());
+    }
+
     #[tokio::test]
     async fn sas_credentials_travel_in_the_query_and_never_sign() {
         let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, blob_headers(), String::new())]).await;
@@ -1239,6 +1471,184 @@ mod tests {
         ]
     }
 
+    #[tokio::test]
+    async fn object_not_found_requires_provider_evidence_or_one_successful_head_probe() {
+        for method in [Method::HEAD, Method::GET] {
+            for (status, code, expected) in [
+                (404, Some("BlobNotFound"), "not_found"),
+                (403, Some("BlobNotFound"), "access_denied"),
+                (404, Some("ContainerNotFound"), "other"),
+                (404, Some("BlobVersionNotFound"), "other"),
+                (404, Some("UnrecognizedError"), "other"),
+                (404, None, if method == Method::HEAD { "not_found" } else { "other" }),
+                (404, Some("ResourceNotFound"), if method == Method::HEAD { "not_found" } else { "other" }),
+            ] {
+                let probes = method == Method::HEAD && status == 404 && matches!(code, None | Some("ResourceNotFound"));
+                let headers = code
+                    .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())])
+                    .unwrap_or_default();
+                let mut responses = vec![ScriptedResponse::new(status, headers, "untrusted-error-body".to_string())];
+                if probes {
+                    responses.push(ScriptedResponse::new(200, Vec::new(), String::new()));
+                }
+                let (endpoint, recorded) = scripted_server(responses).await;
+                let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
+                let result = if method == Method::HEAD {
+                    backend.head("missing").await.map(|_| ())
+                } else {
+                    backend.get("missing", None).await.map(|_| ())
+                };
+                let err = result.expect_err("object error must remain an error");
+                assert_eq!(err.class_label(), expected, "{method} {status} {code:?}: {err:?}");
+                assert!(!err.is_retryable(), "{err:?}");
+                assert!(!err.to_string().contains("untrusted-error-body"));
+                let mut requests = vec![(method.as_str(), "/legacy/missing")];
+                if probes {
+                    requests.push(("HEAD", "/legacy?restype=container"));
+                }
+                assert_requests(&recorded, &requests);
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn s3_not_found_alias_never_proves_native_object_absence() {
+        for selector in [None, Some("versionid"), Some("snapshot")] {
+            for operation in ["head", "get", "list", "tags", "probe"] {
+                if selector.is_some() && !matches!(operation, "head" | "get") {
+                    continue;
+                }
+                for (status, expected, retryable) in [
+                    (403, "access_denied", false),
+                    (404, "other", false),
+                    (416, "other", false),
+                    (500, "server_error", true),
+                ] {
+                    let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                        status,
+                        vec![(HEADER_ERROR_CODE, "NoSuchKey".to_string())],
+                        "untrusted-error-body".to_string(),
+                    )])
+                    .await;
+                    let credential = selector.map_or_else(
+                        || Credential::SharedKey(vec![7_u8; 32]),
+                        |selector| Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]),
+                    );
+                    let backend = backend(&endpoint, credential);
+                    let result = match operation {
+                        "head" => backend.head("missing").await.map(|_| ()),
+                        "get" => backend.get("missing", None).await.map(|_| ()),
+                        "list" => backend.list(&SourceListRequest::default()).await.map(|_| ()),
+                        "tags" => backend.tagging("missing").await.map(|_| ()),
+                        "probe" => backend.probe().await,
+                        _ => unreachable!(),
+                    };
+                    let err = result.expect_err("an S3 error alias is not Azure absence evidence");
+                    assert_eq!(err.class_label(), expected, "{operation} {selector:?} HTTP {status}: {err:?}");
+                    assert_eq!(err.is_retryable(), retryable, "{operation} {selector:?} HTTP {status}: {err:?}");
+                    if status == 500 {
+                        assert!(matches!(err, SourceError::ServerError(500)));
+                    }
+                    assert!(!err.to_string().contains("untrusted-error-body"));
+                    let (method, mut target) = match operation {
+                        "head" => ("HEAD", "/legacy/missing".to_string()),
+                        "get" => ("GET", "/legacy/missing".to_string()),
+                        "list" => ("GET", "/legacy?restype=container&comp=list".to_string()),
+                        "tags" => ("GET", "/legacy/missing?comp=tags".to_string()),
+                        "probe" => ("HEAD", "/legacy?restype=container".to_string()),
+                        _ => unreachable!(),
+                    };
+                    if let Some(selector) = selector {
+                        target.push_str(&format!("?{selector}=old-version"));
+                    }
+                    assert_requests(&recorded, &[(method, target.as_str())]);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn ambiguous_head_preserves_the_container_probe_failure() {
+        for (status, expected, retryable) in [
+            (403, "access_denied", false),
+            (404, "other", false),
+            (429, "throttled", true),
+            (500, "server_error", true),
+            (503, "throttled", true),
+        ] {
+            let (endpoint, recorded) = scripted_server(vec![
+                ScriptedResponse::new(404, Vec::new(), String::new()),
+                // A BlobNotFound header on a container request cannot prove
+                // that the object is missing, regardless of this status.
+                ScriptedResponse::new(status, vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())], String::new()),
+            ])
+            .await;
+            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
+                .head("missing")
+                .await
+                .expect_err("failed probe must not become object absence");
+            assert_eq!(err.class_label(), expected, "probe {status}: {err:?}");
+            assert_eq!(err.is_retryable(), retryable, "probe {status}: {err:?}");
+            if status == 500 {
+                assert!(matches!(err, SourceError::ServerError(500)));
+            }
+            assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("HEAD", "/legacy?restype=container")]);
+        }
+    }
+
+    #[tokio::test]
+    async fn version_and_snapshot_absence_are_not_missing_current_blobs() {
+        for selector in ["versionid", "snapshot"] {
+            for code in [None, Some("BlobNotFound"), Some("ResourceNotFound")] {
+                for method in [Method::HEAD, Method::GET] {
+                    let headers = code
+                        .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())])
+                        .unwrap_or_default();
+                    let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(404, headers, String::new())]).await;
+                    let backend = backend(&endpoint, Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]));
+                    let result = if method == Method::HEAD {
+                        backend.head("object").await.map(|_| ())
+                    } else {
+                        backend.get("object", None).await.map(|_| ())
+                    };
+                    let err = result.expect_err("missing selected version must remain a source error");
+                    assert!(matches!(err, SourceError::Other(_)), "{method} {selector} {code:?}: {err:?}");
+                    assert_requests(&recorded, &[(method.as_str(), &format!("/legacy/object?{selector}=old-version"))]);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn blob_not_found_header_is_not_object_absence_for_list_or_tags() {
+        for tags in [false, true] {
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                404,
+                vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())],
+                String::new(),
+            )])
+            .await;
+            let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
+            let result = if tags {
+                backend.tagging("missing").await.map(|_| ())
+            } else {
+                backend.list(&SourceListRequest::default()).await.map(|_| ())
+            };
+            assert!(matches!(result, Err(SourceError::Other(_))), "tags={tags}: {result:?}");
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    if tags {
+                        "/legacy/missing?comp=tags"
+                    } else {
+                        "/legacy?restype=container&comp=list"
+                    },
+                )],
+            );
+        }
+    }
+
     #[tokio::test]
     async fn azure_backend_satisfies_the_shared_backend_contract() {
         let mut ranged = contract_blob_headers();
@@ -1246,7 +1656,7 @@ mod tests {
         // A HEAD reports the object size with no body, exactly as Azure does.
         let mut head_only = contract_blob_headers();
         head_only.push(("Content-Length", "5".to_string()));
-        let (endpoint, _) = scripted_server(vec![
+        let (endpoint, recorded) = scripted_server(vec![
             ScriptedResponse::new(200, head_only, String::new()),
             ScriptedResponse::new(200, contract_blob_headers(), "hello".to_string()),
             ScriptedResponse::new(206, ranged, "ell".to_string()),
@@ -1276,6 +1686,23 @@ mod tests {
             },
         )
         .await;
+        assert_requests(
+            &recorded,
+            &[
+                ("HEAD", "/legacy/dir/a.txt"),
+                ("GET", "/legacy/dir/a.txt"),
+                ("GET", "/legacy/dir/a.txt"),
+                ("GET", "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&maxresults=2"),
+                (
+                    "GET",
+                    "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=cursor-1&maxresults=2",
+                ),
+                ("GET", "/legacy/dir/a.txt?comp=tags"),
+                ("HEAD", "/legacy?restype=container"),
+                ("HEAD", "/legacy/missing"),
+                ("HEAD", "/legacy/secret"),
+            ],
+        );
     }
 
     #[tokio::test]

rustfs/src/on_demand_migration/config.rs

@@ -106,12 +106,12 @@ pub struct SourceConfig {
     pub tls: TlsConfig,
     /// Required for [`Provider::Azure`] and rejected for every other
     /// provider.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub azure: Option<AzureSourceConfig>,
     /// Required for [`Provider::GcsNative`] and rejected for every other
     /// provider. [`Provider::Gcs`] keeps using `credentials` because it
     /// speaks the S3 interoperability API.
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub gcs: Option<GcsSourceConfig>,
 }
 
@@ -808,6 +808,10 @@ impl EndpointKey {
 mod tests {
     use super::*;
 
+    mod before_native_sources {
+        include!("../../fixtures/on_demand_migration/source_config_e2a.rs");
+    }
+
     const FULL_JSON: &str = r#"{
   "version": 1,
   "enabled": true,
@@ -878,6 +882,32 @@ mod tests {
         assert_eq!(minimal.policy.source_timeout.first_byte_ms, 15_000);
     }
 
+    #[test]
+    fn s3_config_writes_remain_readable_by_the_strict_pre_native_reader() {
+        // FULL_JSON is the complete config fixture already present in e2a921bc.
+        for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] {
+            let mut old_wire: serde_json::Value = serde_json::from_str(FULL_JSON).expect("historical config fixture");
+            old_wire["source"]["provider"] = provider.into();
+            let config = OnDemandMigrationConfig::from_json(&serde_json::to_vec(&old_wire).expect("historical wire"))
+                .expect("current reader accepts the historical source");
+            let wire = config.to_json().expect("persist current config");
+            let actual: serde_json::Value = serde_json::from_slice(&wire).expect("persisted config JSON");
+            let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone())
+                .expect("an existing S3 source must remain readable by the strict e2a source consumer");
+            assert_eq!(serde_json::to_value(old_source).expect("old reader wire"), old_wire["source"]);
+            assert_eq!(actual, old_wire, "provider={provider}: no existing config field or value may change");
+
+            for field in ["azure", "gcs"] {
+                let mut rejected = old_wire["source"].clone();
+                rejected[field] = serde_json::Value::Null;
+                assert!(
+                    serde_json::from_value::<before_native_sources::SourceConfig>(rejected).is_err(),
+                    "the frozen old reader must reject {field}, even when null"
+                );
+            }
+        }
+    }
+
     #[test]
     fn unknown_fields_are_rejected_at_every_level() {
         for (label, json) in [
@@ -1080,6 +1110,19 @@ mod tests {
         for cfg in [azure_cfg(), gcs_native_cfg()] {
             let json = cfg.to_json().expect("config must serialize");
             assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg);
+            let wire: serde_json::Value = serde_json::from_slice(&json).expect("native config JSON");
+            let (present, absent, expected) = match cfg.source.provider {
+                Provider::Azure => ("azure", "gcs", serde_json::to_value(&cfg.source.azure).expect("Azure block")),
+                Provider::GcsNative => ("gcs", "azure", serde_json::to_value(&cfg.source.gcs).expect("GCS block")),
+                _ => unreachable!("native fixture"),
+            };
+            assert!(expected.is_object(), "native credentials must be present");
+            assert_eq!(wire["source"][present], expected);
+            assert!(wire["source"].get(absent).is_none());
+            assert!(
+                serde_json::from_value::<before_native_sources::SourceConfig>(wire["source"].clone()).is_err(),
+                "native providers still require upgraded readers"
+            );
         }
         // The wire labels are part of the admin contract.
         assert!(

rustfs/src/on_demand_migration/gcs.rs

@@ -55,10 +55,6 @@ use url::Url;
 /// Read-only object scope: this backend never writes to the source.
 const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
 const METADATA_PREFIX: &str = "x-goog-meta-";
-/// GCS reports its error code in the response body, not a header; the shared
-/// transport takes a header name, so it is given one that never matches and
-/// classification falls back to the status.
-const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
 /// One `objects.list` page is small; refuse an unbounded document.
 const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;
 
@@ -125,7 +121,7 @@ impl GcsNativeSourceBackend {
     }
 
     async fn send_object(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
-        match self.http.send_object(request, NO_ERROR_CODE_HEADER).await {
+        match self.http.send_object(request, None).await {
             Err(SourceError::NotFound) => {
                 // An XML object URL also returns 404 when its bucket is gone.
                 // Reuse the read-only listing probe before caching a key miss.
@@ -225,7 +221,7 @@ impl SourceBackend for GcsNativeSourceBackend {
         }
 
         let request = self.request(Method::GET, url, HeaderMap::new()).await?;
-        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
+        let response = self.http.send(request, None).await?;
         let body = read_text(response, MAX_JSON_BYTES).await?;
         parse_objects_list(&body)
     }
@@ -245,7 +241,7 @@ impl SourceBackend for GcsNativeSourceBackend {
         let mut url = self.objects_url()?;
         url.query_pairs_mut().append_pair("maxResults", "1");
         let request = self.request(Method::GET, url, HeaderMap::new()).await?;
-        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
+        let response = self.http.send(request, None).await?;
         read_text(response, MAX_JSON_BYTES)
             .await
             .and_then(|body| parse_objects_list(&body))?;
@@ -284,28 +280,38 @@ struct ListedObject {
 fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
     let listing: ObjectsList =
         serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
+    if listing.prefixes.iter().any(|prefix| prefix.is_empty()) {
+        return Err(SourceError::Other("source listing prefix has no name".to_string()));
+    }
     let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
     let objects = listing
         .items
         .into_iter()
         .map(|item| {
+            if item.name.is_empty() {
+                return Err(SourceError::Other("source listing object has no name".to_string()));
+            }
+            let size = item
+                .size
+                .and_then(|size| size.parse::<u64>().ok())
+                .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?;
             let etag = item
                 .md5_hash
                 .as_deref()
                 .and_then(base64_md5_to_hex)
                 .or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
                 .filter(|etag| !etag.is_empty());
-            SourceObject {
+            Ok(SourceObject {
                 key: item.name,
                 etag,
-                size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
+                size,
                 last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
                 storage_class: item.storage_class,
                 // GCS never encodes a part count in a digest or an ETag.
                 is_multipart_etag: false,
-            }
+            })
         })
-        .collect();
+        .collect::<Result<_, SourceError>>()?;
 
     Ok(SourcePage {
         objects,
@@ -319,7 +325,7 @@ fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
 mod tests {
     use super::*;
     use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
-    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
+    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server};
     use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;
 
     const LIST_PAGE_ONE: &str = r#"{
@@ -372,6 +378,17 @@ mod tests {
         ]
     }
 
+    #[tokio::test]
+    async fn dot_segment_keys_fail_before_any_source_request() {
+        let (endpoint, recorded) = scripted_server(Vec::new()).await;
+        let backend = backend(&endpoint);
+        for key in [".", "..", "dir/./key", "dir/../key", "\u{fffe}/../key"] {
+            assert!(matches!(backend.head(key).await, Err(SourceError::Unsupported(_))), "HEAD {key:?}");
+            assert!(matches!(backend.get(key, None).await, Err(SourceError::Unsupported(_))), "GET {key:?}");
+        }
+        assert!(recorded.lock().expect("recorder lock").is_empty());
+    }
+
     #[test]
     fn objects_list_maps_items_prefixes_and_the_page_token() {
         let page = parse_objects_list(LIST_PAGE_ONE).expect("page should parse");
@@ -562,6 +579,197 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn native_listing_rejects_missing_or_invalid_required_object_fields() {
+        for entry in [
+            r#"{"size":"1"}"#,
+            r#"{"name":"","size":"1"}"#,
+            r#"{"name":"broken"}"#,
+            r#"{"name":"broken","size":null}"#,
+            r#"{"name":"broken","size":""}"#,
+            r#"{"name":"broken","size":"-1"}"#,
+            r#"{"name":"broken","size":"18446744073709551616"}"#,
+            r#"{"name":"broken","size":"not-a-size"}"#,
+            r#"{"name":"broken","size":1}"#,
+        ] {
+            let body = format!(r#"{{"items":[{{"name":"valid","size":"1"}},{entry}],"nextPageToken":"next"}}"#);
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
+            let err = backend(&endpoint)
+                .list(&SourceListRequest {
+                    prefix: Some("dir/"),
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await
+                .expect_err("malformed object must reject the complete native page");
+            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
+            assert!(!err.is_retryable());
+            assert_requests(
+                &recorded,
+                &[(
+                    "GET",
+                    "/storage/v1/b/legacy/o?prefix=dir%2F&delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2",
+                )],
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn native_listing_rejects_empty_prefix_entries() {
+        for body in [
+            r#"{"items":[{"name":"valid","size":"1"}],"prefixes":[""],"nextPageToken":"next"}"#,
+            r#"{"prefixes":[""],"nextPageToken":"next"}"#,
+            r#"{"prefixes":["目录/子/",""],"nextPageToken":"next"}"#,
+        ] {
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+            let result = backend(&endpoint)
+                .list(&SourceListRequest {
+                    delimiter: Some("/"),
+                    continuation_token: Some("opaque+/="),
+                    max_keys: 2,
+                    ..Default::default()
+                })
+                .await;
+            let err = result.expect_err("an empty prefix must reject the entire page and its cursor");
+            assert!(matches!(err, SourceError::Other(_)), "{body}: {err:?}");
+            assert!(!err.is_retryable());
+            assert_requests(
+                &recorded,
+                &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2")],
+            );
+        }
+
+        let body = r#"{"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#;
+        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+        let page = backend(&endpoint)
+            .list(&SourceListRequest {
+                delimiter: Some("/"),
+                max_keys: 1,
+                ..Default::default()
+            })
+            .await
+            .expect("a valid prefix-only page must remain usable");
+        assert!(page.objects.is_empty());
+        assert_eq!(page.common_prefixes, ["目录/子/"]);
+        assert!(page.is_truncated);
+        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
+        assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&maxResults=1")]);
+    }
+
+    #[tokio::test]
+    async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() {
+        let body = r#"{"items":[{"name":"目录/空 & file","size":"0"}],"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#;
+        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
+        let page = backend(&endpoint)
+            .list(&SourceListRequest {
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("valid native page");
+        assert_eq!(page.objects.len(), 1);
+        assert_eq!(page.objects[0].key, "目录/空 & file");
+        assert_eq!(page.objects[0].size, 0);
+        assert_eq!(page.common_prefixes, ["目录/子/"]);
+        assert!(page.is_truncated);
+        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
+        assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?maxResults=2")]);
+    }
+
+    #[tokio::test]
+    async fn missing_object_head_requires_one_successful_bucket_probe() {
+        for (status, body, expected, retryable) in [
+            (200, "{}", "not_found", false),
+            (403, "", "access_denied", false),
+            (404, "", "other", false),
+            (429, "", "throttled", true),
+            (500, "", "server_error", true),
+            (503, "", "throttled", true),
+            (200, "not JSON", "other", false),
+        ] {
+            let (endpoint, recorded) = scripted_server(vec![
+                ScriptedResponse::new(404, Vec::new(), String::new()),
+                ScriptedResponse::new(status, Vec::new(), body.to_string()),
+            ])
+            .await;
+            let err = backend(&endpoint).head("missing").await.expect_err("missing HEAD must fail");
+            assert_eq!(err.class_label(), expected, "probe {status} {body:?}: {err:?}");
+            assert_eq!(err.is_retryable(), retryable, "probe {status} {body:?}: {err:?}");
+            if status == 500 {
+                assert!(matches!(err, SourceError::ServerError(500)));
+            }
+            assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("GET", "/storage/v1/b/legacy/o?maxResults=1")]);
+        }
+    }
+
+    #[tokio::test]
+    async fn denied_object_reads_do_not_probe_or_become_object_absence() {
+        for method in [Method::HEAD, Method::GET] {
+            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                403,
+                vec![("x-goog-unused-error-code", "NoSuchKey".to_string())],
+                "untrusted-error-body".to_string(),
+            )])
+            .await;
+            let backend = backend(&endpoint);
+            let result = if method == Method::HEAD {
+                backend.head("missing").await.map(|_| ())
+            } else {
+                backend.get("missing", None).await.map(|_| ())
+            };
+            let err = result.expect_err("denied object read must remain a failure");
+            assert_eq!(err.class_label(), "access_denied");
+            assert!(!err.is_retryable());
+            assert!(!err.to_string().contains("untrusted-error-body"));
+            assert_requests(&recorded, &[(method.as_str(), "/legacy/missing")]);
+        }
+    }
+    #[tokio::test]
+    async fn non_object_errors_ignore_untrusted_error_code_headers() {
+        for probe in [false, true] {
+            for (status, expected, retryable) in [(403, "access_denied", false), (500, "server_error", true)] {
+                let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
+                    status,
+                    vec![("x-goog-unused-error-code", "NoSuchKey".to_string())],
+                    "untrusted-error-body".to_string(),
+                )])
+                .await;
+                let backend = backend(&endpoint);
+                let result = if probe {
+                    backend.probe().await
+                } else {
+                    backend
+                        .list(&SourceListRequest {
+                            max_keys: 2,
+                            ..Default::default()
+                        })
+                        .await
+                        .map(|_| ())
+                };
+                let err = result.expect_err("a synthetic provider header cannot change the source status");
+                assert_eq!(err.class_label(), expected, "probe={probe} status={status}: {err:?}");
+                assert_eq!(err.is_retryable(), retryable);
+                assert!(!err.to_string().contains("untrusted-error-body"));
+                if status == 500 {
+                    assert!(matches!(err, SourceError::ServerError(500)));
+                }
+                assert_requests(
+                    &recorded,
+                    &[(
+                        "GET",
+                        if probe {
+                            "/storage/v1/b/legacy/o?maxResults=1"
+                        } else {
+                            "/storage/v1/b/legacy/o?maxResults=2"
+                        },
+                    )],
+                );
+            }
+        }
+    }
+
     /// GCS states its error code in the response body, which this backend never
     /// reads, so every class must follow from the status alone. The classes are
     /// what the runtime acts on: only `NotFound` is negative-cached, and only a

rustfs/src/on_demand_migration/list_through.rs

@@ -94,13 +94,16 @@ pub struct MergePick {
 }
 
 /// The continuation-token envelope. Opaque to clients: it is serialized as
-/// framed JSON and then base64-encoded by the same helper as a local marker.
+/// JSON, optionally framed, then base64-encoded like a local marker.
 ///
 /// A `null` cursor with `done = false` means "list that side from the start";
 /// `done = true` means the side is finished and must not be listed again.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct ListThroughToken {
+    /// Transport framing observed by the decoder, never an envelope field.
+    #[serde(skip)]
+    pub framed: bool,
     /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
     pub t: String,
     pub v: u32,
@@ -127,6 +130,7 @@ pub struct ListThroughToken {
 impl ListThroughToken {
     fn new(local: SideCursor, source: SideCursor, last_key: Option<String>) -> Self {
         Self {
+            framed: false,
             t: LIST_THROUGH_TOKEN_TAG.to_string(),
             v: LIST_THROUGH_TOKEN_VERSION,
             local: local.token,
@@ -141,7 +145,12 @@ impl ListThroughToken {
     pub fn encode(&self) -> String {
         // The envelope is built here from owned strings, so serialization
         // cannot fail; the fallback keeps the signature infallible.
-        format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
+        let json = serde_json::to_string(self).unwrap_or_default();
+        if self.framed {
+            format!("{LIST_THROUGH_TOKEN_PREFIX}{json}")
+        } else {
+            json
+        }
     }
 }
 
@@ -165,16 +174,30 @@ pub enum ListThroughTokenError {
 
 /// Classifies an already base64-decoded continuation token.
 ///
-/// Only a framed JSON object is read as a merged token;
-/// anything else is a local marker, so a bucket that turns `list_through` off
-/// keeps paginating with the tokens it handed out. A token that *is* an
-/// envelope but was tampered with (unknown version, unknown field, truncated
-/// JSON) is an error, never a silent fallback.
+/// Framed envelopes and complete historical writer envelopes are merged tokens.
+/// Partial JSON-shaped keys remain local markers. A key identical to a complete
+/// historical envelope is inherently ambiguous and retains merged semantics.
+/// Recognized envelopes share the same version, count and field validation.
 pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
-    let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
-        return Ok(ListThroughCursor::Local(decoded.to_string()));
+    let (payload, framed) = match decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) {
+        Some(payload) => (payload, true),
+        None if decoded.starts_with('{') => (decoded, false),
+        None => return Ok(ListThroughCursor::Local(decoded.to_string())),
     };
-    let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
+    let value = match serde_json::from_str::<serde_json::Value>(payload) {
+        Ok(value) => value,
+        Err(_) if framed => return Err(ListThroughTokenError::Malformed),
+        Err(_) => return Ok(ListThroughCursor::Local(decoded.to_string())),
+    };
+    // RUSTFS_COMPAT_TODO(odm-list-bare-envelope): old writers issued bare JSON. Remove after all supported readers understand framing and outstanding bare listings have drained or explicitly restarted.
+    if !framed
+        && (value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG)
+            || ["v", "local", "local_done", "source", "source_done", "last_key"]
+                .iter()
+                .any(|field| value.get(field).is_none()))
+    {
+        return Ok(ListThroughCursor::Local(decoded.to_string()));
+    }
     if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
         return Err(ListThroughTokenError::Malformed);
     }
@@ -198,7 +221,10 @@ pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, Lis
         None => return Err(ListThroughTokenError::Malformed),
     }
     serde_json::from_value::<ListThroughToken>(value)
-        .map(|token| ListThroughCursor::Merged(Box::new(token)))
+        .map(|mut token| {
+            token.framed = framed;
+            ListThroughCursor::Merged(Box::new(token))
+        })
         .map_err(|_| ListThroughTokenError::Malformed)
 }
 
@@ -643,6 +669,126 @@ impl Default for SourceListRateLimiter {
     }
 }
 
+// Frozen framed-only codec from e1608fbd9ca934d157b5de46c80b4393f2dd3dd6.
+// Keep its own DTO and constants: current-reader round trips cannot establish
+// whether a deployed framed-only reader accepts the bytes we issue.
+#[cfg(test)]
+pub(crate) mod e160_framed_reader {
+    use serde::{Deserialize, Serialize};
+
+    /// The continuation-token version used by ordinary progressing pages.
+    pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
+    const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
+
+    /// The sixteenth consecutive merged page without a key or new EOF fails.
+    /// This also bounds legitimate sparse listings; it is not a cycle detector.
+    pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
+
+    /// Envelope marker. A bucket that is *not* merging hands out the local
+    /// listing's own marker, so the decoder needs a positive signal before it
+    /// treats an opaque token as a merged one.
+    const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
+    // Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix),
+    // so this framing cannot collide with a local key used as an opaque marker.
+    const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:";
+
+    /// The continuation-token envelope. Opaque to clients: it is serialized as
+    /// framed JSON and then base64-encoded by the same helper as a local marker.
+    ///
+    /// A `null` cursor with `done = false` means "list that side from the start";
+    /// `done = true` means the side is finished and must not be listed again.
+    #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+    #[serde(deny_unknown_fields)]
+    pub struct ListThroughToken {
+        /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
+        pub t: String,
+        pub v: u32,
+        #[serde(default)]
+        pub local: Option<String>,
+        #[serde(default)]
+        pub local_done: bool,
+        #[serde(default)]
+        pub source: Option<String>,
+        #[serde(default)]
+        pub source_done: bool,
+        /// Last entry the previous page consumed. A side whose page was only
+        /// partially consumed is re-listed from the same cursor and everything at
+        /// or below this key is dropped, which is delimiter-safe: a rolled-up
+        /// common prefix compares as itself, never as its members.
+        #[serde(default)]
+        pub last_key: Option<String>,
+        /// Consecutive empty truncated merged pages, present only in v2 tokens.
+        /// Ordinary v1 tokens retain their original serialized shape.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        pub no_progress: Option<u8>,
+    }
+
+    impl ListThroughToken {
+        pub fn encode(&self) -> String {
+            // The envelope is built here from owned strings, so serialization
+            // cannot fail; the fallback keeps the signature infallible.
+            format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
+        }
+    }
+
+    /// What a decoded (base64-stripped) continuation token turned out to be.
+    #[derive(Clone, Debug, PartialEq, Eq)]
+    pub enum ListThroughCursor {
+        /// A plain local listing marker: the bucket was not merging when the token
+        /// was issued, or the client is paginating a non-merged listing.
+        Local(String),
+        Merged(Box<ListThroughToken>),
+    }
+
+    #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
+    pub enum ListThroughTokenError {
+        #[error("continuation token version {0} is not supported")]
+        UnsupportedVersion(u32),
+        /// The message never echoes the token: it is client-controlled input.
+        #[error("continuation token is malformed")]
+        Malformed,
+    }
+
+    /// Classifies an already base64-decoded continuation token.
+    ///
+    /// Only a framed JSON object is read as a merged token;
+    /// anything else is a local marker, so a bucket that turns `list_through` off
+    /// keeps paginating with the tokens it handed out. A token that *is* an
+    /// envelope but was tampered with (unknown version, unknown field, truncated
+    /// JSON) is an error, never a silent fallback.
+    pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
+        let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        };
+        let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
+        if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
+            return Err(ListThroughTokenError::Malformed);
+        }
+        match value.get("v").and_then(serde_json::Value::as_u64) {
+            Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
+                // v1 readers reject this field even when it is null or zero.
+                if value.get("no_progress").is_some() {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
+                if !value
+                    .get("no_progress")
+                    .and_then(serde_json::Value::as_u64)
+                    .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
+                {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
+            None => return Err(ListThroughTokenError::Malformed),
+        }
+        serde_json::from_value::<ListThroughToken>(value)
+            .map(|token| ListThroughCursor::Merged(Box::new(token)))
+            .map_err(|_| ListThroughTokenError::Malformed)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -797,6 +943,7 @@ mod tests {
     #[test]
     fn a_degraded_page_keeps_the_source_cursor_for_the_next_one() {
         let resume = ListThroughToken {
+            framed: false,
             t: LIST_THROUGH_TOKEN_TAG.to_string(),
             v: LIST_THROUGH_TOKEN_VERSION,
             local: Some("local-1".to_string()),
@@ -1033,7 +1180,7 @@ mod tests {
 
     #[test]
     fn token_round_trips_and_rejects_tampering() {
-        let token = ListThroughToken::new(
+        let mut token = ListThroughToken::new(
             SideCursor {
                 token: Some("l".to_string()),
                 done: false,
@@ -1041,6 +1188,7 @@ mod tests {
             SideCursor { token: None, done: true },
             Some("k".to_string()),
         );
+        token.framed = true;
         let encoded = token.encode();
         assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
 
@@ -1089,35 +1237,148 @@ mod tests {
 
     #[test]
     fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
-        fn framed(payload: &str) -> String {
-            format!("{LIST_THROUGH_TOKEN_PREFIX}{payload}")
-        }
-
         let token = progress_token(None, true, false);
         assert_eq!(
             token.encode(),
-            concat!(
-                "\0odm-list:",
-                r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
-            )
+            r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
         );
-        for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
-            let token = progress_token(Some(count), true, false);
-            assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
-        }
-        for version in [1, 2] {
-            for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
-                let encoded = framed(&format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#));
+        for framed in [false, true] {
+            let prefix = if framed { LIST_THROUGH_TOKEN_PREFIX } else { "" };
+            for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
+                let mut token = progress_token(Some(count), true, false);
+                token.framed = framed;
+                assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
+            }
+            // Bare recognition requires the complete shape emitted by old writers;
+            // partial JSON objects are also valid local keys.
+            for version in [1, 2] {
+                for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
+                    let encoded = format!(
+                        r#"{prefix}{{"t":"odm-list","v":{version},"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":{value}}}"#
+                    );
+                    assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
+                }
+            }
+            for encoded in [
+                r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1}"#,
+                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#,
+                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"extra":true}"#,
+                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"framed":true}"#,
+            ] {
+                let encoded = format!("{prefix}{encoded}");
                 assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
             }
+            let bumped = format!("{prefix}{}", token.encode().replace("\"v\":1", "\"v\":9"));
+            assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(9)));
         }
-        for payload in [
-            r#"{"t":"odm-list","v":1,"no_progress":1}"#,
-            r#"{"t":"odm-list","v":2}"#,
-            r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
+    }
+
+    // Frozen decoder from 447f3c704, before framing was introduced. Keeping this
+    // independent of the current decoder catches a default-writer rollout break.
+    fn decode_before_framing(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
+        if !decoded.starts_with('{') {
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        }
+        let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
+            // Not JSON at all: an object key may legitimately start with '{'.
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        };
+        if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
+            return Ok(ListThroughCursor::Local(decoded.to_string()));
+        }
+        match value.get("v").and_then(serde_json::Value::as_u64) {
+            Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
+                // v1 readers reject this field even when it is null or zero.
+                if value.get("no_progress").is_some() {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
+                if !value
+                    .get("no_progress")
+                    .and_then(serde_json::Value::as_u64)
+                    .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
+                {
+                    return Err(ListThroughTokenError::Malformed);
+                }
+            }
+            Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
+            None => return Err(ListThroughTokenError::Malformed),
+        }
+        serde_json::from_value::<ListThroughToken>(value)
+            .map(|token| ListThroughCursor::Merged(Box::new(token)))
+            .map_err(|_| ListThroughTokenError::Malformed)
+    }
+
+    #[test]
+    fn historical_writer_fixtures_and_default_output_remain_readable() {
+        for (wire, version, count) in [
+            (
+                r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#,
+                1,
+                None,
+            ),
+            (
+                r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#,
+                2,
+                Some(15),
+            ),
         ] {
-            let encoded = framed(payload);
-            assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
+            let ListThroughCursor::Merged(mut token) = decode_continuation_token(wire).expect("historical issued token") else {
+                panic!("a historical cursor must not silently become a local marker, even if a key has identical JSON");
+            };
+            assert_eq!(token.local.as_deref(), Some("local-2"));
+            assert_eq!(token.source.as_deref(), Some("source-2"));
+            assert_eq!(token.last_key.as_deref(), Some("k"));
+            assert_eq!(token.v, version);
+            assert_eq!(token.no_progress, count);
+            assert!(!token.framed);
+            assert_eq!(token.encode(), wire, "bare output retains the historical bytes");
+            assert_eq!(decode_before_framing(&token.encode()), Ok(ListThroughCursor::Merged(token.clone())));
+            token.framed = true;
+            let framed = format!("\0odm-list:{wire}");
+            assert_eq!(token.encode(), framed, "framing leaves the JSON payload unchanged");
+            assert_eq!(decode_continuation_token(&framed), Ok(ListThroughCursor::Merged(token)));
+        }
+    }
+
+    #[test]
+    fn frozen_e160_reader_distinguishes_framing_and_keeps_strict_budget_validation() {
+        use super::e160_framed_reader as old;
+
+        for raw in [
+            r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#,
+            r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#,
+        ] {
+            assert_eq!(old::decode_continuation_token(raw), Ok(old::ListThroughCursor::Local(raw.to_string())));
+            let framed = format!("\0odm-list:{raw}");
+            let old::ListThroughCursor::Merged(old_token) = old::decode_continuation_token(&framed).expect("old writer bytes")
+            else {
+                panic!("e160 recognizes its own frame");
+            };
+            assert_eq!(old_token.encode(), framed);
+            let ListThroughCursor::Merged(current) = decode_continuation_token(&framed).expect("dual reader") else {
+                panic!("dual readers preserve old framed chains");
+            };
+            assert_eq!(current.encode(), framed);
+            assert_eq!(current.local, old_token.local);
+            assert_eq!(current.local_done, old_token.local_done);
+            assert_eq!(current.source, old_token.source);
+            assert_eq!(current.source_done, old_token.source_done);
+            assert_eq!(current.last_key, old_token.last_key);
+            assert_eq!(current.v, old_token.v);
+            assert_eq!(current.no_progress, old_token.no_progress);
+        }
+        for count in ["null", "0", "16", "-1", "1.5", "\"1\"", "256"] {
+            let raw = format!(
+                "\0odm-list:{{\"t\":\"odm-list\",\"v\":2,\"local\":null,\"local_done\":true,\"source\":\"A\",\"source_done\":false,\"last_key\":null,\"no_progress\":{count}}}"
+            );
+            assert_eq!(
+                old::decode_continuation_token(&raw),
+                Err(old::ListThroughTokenError::Malformed),
+                "{count}"
+            );
+            assert_eq!(decode_continuation_token(&raw), Err(ListThroughTokenError::Malformed), "{count}");
         }
     }
 

rustfs/src/on_demand_migration/native_http.rs

@@ -99,6 +99,7 @@ impl NativeHttp {
     pub(super) fn for_test(endpoint: Url) -> Self {
         Self {
             client: reqwest::Client::builder()
+                .no_proxy()
                 .redirect(reqwest::redirect::Policy::none())
                 .build()
                 .expect("test http client should build"),
@@ -116,19 +117,27 @@ impl NativeHttp {
                 .path_segments_mut()
                 .map_err(|_| SourceError::Other("source endpoint cannot carry a path".to_string()))?;
             path.clear();
-            path.extend(segments);
+            for segment in segments {
+                // URL normalization drops standalone dot segments. Sending
+                // that URL could fetch another object and backfill its bytes
+                // under the originally requested key.
+                if matches!(segment, "." | "..") {
+                    return Err(SourceError::Unsupported("source path contains an unsupported dot segment".to_string()));
+                }
+                path.push(segment);
+            }
         }
         Ok(url)
     }
 
     /// Sends the request and returns the response only for a 2xx status.
-    /// Non-2xx statuses are classified from the status and the provider's own
+    /// Non-2xx statuses are classified from the status and an optional provider
     /// error-code header; response bodies are not read, so no provider message
     /// can smuggle credentials or markup into a log line.
     pub(super) async fn send(
         &self,
         request: reqwest::Request,
-        error_code_header: &str,
+        error_code_header: Option<&str>,
     ) -> Result<reqwest::Response, SourceError> {
         self.send_classified(request, error_code_header, false).await
     }
@@ -137,7 +146,7 @@ impl NativeHttp {
     pub(super) async fn send_object(
         &self,
         request: reqwest::Request,
-        error_code_header: &str,
+        error_code_header: Option<&str>,
     ) -> Result<reqwest::Response, SourceError> {
         self.send_classified(request, error_code_header, true).await
     }
@@ -145,26 +154,42 @@ impl NativeHttp {
     async fn send_classified(
         &self,
         request: reqwest::Request,
-        error_code_header: &str,
+        error_code_header: Option<&str>,
         not_found_on_404_without_code: bool,
     ) -> Result<reqwest::Response, SourceError> {
-        let response = self.client.execute(request).await.map_err(classify_transport_error)?;
+        let response = self.execute(request).await?;
+        let status = response.status();
+        match Self::check_response(response, error_code_header) {
+            Err(SourceError::Other(_)) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
+            result => result,
+        }
+    }
+
+    pub(super) async fn execute(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
+        self.client.execute(request).await.map_err(classify_transport_error)
+    }
+
+    pub(super) fn check_response(
+        response: reqwest::Response,
+        error_code_header: Option<&str>,
+    ) -> Result<reqwest::Response, SourceError> {
         let status = response.status();
         if status.is_success() {
             return Ok(response);
         }
-        let code = response
-            .headers()
-            .get(error_code_header)
+        let code = error_code_header
+            .and_then(|header| response.headers().get(header))
             .and_then(|value| value.to_str().ok())
             .map(str::to_string);
         let message = match &code {
             Some(code) => format!("source returned HTTP {status} ({code})"),
             None => format!("source returned HTTP {status}"),
         };
-        match classify_status(status.as_u16(), code.as_deref(), message) {
-            SourceError::Other(_) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
-            err => Err(err),
+        match classify_status(status.as_u16(), code.as_deref(), message.clone()) {
+            // Native object absence needs provider-specific evidence or a
+            // successful bucket probe, never an alias from the S3 classifier.
+            SourceError::NotFound => Err(classify_status(status.as_u16(), None, message)),
+            error => Err(error),
         }
     }
 }
@@ -431,4 +456,44 @@ mod tests {
         assert_eq!(url.as_str(), "https://acct.blob.core.windows.net/container/dir/a%20b%3Fc%23d.txt");
         assert_eq!(url.query(), None, "a key with '?' must not become a query");
     }
+
+    #[test]
+    fn native_http_refuses_dot_segments_instead_of_addressing_another_object() {
+        let http = NativeHttp::for_test(Url::parse("https://source.example.com").expect("origin"));
+        for key in [
+            ".",
+            "..",
+            "./key",
+            "../key",
+            "dir/./key",
+            "dir/../key",
+            "dir/.",
+            "dir/..",
+            "\u{fffe}/../key",
+        ] {
+            let error = http
+                .url(std::iter::once("bucket").chain(key.split('/')))
+                .expect_err("dot segments must not disappear");
+            assert!(matches!(error, SourceError::Unsupported(_)), "{key:?}: {error}");
+        }
+    }
+
+    #[test]
+    fn native_http_preserves_ordinary_dots_empty_segments_and_literal_escapes() {
+        let http = NativeHttp::for_test(Url::parse("https://source.example.com").expect("origin"));
+        for (key, path) in [
+            ("file.txt", "/bucket/file.txt"),
+            (".hidden/.../tail.", "/bucket/.hidden/.../tail."),
+            ("/dir//key/", "/bucket//dir//key/"),
+            ("%2e/%2E%2E/key", "/bucket/%252e/%252E%252E/key"),
+            ("a+b &?#", "/bucket/a+b%20&%3F%23"),
+        ] {
+            let url = http
+                .url(std::iter::once("bucket").chain(key.split('/')))
+                .expect("representable key");
+            assert_eq!(url.path(), path, "{key:?}");
+            assert!(url.query().is_none());
+            assert!(url.fragment().is_none());
+        }
+    }
 }

rustfs/src/on_demand_migration/source_client.rs

@@ -337,7 +337,7 @@ const THROTTLE_CODES: &[&str] = &[
     "RequestThrottled",
     "ServerBusy",
 ];
-const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "BlobNotFound"];
+const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"];
 const ACCESS_DENIED_CODES: &[&str] = &[
     "AccessDenied",
     "InvalidAccessKeyId",
@@ -1813,10 +1813,11 @@ mod tests {
 
     /// The S3 backend behind the scripted connector, without the prefix-mapping
     /// client on top: the contract is a property of the backend itself.
-    async fn scripted_s3_backend(responses: Vec<Scripted>) -> S3SourceBackend {
+    async fn scripted_s3_backend(responses: Vec<Scripted>) -> (S3SourceBackend, Recorded) {
         let spec = spec(None);
+        let requests: Recorded = Arc::new(Mutex::new(Vec::new()));
         let connector = SharedHttpConnector::new(ScriptedConnector {
-            requests: Arc::new(Mutex::new(Vec::new())),
+            requests: Arc::clone(&requests),
             responses: Arc::new(Mutex::new(responses.into_iter().collect())),
         });
         let http_client = http_client_fn(move |_settings, _components| connector.clone());
@@ -1826,17 +1827,20 @@ mod tests {
             .expect("test spec should build")
             .http_client(http_client)
             .interceptor(SourceProxyMarkerInterceptor::new());
-        S3SourceBackend {
-            client: S3Client::from_conf(config.build()),
-            bucket: spec.bucket.clone(),
-        }
+        (
+            S3SourceBackend {
+                client: S3Client::from_conf(config.build()),
+                bucket: spec.bucket.clone(),
+            },
+            requests,
+        )
     }
 
     #[tokio::test]
     async fn s3_backend_satisfies_the_shared_backend_contract() {
         let mut ranged = contract_object_headers(3);
         ranged.push(("content-range", "bytes 1-3/5".to_string()));
-        let backend = scripted_s3_backend(vec![
+        let (backend, requests) = scripted_s3_backend(vec![
             ok(contract_object_headers(5), ""),
             ok(contract_object_headers(5), "hello"),
             ok(ranged, "ell"),
@@ -1845,6 +1849,7 @@ mod tests {
             ok(Vec::new(), CONTRACT_TAGGING),
             ok(Vec::new(), ""),
             status(404, ""),
+            // An object HEAD 404 requires the existing S3 bucket HEAD probe.
             ok(Vec::new(), ""),
             status(403, ACCESS_DENIED_BODY),
         ])
@@ -1859,6 +1864,32 @@ mod tests {
             },
         )
         .await;
+        let requests = recorded(&requests);
+        let actual: Vec<_> = requests
+            .iter()
+            .map(|request| {
+                (
+                    request.method.as_str(),
+                    url::Url::parse(&request.uri).expect("recorded S3 URL").path().to_string(),
+                )
+            })
+            .collect();
+        let expected = [
+            ("HEAD", "/source-bucket/dir/a.txt"),
+            ("GET", "/source-bucket/dir/a.txt"),
+            ("GET", "/source-bucket/dir/a.txt"),
+            ("GET", "/source-bucket/"),
+            ("GET", "/source-bucket/"),
+            ("GET", "/source-bucket/dir/a.txt"),
+            ("HEAD", "/source-bucket/"),
+            ("HEAD", "/source-bucket/missing"),
+            ("HEAD", "/source-bucket/"),
+            ("HEAD", "/source-bucket/secret"),
+        ];
+        assert_eq!(actual, expected.map(|(method, path)| (method, path.to_string())));
+        for request in &requests {
+            assert_outbound_markers(request);
+        }
     }
 
     fn prefix_client(prefix: Option<String>) -> SourceClient {

rustfs/src/on_demand_migration/test_http_fixture.rs

@@ -56,6 +56,16 @@ impl RecordedRequest {
 
 pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;
 
+/// Checks the full request sequence, including the absence of extra probes.
+pub(super) fn assert_requests(recorder: &Recorder, expected: &[(&str, &str)]) {
+    let recorded = recorder.lock().expect("recorder lock");
+    let actual: Vec<_> = recorded
+        .iter()
+        .map(|request| (request.method.as_str(), request.target.as_str()))
+        .collect();
+    assert_eq!(actual, expected, "unexpected native source request sequence");
+}
+
 /// Binds a loopback listener that answers `responses` in order and returns its
 /// origin plus the recorder. The task ends once the script is exhausted.
 pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {

rustfs/src/site_replication/hooks.rs

@@ -302,7 +302,164 @@ pub(crate) fn site_replication_state_replicates_ilm_expiry(state: &SiteReplicati
     state.peers.values().any(|peer| peer.replicate_ilm_expiry)
 }
 
-pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteReplicationBootstrapPlan> {
+/// Secret-bearing half of the IAM snapshot. `SRInfo` is served to admin
+/// callers (`site-replication/info`, status, add preflight) and must stay
+/// secret-free, so the bootstrap plan receives credentials through this
+/// separate value, built only on the paths that deliver to peers (site add
+/// bootstrap, repair, retry snapshot resend). Never persisted, never served.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct SiteReplicationIamCredentials {
+    /// Built-in users (access key -> credential); temp and service accounts
+    /// are excluded, external/IdP users never appear here.
+    pub(crate) users: BTreeMap<String, SiteReplicationUserCredential>,
+    /// Every service account except the site replicator's own, already
+    /// shaped as the `service-account` create item the live hook emits.
+    pub(crate) service_accounts: Vec<SiteReplicationServiceAccountSnapshot>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SiteReplicationUserCredential {
+    pub(crate) secret_key: String,
+    pub(crate) status: AccountStatus,
+    /// The user record's own update time (the axis the receiver's staleness
+    /// check compares against), unlike `UserInfo::updated_at` which
+    /// `list_users` overwrites with the policy mapping's time.
+    pub(crate) updated_at: Option<OffsetDateTime>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct SiteReplicationServiceAccountSnapshot {
+    pub(crate) create: SRSvcAccCreate,
+    pub(crate) envelope: Option<SRSvcAccReplicationEnvelope>,
+    pub(crate) updated_at: Option<OffsetDateTime>,
+}
+
+pub(crate) const SERVICE_ACCOUNT_ENVELOPE_VERSION: u64 = 2;
+
+pub(crate) fn encode_service_account_replication_policy(
+    claims: &HashMap<String, Value>,
+    session_policy: Option<&str>,
+) -> S3Result<(SRSessionPolicy, Option<SRSvcAccReplicationEnvelope>)> {
+    if !claims.contains_key(OIDC_VIRTUAL_PARENT_CLAIM) {
+        return session_policy
+            .map(SRSessionPolicy::from_json)
+            .transpose()
+            .map(|policy| policy.unwrap_or_default())
+            .map(|policy| (policy, None))
+            .map_err(|err| s3_error!(InvalidArgument, "marshal policy failed: {:?}", err));
+    }
+
+    let policy = match session_policy {
+        Some(policy) => serde_json::from_str::<Policy>(policy)
+            .map_err(|err| s3_error!(InvalidArgument, "invalid service account replication policy: {:?}", err))?,
+        None => Policy::default(),
+    };
+    if policy.statements.is_empty() && (!policy.id.is_empty() || !policy.version.is_empty())
+        || policy.version.is_empty() && !policy.statements.is_empty()
+    {
+        return Err(s3_error!(InvalidArgument, "service account replication policy is not normalized"));
+    }
+    let policy = serde_json::to_string(&policy)
+        .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?;
+    let policy = SRSessionPolicy::from_json(&policy)
+        .map_err(|err| s3_error!(InternalError, "marshal service account replication policy failed: {:?}", err))?;
+    Ok((
+        policy,
+        Some(SRSvcAccReplicationEnvelope {
+            version: SERVICE_ACCOUNT_ENVELOPE_VERSION,
+        }),
+    ))
+}
+
+/// Read the credentials the IAM snapshot needs straight from the IAM store:
+/// `list_users` deliberately strips secret keys and skips service accounts,
+/// which is right for an admin listing and wrong for a peer snapshot (the
+/// plan builder used to drop every user for lack of a secret, so a status
+/// change or secret rotation committed while a peer was unreachable never
+/// reached it — backlog#2289).
+pub(crate) async fn build_sr_iam_credentials() -> S3Result<SiteReplicationIamCredentials> {
+    let mut credentials = SiteReplicationIamCredentials::default();
+    let Some(iam_sys) = current_iam_handle() else {
+        return Ok(credentials);
+    };
+
+    let mut users = HashMap::new();
+    iam_sys.load_users(UserType::Reg, &mut users).await.map_err(ApiError::from)?;
+    for (access_key, identity) in users {
+        if identity.credentials.is_temp() || identity.credentials.is_service_account() {
+            continue;
+        }
+        credentials.users.insert(
+            access_key,
+            SiteReplicationUserCredential {
+                secret_key: identity.credentials.secret_key,
+                status: if identity.credentials.status == "off" {
+                    AccountStatus::Disabled
+                } else {
+                    AccountStatus::Enabled
+                },
+                updated_at: identity.update_at,
+            },
+        );
+    }
+
+    let mut service_accounts = HashMap::new();
+    iam_sys
+        .load_users(UserType::Svc, &mut service_accounts)
+        .await
+        .map_err(ApiError::from)?;
+    let mut service_accounts: Vec<_> = service_accounts.into_iter().collect();
+    service_accounts.sort_by(|(a, _), (b, _)| a.cmp(b));
+    for (access_key, identity) in service_accounts {
+        // The replicator account is installed by join / rotate, never by a snapshot.
+        if access_key == SITE_REPLICATOR_SERVICE_ACCOUNT || !identity.credentials.is_service_account() {
+            continue;
+        }
+        let claims = iam_sys.get_claims_for_svc_acc(&access_key).await.map_err(ApiError::from)?;
+        let (account, session_policy) = iam_sys.get_service_account(&access_key).await.map_err(ApiError::from)?;
+        let session_policy = session_policy
+            .map(|policy| serde_json::to_string(&policy))
+            .transpose()
+            .map_err(|err| {
+                S3Error::with_message(
+                    S3ErrorCode::InternalError,
+                    format!("marshal service account session policy failed: {err:?}"),
+                )
+            })?;
+        let (session_policy, envelope) = encode_service_account_replication_policy(&claims, session_policy.as_deref())?;
+        credentials.service_accounts.push(SiteReplicationServiceAccountSnapshot {
+            create: SRSvcAccCreate {
+                parent: identity.credentials.parent_user,
+                access_key,
+                secret_key: identity.credentials.secret_key,
+                groups: identity.credentials.groups.unwrap_or_default(),
+                claims,
+                session_policy,
+                status: identity.credentials.status,
+                name: account.name.unwrap_or_default(),
+                description: account.description.unwrap_or_default(),
+                expiration: account.expiration,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            },
+            envelope,
+            updated_at: identity.update_at,
+        });
+    }
+
+    Ok(credentials)
+}
+
+/// The bootstrap plan for peer delivery: `info` (secret-free) plus the IAM
+/// credentials read at this moment.
+pub(crate) async fn build_site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteReplicationBootstrapPlan> {
+    let credentials = build_sr_iam_credentials().await?;
+    site_replication_bootstrap_plan(info, &credentials)
+}
+
+pub(crate) fn site_replication_bootstrap_plan(
+    info: &SRInfo,
+    credentials: &SiteReplicationIamCredentials,
+) -> S3Result<SiteReplicationBootstrapPlan> {
     let mut plan = SiteReplicationBootstrapPlan::default();
     let replicate_ilm_expiry = site_replication_info_replicates_ilm_expiry(info);
 
@@ -318,24 +475,57 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
     }
 
     for (access_key, user) in &info.user_info_map {
-        if let Some(secret_key) = &user.secret_key {
-            plan.iam_items.push(SRIAMItem {
-                r#type: "iam-user".to_string(),
-                iam_user: Some(rustfs_madmin::SRIAMUser {
-                    access_key: access_key.clone(),
-                    is_delete_req: false,
-                    user_req: Some(AddOrUpdateUserReq {
-                        secret_key: secret_key.clone(),
-                        policy: user.policy_name.clone(),
-                        status: user.status.clone(),
-                    }),
-                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        // Credentials come from the store snapshot; an inline `secret_key` on
+        // the SRInfo entry (older callers, tests) is accepted as a fallback.
+        // Users with neither (external / IdP identities) have nothing a peer
+        // could install and are skipped.
+        let credential = credentials.users.get(access_key);
+        let Some(secret_key) = credential
+            .map(|credential| credential.secret_key.clone())
+            .or_else(|| user.secret_key.clone())
+            .filter(|secret_key| !secret_key.is_empty())
+        else {
+            continue;
+        };
+        let status = credential
+            .map(|credential| credential.status.clone())
+            .unwrap_or_else(|| user.status.clone());
+        let updated_at = credential.and_then(|credential| credential.updated_at).or(user.updated_at);
+        plan.iam_items.push(SRIAMItem {
+            r#type: "iam-user".to_string(),
+            iam_user: Some(rustfs_madmin::SRIAMUser {
+                access_key: access_key.clone(),
+                is_delete_req: false,
+                user_req: Some(AddOrUpdateUserReq {
+                    secret_key,
+                    policy: user.policy_name.clone(),
+                    status,
                 }),
-                updated_at: user.updated_at,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            updated_at,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        });
+    }
+
+    // Service accounts follow their parents: the receiver creates a missing
+    // account under `parent` and updates an existing one (secret, status,
+    // session policy), so a rotation or disable committed during an outage
+    // converges through the same snapshot as users do.
+    for account in &credentials.service_accounts {
+        plan.iam_items.push(SRIAMItem {
+            r#type: "service-account".to_string(),
+            svc_acc_change: Some(SRSvcAccChange {
+                create: Some(account.create.clone()),
+                oidc_service_account_envelope: account.envelope.clone(),
                 api_version: Some(SITE_REPL_API_VERSION.to_string()),
                 ..Default::default()
-            });
-        }
+            }),
+            updated_at: account.updated_at,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        });
     }
 
     for (name, desc) in &info.group_desc_map {
@@ -518,7 +708,12 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
     } else {
         path
     };
-    broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await?;
+    // Both steps run to completion on their own: the broadcast attempts every
+    // peer and reports the first failure (backlog#2293), so stopping here on
+    // that error would skip `configure-replication` for the peers whose
+    // `make` just succeeded — and nothing records a retry for that gap. The
+    // failed peer's retry events cover both steps independently.
+    let make_result = broadcast_site_replication_json_using_runtime(runtime, &path, &serde_json::json!({})).await;
 
     let configure_path = bootstrap_bucket_op_path(bucket, "configure-replication");
     let configure_path = if let Some(token) = bootstrap_token {
@@ -526,7 +721,8 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
     } else {
         configure_path
     };
-    broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
+    let configure_result = broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await;
+    make_result.and(configure_result)
 }
 
 const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
@@ -832,6 +1028,21 @@ pub async fn site_replication_iam_change_hook(item: SRIAMItem) -> S3Result<()> {
     let Some(runtime) = runtime_site_replication_targets().await? else {
         return Ok(());
     };
+    // A local revoke must out-rank a stale grant a peer delivers later, so its
+    // mark is committed before the broadcast (backlog#2291). The broadcast
+    // still goes out when the mark cannot be persisted: the peers' own records
+    // remain the primary gate, the mark only covers the deleted case.
+    if let Err(err) = record_iam_deletion_marks_for_item(&item).await {
+        warn!(
+            component = LOG_COMPONENT_ADMIN,
+            subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
+            event = EVENT_ADMIN_SITE_REPLICATION_STATE,
+            item_type = %item.r#type,
+            result = "iam_deletion_mark_not_recorded",
+            error = ?err,
+            "failed to record local IAM deletion mark before broadcast"
+        );
+    }
     let mut first_error: Option<S3Error> = None;
     for peer in runtime.state.peers.values() {
         if peer.deployment_id == runtime.local_peer.deployment_id

rustfs/src/site_replication/mod.rs

@@ -79,13 +79,16 @@ use http::header::{CONTENT_TYPE, HOST};
 use http::{HeaderMap, HeaderValue, Uri};
 use hyper::{Method, StatusCode};
 use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH};
+use rustfs_iam::federation::OIDC_VIRTUAL_PARENT_CLAIM;
 use rustfs_iam::store::{MappedPolicy, UserType, sr_wire_user_type};
 use rustfs_iam::sys::SITE_REPLICATOR_SERVICE_ACCOUNT;
 use rustfs_madmin::{
-    AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus, SITE_REPL_API_VERSION,
-    SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq, SRResyncOpStatus,
-    SRRetryStats, SRStateInfo, SyncStatus,
+    AccountStatus, AddOrUpdateUserReq, GroupAddRemove, GroupStatus, PeerInfo, PeerSite, ReplicateEditStatus,
+    SITE_REPL_API_VERSION, SRBucketInfo, SRBucketMeta, SRGroupInfo, SRIAMItem, SRIAMPolicy, SRInfo, SRPolicyMapping, SRRemoveReq,
+    SRResyncOpStatus, SRRetryStats, SRSessionPolicy, SRStateInfo, SRSvcAccChange, SRSvcAccCreate, SRSvcAccDelete,
+    SRSvcAccReplicationEnvelope, SyncStatus,
 };
+use rustfs_policy::policy::Policy;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use rustfs_tls_runtime::{GlobalPublishedOutboundTlsState, TlsGeneration};
@@ -107,6 +110,26 @@ use tracing::{info, warn};
 use url::{Url, form_urlencoded};
 use uuid::Uuid;
 
+/// Serialize `value` with every JSON object's keys sorted, for hashing and
+/// equality checks. `HashMap` fields (service-account claims) iterate in a
+/// per-instance random order and `serde_json` is built with `preserve_order`,
+/// so two identical plans would otherwise hash differently: the repair
+/// preflight token went stale between dry-run and execute, and a retry
+/// snapshot resend never looked "stable" (backlog#2289 follow-up).
+pub(crate) fn canonical_json_vec<T: Serialize>(value: &T) -> serde_json::Result<Vec<u8>> {
+    fn sort_keys(value: Value) -> Value {
+        match value {
+            Value::Object(map) => {
+                let sorted: BTreeMap<String, Value> = map.into_iter().map(|(key, value)| (key, sort_keys(value))).collect();
+                Value::Object(sorted.into_iter().collect())
+            }
+            Value::Array(items) => Value::Array(items.into_iter().map(sort_keys).collect()),
+            other => other,
+        }
+    }
+    serde_json::to_vec(&sort_keys(serde_json::to_value(value)?))
+}
+
 pub(crate) const LOG_COMPONENT_ADMIN: &str = "admin";
 
 pub(crate) const LOG_SUBSYSTEM_SITE_REPLICATION: &str = "site_replication";

rustfs/src/site_replication/repair.rs

@@ -234,9 +234,9 @@ impl SiteReplicationRepairTask<'_> {
 
     pub(crate) fn id(&self) -> S3Result<String> {
         let payload = match self {
-            Self::Iam(item) => serde_json::to_vec(item),
+            Self::Iam(item) => canonical_json_vec(item),
             Self::BucketMake(_) | Self::Replication(_) => serde_json::to_vec(&serde_json::json!({})),
-            Self::BucketMetadata(item) => serde_json::to_vec(item),
+            Self::BucketMetadata(item) => canonical_json_vec(item),
         }
         .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize repair task failed: {err}")))?;
         let mut digest = Sha256::new();
@@ -726,7 +726,7 @@ pub(crate) async fn execute_site_replication_repair_locked(
         return Err(s3_error!(InvalidRequest, "site replication is not configured"));
     }
     let info = build_sr_info(&state, &request.local_peer).await?;
-    let plan = site_replication_bootstrap_plan(&info)?;
+    let plan = build_site_replication_bootstrap_plan(&info).await?;
     let plan_token = site_replication_repair_plan_token(&state, &plan)?;
     let preflight_token = site_replication_repair_preflight_token(&state, &plan, request.signing_key.as_bytes())?;
     let sites = site_replication_repair_sites(&state, &request.local_peer, &plan, request.signing_key.as_bytes())?;

rustfs/src/site_replication/retry.rs

@@ -397,12 +397,12 @@ pub(crate) fn iam_deletion_replay_matches(record: &SiteReplicationIamDeletionRep
 /// newer revision of one another.
 pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
     match item.r#type.as_str() {
-        "policy" if item.policy.is_none() => Some(format!("policy:{}", item.name)),
+        "policy" if item.policy.is_none() => Some(iam_policy_deletion_mark_entity(&item.name)),
         "iam-user" => item
             .iam_user
             .as_ref()
             .filter(|user| user.is_delete_req)
-            .map(|user| format!("iam-user:{}", user.access_key)),
+            .map(|user| iam_user_deletion_mark_entity(&user.access_key)),
         "group-info" => item
             .group_info
             .as_ref()
@@ -416,7 +416,7 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
             .policy_mapping
             .as_ref()
             .filter(|mapping| mapping.policy.is_empty())
-            .map(|mapping| format!("policy-mapping:{}:{}:{}", mapping.user_or_group, mapping.user_type, mapping.is_group)),
+            .map(|mapping| iam_policy_mapping_deletion_mark_entity(&mapping.user_or_group, mapping.user_type, mapping.is_group)),
         "service-account" => item
             .svc_acc_change
             .as_ref()
@@ -426,6 +426,82 @@ pub(crate) fn iam_item_deletion_entity(item: &SRIAMItem) -> Option<String> {
     }
 }
 
+/// The entities whose deletion a deletion-shaped IAM item commits, keyed the
+/// way the receive-side staleness gate looks them up once the local record is
+/// gone (backlog#2291); empty for creates and updates. Group member removal
+/// yields one entity per removed member so a stale re-add of that member can
+/// be judged, and a group delete (no members) yields the group itself.
+pub(crate) fn iam_item_deletion_mark_entities(item: &SRIAMItem) -> Vec<String> {
+    if item.r#type == "group-info" {
+        let Some(update) = item
+            .group_info
+            .as_ref()
+            .map(|group| &group.update_req)
+            .filter(|update| update.is_remove)
+        else {
+            return Vec::new();
+        };
+        if update.members.is_empty() {
+            return vec![iam_group_deletion_mark_entity(&update.group)];
+        }
+        return update
+            .members
+            .iter()
+            .map(|member| iam_group_member_deletion_mark_entity(&update.group, member))
+            .collect();
+    }
+    iam_item_deletion_entity(item).into_iter().collect()
+}
+
+pub(crate) fn iam_policy_deletion_mark_entity(name: &str) -> String {
+    format!("policy:{name}")
+}
+
+pub(crate) fn iam_user_deletion_mark_entity(access_key: &str) -> String {
+    format!("iam-user:{access_key}")
+}
+
+/// `user_type` is the SR wire integer, as carried by the item on both sides.
+pub(crate) fn iam_policy_mapping_deletion_mark_entity(user_or_group: &str, user_type: i64, is_group: bool) -> String {
+    format!("policy-mapping:{user_or_group}:{user_type}:{is_group}")
+}
+
+pub(crate) fn iam_group_deletion_mark_entity(group: &str) -> String {
+    format!("group:{group}")
+}
+
+pub(crate) fn iam_group_member_deletion_mark_entity(group: &str, member: &str) -> String {
+    format!("group-member:{group}:{member}")
+}
+
+/// Persist the deletion marks of `item` (its source `updated_at` per entity
+/// of [`iam_item_deletion_mark_entities`]) through the state transaction.
+/// No-op for creates/updates and for items without a source timestamp
+/// (older peers): a mark without a source clock could not be ordered against
+/// later items. Called before a local deletion is broadcast and after a
+/// replicated deletion is applied, so both sides out-rank a stale grant that
+/// arrives later.
+pub(crate) async fn record_iam_deletion_marks_for_item(item: &SRIAMItem) -> S3Result<()> {
+    let entities = iam_item_deletion_mark_entities(item);
+    let Some(deleted_at) = item.updated_at.filter(|_| !entities.is_empty()) else {
+        return Ok(());
+    };
+    commit_iam_deletion_marks(entities, deleted_at).await
+}
+
+/// [`record_iam_deletion_marks`] under the state transaction; the write is
+/// skipped when no mark moves.
+pub(crate) async fn commit_iam_deletion_marks(entities: Vec<String>, deleted_at: OffsetDateTime) -> S3Result<()> {
+    update_site_replication_state_when_changed(move |state| {
+        Ok(if record_iam_deletion_marks(state, &entities, deleted_at) {
+            StateCommit::Changed(())
+        } else {
+            StateCommit::Unchanged(())
+        })
+    })
+    .await
+}
+
 /// Failure bookkeeping for one IAM item delivery: upsert the collapsed retry
 /// event and, when the item is a deletion, record its body for replay. Both
 /// live in the same state so the caller commits them in one transaction — a
@@ -791,8 +867,8 @@ impl RetrySnapshot {
 
     pub(crate) fn fingerprint(&self) -> S3Result<Vec<Vec<u8>>> {
         let mut payloads = match self {
-            Self::Iam(items) => items.iter().map(serde_json::to_vec).collect::<Result<Vec<_>, _>>(),
-            Self::BucketMetadata(items) => items.iter().map(serde_json::to_vec).collect::<Result<Vec<_>, _>>(),
+            Self::Iam(items) => items.iter().map(canonical_json_vec).collect::<Result<Vec<_>, _>>(),
+            Self::BucketMetadata(items) => items.iter().map(canonical_json_vec).collect::<Result<Vec<_>, _>>(),
         }
         .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize retry snapshot failed: {err}")))?;
         payloads.sort_unstable();
@@ -954,6 +1030,7 @@ pub(crate) enum IamSnapshotKey {
     User(String),
     Group(String),
     PolicyMapping { target: String, user_type: i64, is_group: bool },
+    ServiceAccount(String),
 }
 
 pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option<IamSnapshotKey> {
@@ -972,6 +1049,11 @@ pub(crate) fn iam_snapshot_key(item: &SRIAMItem) -> Option<IamSnapshotKey> {
             user_type: mapping.user_type,
             is_group: mapping.is_group,
         }),
+        "service-account" => item
+            .svc_acc_change
+            .as_ref()
+            .and_then(|change| change.create.as_ref())
+            .map(|create| IamSnapshotKey::ServiceAccount(create.access_key.clone())),
         _ => None,
     }
 }
@@ -1006,6 +1088,24 @@ pub(crate) fn iam_snapshot_tombstones(item: &SRIAMItem, observed_at: OffsetDateT
                 mapping.policy.clear();
             }
         }
+        "service-account" => {
+            let Some(access_key) = item
+                .svc_acc_change
+                .as_ref()
+                .and_then(|change| change.create.as_ref())
+                .map(|create| create.access_key.clone())
+            else {
+                return Vec::new();
+            };
+            tombstone.svc_acc_change = Some(SRSvcAccChange {
+                delete: Some(SRSvcAccDelete {
+                    access_key,
+                    api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                }),
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+                ..Default::default()
+            });
+        }
         _ => return Vec::new(),
     }
     vec![tombstone]
@@ -1701,7 +1801,7 @@ pub(crate) async fn drain_site_replication_retry_queue_locked(
     // tick and only when a snapshot resend is actually due.
     let plan = if needs_plan {
         let info = build_sr_info(&runtime.state, &runtime.local_peer).await?;
-        Some(site_replication_bootstrap_plan(&info)?)
+        Some(build_site_replication_bootstrap_plan(&info).await?)
     } else {
         None
     };
@@ -1841,7 +1941,7 @@ pub(crate) async fn drain_one_site_replication_retry_event(
                     }
                 }
                 let fresh_info = build_sr_info(&runtime.state, &runtime.local_peer).await?;
-                let fresh_plan = site_replication_bootstrap_plan(&fresh_info)?;
+                let fresh_plan = build_site_replication_bootstrap_plan(&fresh_info).await?;
                 let fresh_snapshot = RetrySnapshot::from_plan(&action, &fresh_plan).expect("snapshot action has a snapshot");
                 if fresh_snapshot.fingerprint()? == current_fingerprint {
                     if is_iam {

rustfs/src/site_replication/state.rs

@@ -64,6 +64,104 @@ pub(crate) struct SiteReplicationState {
     /// newer edit that already landed.
     #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub(crate) applied_edit_generations: BTreeMap<String, u64>,
+    /// Source timestamp of the newest IAM deletion committed on this site,
+    /// keyed by the deleted entity (`iam_item_deletion_mark_entities`). A
+    /// deletion leaves no local record to judge a later item against, so this
+    /// is what lets the receive-side staleness gate reject a grant that is
+    /// older than the revoke it would otherwise undo (backlog#2291). Marks
+    /// are kept for [`SITE_REPLICATION_IAM_DELETION_MARK_RETENTION`] and never
+    /// evicted by count: see that constant for why a count bound would open
+    /// exactly the window the marks exist to close.
+    #[serde(default, with = "rfc3339_map", skip_serializing_if = "BTreeMap::is_empty")]
+    pub(crate) iam_deletion_marks: BTreeMap<String, OffsetDateTime>,
+}
+
+/// How long an IAM deletion mark outlives the deletion it records.
+///
+/// A mark fences the delivery paths that can still carry an older grant for
+/// the deleted entity: a live delivery delayed in transit, the same grant
+/// arriving on a sibling node while the revoke is being applied, and a
+/// snapshot (bootstrap / repair / resend) built by a peer that has not yet
+/// received the deletion — which is bounded by this site's own retry queue
+/// towards that peer, whose backoff tops out at one day
+/// (`SITE_REPLICATION_RETRY_DRAIN_MAX_BACKOFF_SECS`). The retry drain itself
+/// never replays a stale grant: it resends snapshots of the current records
+/// and the recorded deletion bodies. Thirty days is an order of magnitude
+/// beyond every one of those windows. Marks are pruned by age only — a count
+/// bound would drop a mark that is still inside the delivery window as soon
+/// as enough newer deletions happen, letting the delayed grant re-create the
+/// entity, which is the very hole the marks close.
+pub(crate) const SITE_REPLICATION_IAM_DELETION_MARK_RETENTION: time::Duration = time::Duration::days(30);
+
+/// Record that deletions of `entities` with source timestamp `deleted_at`
+/// were committed here. Newest wins per entity: an older deletion never
+/// lowers a mark. Marks older than the retention are pruned in the same
+/// pass. Returns whether the state changed.
+pub(crate) fn record_iam_deletion_marks(
+    state: &mut SiteReplicationState,
+    entities: &[String],
+    deleted_at: OffsetDateTime,
+) -> bool {
+    record_iam_deletion_marks_at(state, entities, deleted_at, OffsetDateTime::now_utc())
+}
+
+/// [`record_iam_deletion_marks`] pruning against an explicit `now`.
+pub(crate) fn record_iam_deletion_marks_at(
+    state: &mut SiteReplicationState,
+    entities: &[String],
+    deleted_at: OffsetDateTime,
+    now: OffsetDateTime,
+) -> bool {
+    let mut changed = false;
+    for entity in entities {
+        if state
+            .iam_deletion_marks
+            .get(entity)
+            .is_some_and(|existing| *existing >= deleted_at)
+        {
+            continue;
+        }
+        state.iam_deletion_marks.insert(entity.clone(), deleted_at);
+        changed = true;
+    }
+    let expired_before = now - SITE_REPLICATION_IAM_DELETION_MARK_RETENTION;
+    let before = state.iam_deletion_marks.len();
+    state.iam_deletion_marks.retain(|_, deleted_at| *deleted_at >= expired_before);
+    changed || state.iam_deletion_marks.len() != before
+}
+
+/// Newest deletion mark among `entities`, or `None` when no deletion of any
+/// of them was recorded here. The receive-side staleness gate feeds this in
+/// as the local timestamp when the targeted record is absent.
+pub(crate) fn iam_deletion_mark(state: &SiteReplicationState, entities: &[String]) -> Option<OffsetDateTime> {
+    entities
+        .iter()
+        .filter_map(|entity| state.iam_deletion_marks.get(entity).copied())
+        .max()
+}
+
+/// RFC 3339 map values, matching the other timestamps in the state object
+/// (`time::serde::rfc3339` only applies to a single field).
+mod rfc3339_map {
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+    use std::collections::BTreeMap;
+    use time::OffsetDateTime;
+
+    #[derive(Serialize, Deserialize)]
+    #[serde(transparent)]
+    struct Stamp(#[serde(with = "time::serde::rfc3339")] OffsetDateTime);
+
+    pub(super) fn serialize<S: Serializer>(map: &BTreeMap<String, OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.collect_map(map.iter().map(|(entity, deleted_at)| (entity, Stamp(*deleted_at))))
+    }
+
+    pub(super) fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result<BTreeMap<String, OffsetDateTime>, D::Error> {
+        let map = BTreeMap::<String, Stamp>::deserialize(deserializer)?;
+        Ok(map
+            .into_iter()
+            .map(|(entity, Stamp(deleted_at))| (entity, deleted_at))
+            .collect())
+    }
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -323,6 +421,33 @@ where
     update_site_replication_state_when_changed(move |state| update(state).map(StateCommit::Changed)).await
 }
 
+/// The state transaction for work that has to await inside it: an IAM write
+/// that must be ordered with the staleness verdict taken before it and the
+/// deletion mark committed after it (backlog#2291). Same boundary as
+/// [`update_site_replication_state`] — load and persist under the
+/// distributed state-object write lock, so two nodes of this site cannot
+/// interleave their verdicts and writes — and the same rules inside: no peer
+/// network calls and no other config locks. The closure hands the state back
+/// as `Some` when it changed it; `None` skips the write.
+pub(crate) async fn with_site_replication_state_transaction<T, F, Fut>(transaction: F) -> S3Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce(SiteReplicationState) -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = S3Result<(T, Option<SiteReplicationState>)>> + Send + 'static,
+{
+    with_site_replication_state_lock(move || async move {
+        let store = current_object_store_handle()
+            .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
+        let state = load_site_replication_state_no_lock(store.clone()).await?;
+        let (result, changed) = transaction(state).await?;
+        if let Some(state) = changed {
+            persist_site_replication_state_no_lock(store, state).await?;
+        }
+        Ok(result)
+    })
+    .await
+}
+
 /// [`update_site_replication_state`] for closures that may find nothing to
 /// do — see [`StateCommit`].
 pub(crate) async fn update_site_replication_state_when_changed<T, F>(update: F) -> S3Result<T>

rustfs/src/site_replication/tests.rs

@@ -554,6 +554,145 @@ fn test_iam_item_deletion_entity_shapes() {
     assert!(iam_item_deletion_entity(&policy_set).is_none());
 }
 
+/// Deletion marks (backlog#2291) key on the same entities as the replay
+/// records, except that a group member removal is marked per member (so a
+/// stale re-add of one member can be judged) and a group delete marks the
+/// group itself. Creates and updates leave no mark.
+#[test]
+fn test_iam_item_deletion_mark_entities_shapes() {
+    assert_eq!(
+        iam_item_deletion_mark_entities(&user_delete_item("alice")),
+        vec!["iam-user:alice".to_string()]
+    );
+    assert_eq!(
+        iam_item_deletion_mark_entities(&policy_delete_item("readonly")),
+        vec!["policy:readonly".to_string()]
+    );
+
+    let mut group_remove = SRIAMItem {
+        r#type: "group-info".to_string(),
+        group_info: Some(SRGroupInfo {
+            update_req: GroupAddRemove {
+                group: "devs".to_string(),
+                members: vec!["bob".to_string(), "alice".to_string()],
+                status: GroupStatus::Enabled,
+                is_remove: true,
+            },
+            api_version: None,
+        }),
+        ..Default::default()
+    };
+    assert_eq!(
+        iam_item_deletion_mark_entities(&group_remove),
+        vec!["group-member:devs:bob".to_string(), "group-member:devs:alice".to_string()]
+    );
+    group_remove
+        .group_info
+        .as_mut()
+        .expect("group info")
+        .update_req
+        .members
+        .clear();
+    assert_eq!(
+        iam_item_deletion_mark_entities(&group_remove),
+        vec!["group:devs".to_string()],
+        "a removal without members deletes the group"
+    );
+    group_remove.group_info.as_mut().expect("group info").update_req.is_remove = false;
+    assert!(iam_item_deletion_mark_entities(&group_remove).is_empty());
+
+    let mapping_clear = SRIAMItem {
+        r#type: "policy-mapping".to_string(),
+        policy_mapping: Some(SRPolicyMapping {
+            user_or_group: "alice".to_string(),
+            user_type: 0,
+            is_group: false,
+            policy: String::new(),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    assert_eq!(
+        iam_item_deletion_mark_entities(&mapping_clear),
+        vec!["policy-mapping:alice:0:false".to_string()]
+    );
+
+    let mut user_create = user_delete_item("alice");
+    user_create.iam_user.as_mut().expect("iam user").is_delete_req = false;
+    assert!(iam_item_deletion_mark_entities(&user_create).is_empty());
+}
+
+/// Newest wins per entity, marks are pruned by age only (never by count: a
+/// count bound would drop a mark still inside the delivery window as soon as
+/// enough newer deletions happen), and the timestamps survive the state
+/// object as RFC 3339.
+#[test]
+fn test_record_iam_deletion_marks_newest_wins_and_expires_by_age_only() {
+    let at = |seconds: i64| OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds);
+    let now = at(1_000_000);
+    let mut state = SiteReplicationState::default();
+    let alice = vec!["iam-user:alice".to_string()];
+
+    assert!(record_iam_deletion_marks_at(&mut state, &alice, at(20), now));
+    assert!(
+        !record_iam_deletion_marks_at(&mut state, &alice, at(10), now),
+        "an older deletion does not move the mark"
+    );
+    assert!(
+        !record_iam_deletion_marks_at(&mut state, &alice, at(20), now),
+        "a replayed deletion is not a change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(20)));
+    assert!(record_iam_deletion_marks_at(&mut state, &alice, at(30), now));
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)));
+    assert_eq!(iam_deletion_mark(&state, &["iam-user:bob".to_string()]), None);
+    assert!(!record_iam_deletion_marks_at(&mut state, &[], at(40), now));
+
+    // Many newer deletions never evict an older mark that is still within the retention.
+    let members: Vec<String> = (0..4096).map(|index| format!("group-member:devs:user-{index:04}")).collect();
+    for (index, member) in members.iter().enumerate() {
+        record_iam_deletion_marks_at(&mut state, std::slice::from_ref(member), at(100 + index as i64), now);
+    }
+    assert_eq!(state.iam_deletion_marks.len(), members.len() + 1);
+    assert_eq!(iam_deletion_mark(&state, &alice), Some(at(30)), "no count-based eviction");
+
+    // Marks older than the retention are pruned, on the pass that records a
+    // newer one and on a pass that changes nothing else; younger ones stay.
+    let later = at(100) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION;
+    assert!(
+        record_iam_deletion_marks_at(&mut state, &["iam-user:carol".to_string()], at(200_000), later),
+        "pruning alone is a change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), None, "alice's mark aged out");
+    assert_eq!(
+        iam_deletion_mark(&state, &members[..1]),
+        Some(at(100)),
+        "a mark exactly at the retention edge stays, and so do the younger ones"
+    );
+    assert_eq!(state.iam_deletion_marks.len(), members.len() + 1);
+    assert_eq!(iam_deletion_mark(&state, &["iam-user:carol".to_string()]), Some(at(200_000)));
+    let mut state = SiteReplicationState::default();
+    record_iam_deletion_marks_at(&mut state, &alice, at(30), now);
+    let past_edge = at(30) + SITE_REPLICATION_IAM_DELETION_MARK_RETENTION + time::Duration::seconds(1);
+    assert!(
+        record_iam_deletion_marks_at(&mut state, &[], at(0), past_edge),
+        "a pass that only prunes reports the change"
+    );
+    assert_eq!(iam_deletion_mark(&state, &alice), None);
+    record_iam_deletion_marks_at(&mut state, &alice, at(30), now);
+
+    let json = serde_json::to_value(&state).expect("serialize state");
+    assert_eq!(json["iam_deletion_marks"]["iam-user:alice"], serde_json::json!("1970-01-01T00:00:30Z"));
+    let reloaded = parse_site_replication_state(&serde_json::to_vec(&state).expect("serialize state")).expect("parse state");
+    assert_eq!(reloaded.iam_deletion_marks, state.iam_deletion_marks);
+    assert!(
+        parse_site_replication_state(br#"{"name":"a","service_account_access_key":"","service_account_parent":"","peers":{},"updated_at":null,"resync_status":{}}"#)
+            .expect("state without marks")
+            .iam_deletion_marks
+            .is_empty()
+    );
+}
+
 /// A failed deletion delivery persists a replay record next to the collapsed
 /// retry entry; a fresh entry is stamped `deletions_recorded` so a later
 /// replay can settle it, and a repeated deletion of the same entity keeps the
@@ -1679,7 +1818,8 @@ fn test_site_replication_bootstrap_plan_includes_replayable_snapshot_items() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     assert_eq!(plan.iam_items.iter().map(|item| item.r#type.as_str()).collect::<Vec<_>>(), {
         vec!["policy", "iam-user", "group-info", "policy-mapping"]
@@ -1717,7 +1857,8 @@ fn test_site_replication_bootstrap_plan_skips_lifecycle_by_default() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     assert!(!plan.bucket_items.iter().any(|item| item.r#type == "lc-config"));
 }
@@ -1748,7 +1889,8 @@ fn test_site_replication_bootstrap_plan_emits_timestamped_lifecycle_delete() {
         },
     );
 
-    let plan = site_replication_bootstrap_plan(&info).expect("bootstrap plan should build");
+    let plan =
+        site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("bootstrap plan should build");
 
     let item = plan
         .bucket_items
@@ -1935,8 +2077,8 @@ fn test_site_replication_repair_preflight_token_is_deterministic_for_equal_state
         },
     );
 
-    let plan_a = site_replication_bootstrap_plan(&info).expect("first plan");
-    let plan_b = site_replication_bootstrap_plan(&info).expect("second plan");
+    let plan_a = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("first plan");
+    let plan_b = site_replication_bootstrap_plan(&info, &SiteReplicationIamCredentials::default()).expect("second plan");
     let token_a = site_replication_repair_preflight_token(&state, &plan_a, b"test-signing-key").expect("first token");
     let token_b = site_replication_repair_preflight_token(&state, &plan_b, b"test-signing-key").expect("second token");
 
@@ -3219,3 +3361,345 @@ fn test_reconcile_adds_missing_peer_rules_to_existing_config() {
     assert!(rule_ids.contains(&"site-repl-dep-b"));
     assert!(rule_ids.contains(&"site-repl-dep-c"));
 }
+
+/// backlog#2289: the IAM snapshot (retry resend, repair, site-add bootstrap)
+/// used to be built from `list_users`, whose `UserInfo` never carries a
+/// secret key, so the plan dropped every user and a status change or secret
+/// rotation committed while a peer was unreachable never reached it. The
+/// credentials now come from a separate store read; SRInfo stays secret-free.
+#[test]
+fn test_bootstrap_plan_carries_users_from_the_credential_snapshot() {
+    let mut info = SRInfo::default();
+    // Exactly what `list_users` builds: status, policy, updated_at — never secret_key.
+    info.user_info_map.insert(
+        "alice".to_string(),
+        rustfs_madmin::UserInfo {
+            status: rustfs_madmin::AccountStatus::Disabled,
+            policy_name: Some("readwrite".to_string()),
+            updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
+            ..Default::default()
+        },
+    );
+    info.user_info_map.insert(
+        "external-idp-user".to_string(),
+        rustfs_madmin::UserInfo {
+            status: rustfs_madmin::AccountStatus::Enabled,
+            ..Default::default()
+        },
+    );
+    let user_updated_at = OffsetDateTime::from_unix_timestamp(1_700_000_500).expect("timestamp");
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Disabled,
+            updated_at: Some(user_updated_at),
+        },
+    );
+
+    let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build");
+
+    let users: Vec<_> = plan.iam_items.iter().filter(|item| item.r#type == "iam-user").collect();
+    assert_eq!(users.len(), 1, "only the user with a credential travels: {:?}", plan.iam_items);
+    let alice = users[0].iam_user.as_ref().expect("iam user body");
+    assert_eq!(alice.access_key, "alice");
+    let req = alice.user_req.as_ref().expect("user request");
+    assert_eq!(req.secret_key, "alice-secret");
+    assert_eq!(req.status, rustfs_madmin::AccountStatus::Disabled);
+    assert_eq!(req.policy.as_deref(), Some("readwrite"));
+    // the user record's own axis, not the policy-mapping time list_users reports
+    assert_eq!(users[0].updated_at, Some(user_updated_at));
+}
+
+fn service_account_snapshot(access_key: &str, parent: &str, status: &str) -> SiteReplicationServiceAccountSnapshot {
+    SiteReplicationServiceAccountSnapshot {
+        create: rustfs_madmin::SRSvcAccCreate {
+            parent: parent.to_string(),
+            access_key: access_key.to_string(),
+            secret_key: format!("{access_key}-secret"),
+            groups: Vec::new(),
+            claims: HashMap::new(),
+            session_policy: SRSessionPolicy::default(),
+            status: status.to_string(),
+            name: String::new(),
+            description: String::new(),
+            expiration: None,
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        },
+        envelope: None,
+        updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_600).expect("timestamp")),
+    }
+}
+
+/// backlog#2289: service accounts were absent from every snapshot (the
+/// listing filters them). They now travel as the create item the live hook
+/// emits — after their parents — carrying secret and status.
+#[test]
+fn test_bootstrap_plan_emits_service_accounts_after_their_parents() {
+    let mut info = SRInfo::default();
+    info.user_info_map
+        .insert("alice".to_string(), rustfs_madmin::UserInfo::default());
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Enabled,
+            updated_at: None,
+        },
+    );
+    credentials
+        .service_accounts
+        .push(service_account_snapshot("alice-svc", "alice", "off"));
+
+    let plan = site_replication_bootstrap_plan(&info, &credentials).expect("bootstrap plan should build");
+
+    let types: Vec<_> = plan.iam_items.iter().map(|item| item.r#type.as_str()).collect();
+    assert_eq!(types, vec!["iam-user", "service-account"]);
+    let change = plan.iam_items[1].svc_acc_change.as_ref().expect("service account change");
+    let create = change.create.as_ref().expect("create body");
+    assert_eq!((create.access_key.as_str(), create.parent.as_str()), ("alice-svc", "alice"));
+    assert_eq!(create.secret_key, "alice-svc-secret");
+    assert_eq!(create.status, "off", "a disabled account must arrive disabled");
+    assert!(change.delete.is_none() && change.update.is_none());
+}
+
+/// A service account present in the previous snapshot but gone from the
+/// fresh one is replayed as an explicit delete, like the other IAM kinds.
+#[test]
+fn test_retry_snapshot_tombstones_removed_service_accounts() {
+    let observed_at = OffsetDateTime::from_unix_timestamp(1_700_001_000).expect("timestamp");
+    let mut info = SRInfo::default();
+    info.user_info_map
+        .insert("alice".to_string(), rustfs_madmin::UserInfo::default());
+    let mut credentials = SiteReplicationIamCredentials::default();
+    credentials.users.insert(
+        "alice".to_string(),
+        SiteReplicationUserCredential {
+            secret_key: "alice-secret".to_string(),
+            status: rustfs_madmin::AccountStatus::Enabled,
+            updated_at: None,
+        },
+    );
+    let mut with_account = credentials.clone();
+    with_account
+        .service_accounts
+        .push(service_account_snapshot("alice-svc", "alice", "on"));
+    let previous = site_replication_bootstrap_plan(&info, &with_account).expect("previous plan");
+    let fresh = site_replication_bootstrap_plan(&info, &credentials).expect("fresh plan");
+
+    let replay = RetrySnapshot::replay_after_change(
+        &RetrySnapshot::Iam(previous.iam_items),
+        &RetrySnapshot::Iam(fresh.iam_items),
+        observed_at,
+    );
+    let RetrySnapshot::Iam(items) = replay else {
+        panic!("IAM snapshot expected");
+    };
+    let tombstone = items
+        .iter()
+        .find(|item| item.r#type == "service-account")
+        .expect("service account tombstone");
+    let change = tombstone.svc_acc_change.as_ref().expect("change");
+    assert_eq!(change.delete.as_ref().map(|delete| delete.access_key.as_str()), Some("alice-svc"));
+    assert!(change.create.is_none());
+    assert_eq!(tombstone.updated_at, Some(observed_at));
+}
+
+/// Spawns a one-shot HTTP peer that answers 200 and flips the returned flag
+/// once a request head has arrived.
+async fn spawn_reached_probe_peer() -> (String, Arc<AtomicBool>, tokio::task::JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind healthy peer");
+    let endpoint = format!("http://{}", listener.local_addr().expect("healthy peer address"));
+    let reached = Arc::new(AtomicBool::new(false));
+    let reached_by_server = reached.clone();
+    let server = tokio::spawn(async move {
+        let Ok((mut stream, _)) = listener.accept().await else {
+            return;
+        };
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        loop {
+            let Ok(read) = stream.read(&mut buffer).await else {
+                return;
+            };
+            if read == 0 {
+                return;
+            }
+            request.extend_from_slice(&buffer[..read]);
+            if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                break;
+            }
+        }
+        reached_by_server.store(true, Ordering::SeqCst);
+        let _ = stream
+            .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok")
+            .await;
+    });
+    (endpoint, reached, server)
+}
+
+/// Three-peer runtime whose local peer is `local`; BTreeMap order visits the
+/// failing peer `b` before the healthy peer `c`.
+fn broadcast_runtime_with_failing_peer_before_healthy(failing_endpoint: &str, healthy_endpoint: &str) -> SiteReplicationRuntime {
+    let local_peer = PeerInfo {
+        deployment_id: "local".to_string(),
+        ..peer("local", "http://127.0.0.1:9")
+    };
+    let mut state = SiteReplicationState {
+        name: "local".to_string(),
+        service_account_access_key: "site-replicator-0".to_string(),
+        ..Default::default()
+    };
+    state.peers.insert("local".to_string(), local_peer.clone());
+    state.peers.insert(
+        "b".to_string(),
+        PeerInfo {
+            deployment_id: "b".to_string(),
+            ..peer("b", failing_endpoint)
+        },
+    );
+    state.peers.insert(
+        "c".to_string(),
+        PeerInfo {
+            deployment_id: "c".to_string(),
+            ..peer("c", healthy_endpoint)
+        },
+    );
+    SiteReplicationRuntime {
+        state,
+        local_peer,
+        service_account_secret_key: "site-replicator-secret".to_string(),
+    }
+}
+
+const BROADCAST_PROBE_DELETE_BUCKET_PATH: &str =
+    "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=delete-bucket";
+
+/// The generic JSON broadcast (bucket make/delete, bucket-meta hook, bucket
+/// ops) attempts every remote peer: a peer whose request fails must not stop
+/// delivery to the peers that follow it in deployment-id order, and the
+/// failure is still reported to the caller (backlog#2293).
+#[tokio::test]
+#[serial]
+async fn test_broadcast_json_reaches_healthy_peers_after_a_failed_peer() {
+    // Peer "b": nothing listens on the port, so the connect is refused.
+    let refused = TcpListener::bind("127.0.0.1:0").await.expect("bind refused-peer probe");
+    let refused_endpoint = format!("http://{}", refused.local_addr().expect("refused-peer address"));
+    drop(refused);
+
+    let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await;
+    let runtime = broadcast_runtime_with_failing_peer_before_healthy(&refused_endpoint, &healthy_endpoint);
+
+    let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async {
+        broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await
+    })
+    .await;
+
+    let err = result.expect_err("peer b refuses connections, the broadcast must report it");
+    assert!(
+        reached.load(Ordering::SeqCst),
+        "peer c never received the broadcast once peer b failed: {err}"
+    );
+    server.abort();
+}
+
+/// Same guarantee when the failing peer never gets a transport: an endpoint
+/// that `PeerTransport::for_runtime_peer` rejects must be skipped past (and
+/// reported), not abort the broadcast before the healthy peers (backlog#2293).
+#[tokio::test]
+#[serial]
+async fn test_broadcast_json_reaches_healthy_peers_after_a_peer_without_transport() {
+    // Peer "b": a scheme the peer connection validator refuses outright.
+    let forbidden_endpoint = "ftp://peer-b.example.com";
+
+    let (healthy_endpoint, reached, server) = spawn_reached_probe_peer().await;
+    let runtime = broadcast_runtime_with_failing_peer_before_healthy(forbidden_endpoint, &healthy_endpoint);
+
+    let result = temp_env::async_with_vars([(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], async {
+        broadcast_site_replication_json_with_runtime(&runtime, BROADCAST_PROBE_DELETE_BUCKET_PATH, &serde_json::json!({})).await
+    })
+    .await;
+
+    let err = result.expect_err("peer b has no usable transport, the broadcast must report it");
+    assert!(
+        err.to_string().contains("invalid persisted site replication peer"),
+        "the reported error must be peer b's transport failure: {err}"
+    );
+    assert!(
+        reached.load(Ordering::SeqCst),
+        "peer c never received the broadcast once peer b failed to get a transport: {err}"
+    );
+    server.abort();
+}
+
+fn service_account_item_with_claims(order: &[&str]) -> SRIAMItem {
+    let mut claims = HashMap::new();
+    for key in order {
+        claims.insert((*key).to_string(), serde_json::json!(format!("value-of-{key}")));
+    }
+    SRIAMItem {
+        r#type: "service-account".to_string(),
+        svc_acc_change: Some(SRSvcAccChange {
+            create: Some(rustfs_madmin::SRSvcAccCreate {
+                parent: "alice".to_string(),
+                access_key: "alice-svc".to_string(),
+                secret_key: "alice-svc-secret".to_string(),
+                groups: Vec::new(),
+                claims,
+                session_policy: SRSessionPolicy::default(),
+                status: "on".to_string(),
+                name: String::new(),
+                description: String::new(),
+                expiration: None,
+                api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            }),
+            api_version: Some(SITE_REPL_API_VERSION.to_string()),
+            ..Default::default()
+        }),
+        updated_at: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
+        api_version: Some(SITE_REPL_API_VERSION.to_string()),
+        ..Default::default()
+    }
+}
+
+/// The repair preflight token and the retry-snapshot fingerprint hash the
+/// serialized items. Service-account claims live in a `HashMap`, whose
+/// iteration order differs between instances, so the hash must not depend on
+/// it (the real-VM repair returned 412 "preflight is stale" between dry-run
+/// and execute once snapshots carried service accounts).
+#[test]
+fn test_repair_task_id_and_retry_fingerprint_ignore_claim_map_order() {
+    let forward = service_account_item_with_claims(&["accessKey", "exp", "parent", "sa-policy", "sub", "tenant"]);
+    let backward = service_account_item_with_claims(&["tenant", "sub", "sa-policy", "parent", "exp", "accessKey"]);
+
+    let canonical = canonical_json_vec(&forward).expect("canonical json");
+    let text = String::from_utf8(canonical).expect("utf8");
+    let positions: Vec<usize> = [
+        "\"accessKey\"",
+        "\"exp\"",
+        "\"parent\"",
+        "\"sa-policy\"",
+        "\"sub\"",
+        "\"tenant\"",
+    ]
+    .iter()
+    .map(|key| text.find(key).expect("claim key present"))
+    .collect();
+    assert!(
+        positions.windows(2).all(|pair| pair[0] < pair[1]),
+        "claim keys must serialize sorted: {text}"
+    );
+
+    assert_eq!(
+        SiteReplicationRepairTask::Iam(&forward).id().expect("id"),
+        SiteReplicationRepairTask::Iam(&backward).id().expect("id"),
+        "identical items must yield the same repair task id regardless of claim map order"
+    );
+    assert_eq!(
+        RetrySnapshot::Iam(vec![forward]).fingerprint().expect("fingerprint"),
+        RetrySnapshot::Iam(vec![backward]).fingerprint().expect("fingerprint"),
+        "identical snapshots must fingerprint equal regardless of claim map order"
+    );
+}

rustfs/src/site_replication/transport.rs

@@ -876,6 +876,14 @@ pub(crate) async fn broadcast_site_replication_json<T: Serialize>(path: &str, bo
     broadcast_site_replication_json_with_runtime(&runtime, path, body).await
 }
 
+/// PUT `body` to `path` on every remote peer of the runtime.
+///
+/// Every peer is attempted: one peer's failure — transport construction
+/// included — must not skip the peers that follow it in deployment-id order,
+/// or they silently miss the change with no retry record (backlog#2293). A
+/// success settles the peer/path's queued retry event, a failure enqueues one
+/// under the request `path` (so the drain classifies it as today), and the
+/// first error is returned once all peers were attempted.
 pub(crate) async fn broadcast_site_replication_json_with_runtime<T: Serialize>(
     runtime: &SiteReplicationRuntime,
     path: &str,
@@ -883,20 +891,30 @@ pub(crate) async fn broadcast_site_replication_json_with_runtime<T: Serialize>(
 ) -> S3Result<()> {
     let state = &runtime.state;
     let local_peer = &runtime.local_peer;
+    let mut first_error: Option<S3Error> = None;
 
     for peer in state.peers.values() {
         if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
             continue;
         }
 
-        let transport = PeerTransport::for_runtime_peer(peer).await?;
-        PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key)
-            .with_client(&transport.client)
-            .send_with_retry_event(peer, &runtime.service_account_secret_key, body)
-            .await?;
+        let sent = match PeerTransport::for_runtime_peer(peer).await {
+            Ok(transport) => PeerAdminRequest::put(&transport.connection, path, &state.service_account_access_key)
+                .with_client(&transport.client)
+                .send_with_retry_event(peer, &runtime.service_account_secret_key, body)
+                .await
+                .map(|_| ()),
+            Err(err) => {
+                enqueue_site_replication_retry_event(peer, path, &err).await;
+                Err(err)
+            }
+        };
+        if let Err(err) = sent {
+            first_error.get_or_insert(err);
+        }
     }
 
-    Ok(())
+    first_error.map_or(Ok(()), Err)
 }
 
 pub(crate) fn parse_endpoint_refresh_status(peer: &PeerInfo, body: &[u8]) -> S3Result<()> {

rustfs/src/storage/rpc/node_service.rs

@@ -3052,6 +3052,136 @@ mod tests {
         assert_eq!(err.code(), tonic::Code::InvalidArgument);
     }
 
+    fn heal_start_retry_fixture() -> (
+        Arc<HealManager>,
+        rustfs_heal_contracts::heal_channel::HealChannelRequest,
+        rustfs_protos::heal_control::RequestMetadata,
+    ) {
+        let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None));
+        let mut request = rustfs_heal_contracts::heal_channel::create_heal_request(
+            "bucket".to_string(),
+            Some("prefix".to_string()),
+            true,
+            None,
+        );
+        request.source = rustfs_heal_contracts::heal_channel::HealRequestSource::Admin;
+        request.recursive = Some(true);
+        let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).expect("fixture clock fits in i64");
+        let metadata = rustfs_protos::heal_control::RequestMetadata::new(*Uuid::new_v4().as_bytes(), now, now + 30_000, 7);
+        (manager, request, metadata)
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_exact_forced_envelope_returns_cached_admission() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let request_id = request.id.clone();
+        let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("valid forced start");
+        let lost_response =
+            execute_heal_control_envelope_with_manager(envelope.clone(), metadata.coordinator_epoch, Some(manager.clone()))
+                .await
+                .expect("first request is admitted before its response is lost");
+        assert_eq!(manager.operations_snapshot().await.queue_length, 1);
+
+        // The caller sees no first response, but retries the original envelope.
+        let replayed = execute_heal_control_envelope_with_manager(envelope, metadata.coordinator_epoch, Some(manager.clone()))
+            .await
+            .expect("an exact envelope replay must recover its receipt");
+        assert_eq!(replayed, lost_response);
+        assert_eq!(
+            manager.operations_snapshot().await.queue_length,
+            1,
+            "forceStart must not be executed twice"
+        );
+        let outcome = rustfs_protos::heal_control::decode_result(&replayed)
+            .and_then(|result| result.into_outcome(&request_id, metadata.coordinator_epoch))
+            .expect("matching canonical receipt");
+        assert!(matches!(outcome, rustfs_protos::heal_control::Outcome::Start {
+            task_id, admission: rustfs_protos::heal_control::Admission::Accepted,
+        } if task_id == request_id));
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_new_forced_request_is_a_distinct_start() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let first_id = request.id.clone();
+        let first = rustfs_protos::heal_control::Envelope::start(request.clone(), metadata).expect("first start");
+        let _lost_response = execute_heal_control_envelope_with_manager(first, metadata.coordinator_epoch, Some(manager.clone()))
+            .await
+            .expect("first admission");
+
+        // A fresh HTTP forceStart request intentionally requests another start.
+        let mut next_request = request;
+        next_request.id = Uuid::new_v4().to_string();
+        let next_id = next_request.id.clone();
+        let next_metadata = rustfs_protos::heal_control::RequestMetadata {
+            nonce: *Uuid::new_v4().as_bytes(),
+            ..metadata
+        };
+        let next = rustfs_protos::heal_control::Envelope::start(next_request, next_metadata).expect("new forced start");
+        let response = execute_heal_control_envelope_with_manager(next, metadata.coordinator_epoch, Some(manager.clone()))
+            .await
+            .expect("forceStart preserves its explicit admission semantics");
+        let outcome = rustfs_protos::heal_control::decode_result(&response)
+            .and_then(|result| result.into_outcome(&next_id, metadata.coordinator_epoch))
+            .expect("new receipt");
+        assert!(matches!(outcome, rustfs_protos::heal_control::Outcome::Start {
+            task_id, admission: rustfs_protos::heal_control::Admission::Accepted,
+        } if task_id == next_id && task_id != first_id));
+        assert_eq!(
+            manager.operations_snapshot().await.queue_length,
+            2,
+            "a caller must not treat a new forced request as an idempotent transport retry"
+        );
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_same_id_with_changed_envelope_conflicts_before_admission() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let original = rustfs_protos::heal_control::Envelope::start(request.clone(), metadata).expect("original start");
+        let receipt =
+            execute_heal_control_envelope_with_manager(original.clone(), metadata.coordinator_epoch, Some(manager.clone()))
+                .await
+                .expect("original admission");
+        let mut changed_options = request.clone();
+        changed_options.remove_corrupted = Some(true);
+        let changed_metadata = rustfs_protos::heal_control::RequestMetadata {
+            nonce: *Uuid::new_v4().as_bytes(),
+            ..metadata
+        };
+        for changed in [
+            rustfs_protos::heal_control::Envelope::start(changed_options, metadata).expect("changed options"),
+            rustfs_protos::heal_control::Envelope::start(request, changed_metadata).expect("changed nonce"),
+        ] {
+            let error = execute_heal_control_envelope_with_manager(changed, metadata.coordinator_epoch, Some(manager.clone()))
+                .await
+                .expect_err("one request ID cannot identify different envelope bytes");
+            assert_eq!(error.code(), tonic::Code::AlreadyExists);
+            assert_eq!(manager.operations_snapshot().await.queue_length, 1);
+        }
+        assert_eq!(
+            execute_heal_control_envelope_with_manager(original, metadata.coordinator_epoch, Some(manager))
+                .await
+                .expect("conflicts must preserve the original receipt"),
+            receipt
+        );
+    }
+
+    #[tokio::test]
+    async fn heal_start_retry_wrong_coordinator_epoch_cannot_admit_locally() {
+        let (manager, request, metadata) = heal_start_retry_fixture();
+        let request_id = request.id.clone();
+        let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("start envelope");
+        let error = execute_heal_control_envelope_with_manager(envelope, metadata.coordinator_epoch + 1, Some(manager.clone()))
+            .await
+            .expect_err("a different coordinator epoch cannot accept the request");
+        assert_eq!(error.code(), tonic::Code::FailedPrecondition);
+        assert_eq!(manager.operations_snapshot().await.queue_length, 0);
+        assert!(matches!(
+            manager.get_task_status(&request_id).await,
+            Err(rustfs_heal::Error::TaskNotFound { .. })
+        ));
+    }
+
     #[tokio::test]
     async fn heal_control_executor_preserves_canonical_token_and_drops_query_results() {
         let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None));