Merge branch 'main' into overtrue/fix/lifecycle-rule-validation

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -856,6 +856,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn scanner_dirty_usage_snapshot(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::ScannerDirtyUsageSnapshotRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::ScannerDirtyUsageSnapshotResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn background_heal_status(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,

crates/ecstore/src/api/mod.rs

@@ -253,6 +253,13 @@ pub mod bucket {
         };
     }
 
+    pub mod sealed_credentials {
+        pub use crate::bucket::sealed_credentials::{
+            CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError,
+            SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret,
+        };
+    }
+
     pub mod replication {
         pub use crate::bucket::replication::replication_pool::{
             DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
@@ -526,8 +533,8 @@ pub mod rpc {
     pub use crate::cluster::rpc::{
         AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
         PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-        ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
-        build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
+        ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX,
+        TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
         encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
         gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
         normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs

@@ -16,6 +16,7 @@ use super::runtime_boundary as runtime_sources;
 use crate::bucket::lifecycle::lifecycle;
 use crate::object_api::ObjectInfo;
 use crate::services::event_notification::{EventArgs, send_event};
+use crate::storage_api_contracts::object::{DeletedObject, ObjectToDelete};
 use rustfs_s3_types::EventName;
 use rustfs_scanner_metrics::metrics::IlmAction;
 
@@ -76,6 +77,60 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source:
     emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT);
 }
 
+/// Emit the lifecycle expiration event for one version removed by the batch
+/// `NewerNoncurrentVersions` expiry path.
+///
+/// That path never sent events, so a successful noncurrent-version expiry was
+/// invisible to notification subscribers even though the equivalent
+/// current-version path emits one (backlog#2202).
+pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) {
+    if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) {
+        emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
+    }
+}
+
+/// Decide which event a single batch entry earned, if any.
+///
+/// Only an entry that mutated something may be announced. "No error" is not
+/// enough, and neither is `found`: the disk layer skips an absent version and
+/// reports success (`delete_versions_internal` in `disk/local.rs` continues
+/// past `FileVersionNotFound`), so a batch entry for a version that was
+/// already gone comes back indistinguishable from a committed delete. The
+/// delete plan's own source lookup is the signal that survives that, and the
+/// lifecycle batch path always performs it because every target carries an
+/// exact version identity.
+fn noncurrent_expiration_event(
+    bucket: &str,
+    target: &ObjectToDelete,
+    deleted: &DeletedObject,
+    failed: bool,
+) -> Option<(EventName, ObjectInfo)> {
+    if failed || !deleted.found || deleted.source_missing {
+        return None;
+    }
+    // A version removed by explicit version id is a plain versioned delete
+    // even when that version is itself a delete marker; only a request that
+    // carried no version id can publish a new delete marker. This is the rule
+    // the S3 DeleteObjects path applies (issue #6745). `delete_object_versions`
+    // now refuses targets without an exact version identity, so the
+    // marker-creation shape is unreachable from that caller; the mapping stays
+    // here so a future caller cannot silently announce the wrong mutation.
+    let created_delete_marker = deleted.delete_marker && target.version_id.is_none();
+    let (event_name, version_id) = if created_delete_marker {
+        (EventName::LifecycleExpirationDeleteMarkerCreated, deleted.delete_marker_version_id)
+    } else {
+        (EventName::LifecycleExpirationDelete, deleted.version_id.or(target.version_id))
+    };
+    let object = ObjectInfo {
+        bucket: bucket.to_string(),
+        name: target.object_name.clone(),
+        version_id,
+        delete_marker: deleted.delete_marker,
+        ..Default::default()
+    };
+    Some((event_name, object))
+}
+
 fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) {
     send_event(EventArgs {
         event_name: event_name.to_string(),
@@ -113,6 +168,7 @@ fn non_transitioned_expiration_event_name(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use uuid::Uuid;
 
     #[test]
     fn transitioned_expiration_event_marks_delete_marker_creation() {
@@ -129,4 +185,87 @@ mod tests {
             EventName::LifecycleExpirationDelete
         );
     }
+
+    fn deleted_version(version_id: Uuid) -> DeletedObject {
+        DeletedObject {
+            object_name: "object".to_string(),
+            version_id: Some(version_id),
+            found: true,
+            ..Default::default()
+        }
+    }
+
+    fn target_version(version_id: Option<Uuid>) -> ObjectToDelete {
+        ObjectToDelete {
+            object_name: "object".to_string(),
+            version_id,
+            ..Default::default()
+        }
+    }
+
+    #[test]
+    fn noncurrent_expiration_emits_versioned_delete_with_exact_identity() {
+        let version_id = Uuid::new_v4();
+        let (event_name, object) =
+            noncurrent_expiration_event("bucket", &target_version(Some(version_id)), &deleted_version(version_id), false)
+                .expect("a committed delete must emit");
+        assert_eq!(event_name, EventName::LifecycleExpirationDelete);
+        assert_eq!(object.bucket, "bucket");
+        assert_eq!(object.name, "object");
+        assert_eq!(object.version_id, Some(version_id));
+    }
+
+    /// Removing a noncurrent version that happens to be a delete marker is a
+    /// plain versioned delete, not a delete-marker creation.
+    #[test]
+    fn noncurrent_expiration_of_a_delete_marker_version_is_a_plain_delete() {
+        let version_id = Uuid::new_v4();
+        let deleted = DeletedObject {
+            delete_marker: true,
+            ..deleted_version(version_id)
+        };
+        let (event_name, object) = noncurrent_expiration_event("bucket", &target_version(Some(version_id)), &deleted, false)
+            .expect("a committed delete must emit");
+        assert_eq!(event_name, EventName::LifecycleExpirationDelete);
+        assert_eq!(object.version_id, Some(version_id));
+    }
+
+    #[test]
+    fn noncurrent_expiration_reports_a_created_delete_marker() {
+        let marker_version_id = Uuid::new_v4();
+        let deleted = DeletedObject {
+            object_name: "object".to_string(),
+            delete_marker: true,
+            delete_marker_version_id: Some(marker_version_id),
+            found: true,
+            ..Default::default()
+        };
+        let (event_name, object) =
+            noncurrent_expiration_event("bucket", &target_version(None), &deleted, false).expect("a committed delete must emit");
+        assert_eq!(event_name, EventName::LifecycleExpirationDeleteMarkerCreated);
+        assert_eq!(object.version_id, Some(marker_version_id));
+    }
+
+    /// A batch mixes successes with failures and versions that were already
+    /// gone; only a real mutation may produce an event. A version that was
+    /// already gone comes back with no error and `found` set, so
+    /// `source_missing` is the signal that keeps it silent.
+    #[test]
+    fn noncurrent_expiration_skips_failed_and_missing_versions() {
+        let version_id = Uuid::new_v4();
+        let target = target_version(Some(version_id));
+        assert!(noncurrent_expiration_event("bucket", &target, &deleted_version(version_id), true).is_none());
+
+        let absent = DeletedObject {
+            source_missing: true,
+            ..deleted_version(version_id)
+        };
+        assert!(noncurrent_expiration_event("bucket", &target, &absent, false).is_none());
+
+        let not_found = DeletedObject {
+            found: false,
+            ..deleted_version(version_id)
+        };
+        assert!(noncurrent_expiration_event("bucket", &target, &not_found, false).is_none());
+    }
 }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -87,16 +87,12 @@ use rustfs_filemeta::{
 use rustfs_scanner_metrics::metrics::{
     IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
 };
-use rustfs_utils::{
-    get_env_i64, get_env_usize,
-    path::encode_dir_object,
-    string::{parse_bool, strings_has_prefix_fold},
-};
+use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
 use s3s::dto::{
     BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
     Timestamp,
 };
-use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
+use s3s::header::X_AMZ_RESTORE;
 use sha2::{Digest, Sha256};
 use std::any::Any;
 use std::collections::{BTreeMap, HashMap, HashSet};
@@ -165,7 +161,6 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
     reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
 )]
 pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
-pub const AMZ_ENCRYPTION_AES: &str = "AES256";
 #[allow(
     dead_code,
     reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -2777,7 +2772,8 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
         match recovery_result {
             Ok(stats) => {
                 let elapsed = started_at.elapsed();
-                schedule.record_success(&stats, elapsed);
+                schedule.record_success(&stats);
+                rustfs_io_metrics::record_stage_duration("lifecycle_free_version_recovery", elapsed.as_secs_f64() * 1000.0);
                 let (pending_tasks, active_tasks) = {
                     let state = expiry_state.read().await;
                     (state.pending_tasks(), state.stats.active_tasks())
@@ -2806,7 +2802,7 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
             }
             Err(err) => {
                 let elapsed = started_at.elapsed();
-                schedule.record_failure(elapsed);
+                schedule.record_failure();
                 rustfs_io_metrics::record_stage_duration(
                     "lifecycle_free_version_recovery_failed",
                     elapsed.as_secs_f64() * 1000.0,
@@ -2838,10 +2834,10 @@ async fn wait_for_tier_free_version_recovery(
     } else {
         schedule.next_delay
     };
-    let sleep_delay = next_delay.saturating_sub(schedule.previous_run_duration);
-    schedule.previous_run_duration = StdDuration::ZERO;
+    // Recovery delays are completion-relative. Discounting the previous run
+    // would let a page that took at least one interval restart with no cooldown.
     schedule.jitter_next_delay = false;
-    let sleep = tokio::time::sleep(sleep_delay);
+    let sleep = tokio::time::sleep(next_delay);
     tokio::pin!(sleep);
     let mut recovery_request_consumed = false;
 
@@ -2891,7 +2887,6 @@ struct TierFreeVersionRecoverySchedule {
     next_delay: StdDuration,
     idle_interval: StdDuration,
     failure_interval: StdDuration,
-    previous_run_duration: StdDuration,
     jitter_next_delay: bool,
     bucket_marker: Option<String>,
     object_marker: Option<String>,
@@ -2904,7 +2899,6 @@ impl Default for TierFreeVersionRecoverySchedule {
             next_delay: StdDuration::ZERO,
             idle_interval: TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL,
             failure_interval: TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL,
-            previous_run_duration: StdDuration::ZERO,
             jitter_next_delay: false,
             bucket_marker: None,
             object_marker: None,
@@ -2917,7 +2911,6 @@ impl TierFreeVersionRecoverySchedule {
     fn reset_idle_interval(&mut self) {
         self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
         self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
-        self.previous_run_duration = StdDuration::ZERO;
         self.jitter_next_delay = false;
     }
 
@@ -2928,18 +2921,15 @@ impl TierFreeVersionRecoverySchedule {
         self.reset_idle_interval();
     }
 
-    fn record_failure(&mut self, _run_duration: StdDuration) {
+    fn record_failure(&mut self) {
         self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
         self.next_delay = self.failure_interval;
         self.failure_interval =
             std::cmp::min(self.failure_interval.saturating_mul(2), TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL);
-        // Keep the full backoff even after a long failed run, so a run whose
-        // duration exceeds the interval cannot restart immediately.
-        self.previous_run_duration = StdDuration::ZERO;
         self.jitter_next_delay = false;
     }
 
-    fn record_success(&mut self, stats: &FreeVersionRecoveryStats, run_duration: StdDuration) {
+    fn record_success(&mut self, stats: &FreeVersionRecoveryStats) {
         self.failure_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
         if stats.enqueued > 0 || stats.failed > 0 {
             self.follow_up_sweep = true;
@@ -2949,7 +2939,6 @@ impl TierFreeVersionRecoverySchedule {
         self.object_marker = stats.next_object_marker.clone();
         if stats.truncated {
             self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
-            self.previous_run_duration = run_duration;
             self.jitter_next_delay = false;
             return;
         }
@@ -2960,13 +2949,11 @@ impl TierFreeVersionRecoverySchedule {
             self.follow_up_sweep = false;
             self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
             self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
-            self.previous_run_duration = run_duration;
             self.jitter_next_delay = false;
             return;
         }
 
         self.next_delay = self.idle_interval;
-        self.previous_run_duration = run_duration;
         self.jitter_next_delay = true;
         self.idle_interval = std::cmp::min(self.idle_interval.saturating_mul(2), TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL);
     }
@@ -4810,24 +4797,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation
     reader
 }
 
-pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
+/// Resolve the RestoreObject request options.
+///
+/// Returns the typed [`StorageError`]: flattening these into an opaque
+/// `io::Error` string erased the identity the S3 layer needs to answer
+/// InvalidArgument instead of a generic 500 (backlog#2205).
+pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, Error> {
     let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
     let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
     let vid = version_id.trim();
     if !vid.is_empty() && vid != NULL_VERSION_ID {
         if let Err(_err) = Uuid::parse_str(vid) {
-            return Err(std::io::Error::other(
-                StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
-            ));
+            return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()));
         }
         if !versioned && !version_suspended {
-            return Err(std::io::Error::other(
-                StorageError::InvalidArgument(
-                    bucket.to_string(),
-                    object.to_string(),
-                    format!("version-id specified {} but versioning is not enabled on {}", vid, bucket),
-                )
-                .to_string(),
+            return Err(StorageError::InvalidArgument(
+                bucket.to_string(),
+                object.to_string(),
+                format!("version-id specified {vid} but versioning is not enabled on {bucket}"),
             ));
         }
     }
@@ -4880,43 +4867,18 @@ pub async fn put_restore_opts(
     }
     meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/
 
-    if let Some(type_) = &rreq.type_
-        && type_.as_str() == RestoreRequestType::SELECT
+    // A SELECT restore must never reach the restore writer: the caller writes
+    // the retrieved bytes back to the source bucket/object, so building
+    // SELECT output options here produced a source overwrite carrying only
+    // the OutputLocation metadata instead of a write to `OutputLocation.S3`
+    // (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
+    // is the fail-closed backstop for any other caller.
+    if rreq
+        .type_
+        .as_ref()
+        .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
     {
-        let Some(s3) = select_restore_s3_location(rreq)? else {
-            return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests"));
-        };
-        if let Some(user_metadata) = s3.user_metadata.as_ref() {
-            for metadata in user_metadata {
-                let name = metadata
-                    .name
-                    .as_deref()
-                    .ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?;
-                let value = metadata.value.clone().unwrap_or_default();
-                if strings_has_prefix_fold(name, "x-amz-meta") {
-                    meta.insert(name.to_string(), value);
-                } else {
-                    meta.insert(format!("x-amz-meta-{name}"), value);
-                }
-            }
-        }
-        if let Some(tags) = &s3.tagging {
-            meta.insert(
-                AMZ_OBJECT_TAGGING.to_string(),
-                serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
-            );
-        }
-        if let Some(encryption) = &s3.encryption
-            && encryption.encryption_type.as_str() != ""
-        {
-            meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
-        }
-        return Ok(ObjectOptions {
-            versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
-            version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
-            user_defined: meta,
-            ..Default::default()
-        });
+        return Err(std::io::Error::other("SELECT restore requests are not supported"));
     }
     for (k, v) in oi.user_defined.iter() {
         meta.insert(k.to_string(), v.clone());
@@ -5623,8 +5585,9 @@ mod tests {
     use crate::bucket::lifecycle::replication_sink::{ReplicationStatusType, VersionPurgeStatusType};
     use crate::bucket::lifecycle::runtime_boundary as runtime_sources;
     use crate::bucket::lifecycle::tier_free_version_recovery::{
-        FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
-        set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook,
+        FreeVersionRecoveryStats, RecoveryWalkTestAction, RecoveryWorkBudget, list_tier_free_versions,
+        list_tier_free_versions_with_budget, recover_tier_free_versions_with_cancel, set_recovery_bucket_list_wait_hook,
+        set_recovery_walk_test_hook,
     };
     use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
     use crate::bucket::lifecycle::tier_sweeper::Jentry;
@@ -5712,7 +5675,7 @@ mod tests {
 
         assert_eq!(schedule.next_delay, StdDuration::ZERO);
         for expected in [60, 120, 240, 480, 600, 600, 600, 600] {
-            schedule.record_success(&idle, StdDuration::ZERO);
+            schedule.record_success(&idle);
             assert_eq!(schedule.next_delay, StdDuration::from_secs(expected));
         }
         assert_eq!(schedule.next_delay.as_secs() / TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL.as_secs(), 10);
@@ -5723,15 +5686,13 @@ mod tests {
         let mut schedule = TierFreeVersionRecoverySchedule::default();
 
         for expected in [60, 120, 240, 480, 600, 600] {
-            schedule.record_failure(StdDuration::from_secs(75));
+            schedule.record_failure();
             assert_eq!(schedule.next_delay, StdDuration::from_secs(expected));
-            assert_eq!(schedule.previous_run_duration, StdDuration::ZERO);
         }
 
-        schedule.record_success(&free_version_recovery_stats(0, 0, false), StdDuration::ZERO);
-        schedule.record_failure(StdDuration::from_secs(75));
+        schedule.record_success(&free_version_recovery_stats(0, 0, false));
+        schedule.record_failure();
         assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
-        assert_eq!(schedule.previous_run_duration, StdDuration::ZERO);
     }
 
     #[test]
@@ -5746,22 +5707,22 @@ mod tests {
     fn tier_free_version_recovery_pagination_preserves_full_sweep_backoff() {
         let idle = free_version_recovery_stats(0, 0, false);
         let mut schedule = TierFreeVersionRecoverySchedule::default();
-        schedule.record_success(&idle, StdDuration::ZERO);
-        schedule.record_success(&idle, StdDuration::ZERO);
+        schedule.record_success(&idle);
+        schedule.record_success(&idle);
         assert_eq!(schedule.next_delay, StdDuration::from_secs(120));
         assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
 
-        schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO);
+        schedule.record_success(&free_version_recovery_stats(0, 0, true));
         assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
         assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
         assert_eq!(schedule.bucket_marker.as_deref(), Some("bucket"));
         assert_eq!(schedule.object_marker.as_deref(), Some("object"));
 
-        schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO);
+        schedule.record_success(&free_version_recovery_stats(0, 0, true));
         assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
         assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
 
-        schedule.record_success(&idle, StdDuration::ZERO);
+        schedule.record_success(&idle);
         assert_eq!(schedule.next_delay, StdDuration::from_secs(240));
         assert_eq!(schedule.idle_interval, StdDuration::from_secs(480));
         assert!(schedule.bucket_marker.is_none());
@@ -5771,7 +5732,7 @@ mod tests {
     #[test]
     fn tier_free_version_recovery_wake_during_pagination_keeps_one_full_follow_up() {
         let mut schedule = TierFreeVersionRecoverySchedule::default();
-        schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO);
+        schedule.record_success(&free_version_recovery_stats(0, 0, true));
 
         schedule.request_retry();
         schedule.request_retry();
@@ -5779,7 +5740,7 @@ mod tests {
         assert_eq!(schedule.bucket_marker.as_deref(), Some("bucket"));
         assert_eq!(schedule.object_marker.as_deref(), Some("object"));
 
-        schedule.record_success(&free_version_recovery_stats(0, 0, false), StdDuration::ZERO);
+        schedule.record_success(&free_version_recovery_stats(0, 0, false));
         assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
         assert!(!schedule.follow_up_sweep);
         assert!(schedule.bucket_marker.is_none());
@@ -5794,22 +5755,22 @@ mod tests {
             free_version_recovery_stats(0, 1, true),
         ] {
             let mut schedule = TierFreeVersionRecoverySchedule::default();
-            schedule.record_success(&idle, StdDuration::ZERO);
-            schedule.record_success(&idle, StdDuration::ZERO);
+            schedule.record_success(&idle);
+            schedule.record_success(&idle);
             assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
 
-            schedule.record_success(&work, StdDuration::ZERO);
+            schedule.record_success(&work);
             assert!(schedule.follow_up_sweep);
             assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
             assert!(!schedule.jitter_next_delay);
 
-            schedule.record_success(&idle, StdDuration::ZERO);
+            schedule.record_success(&idle);
             assert!(!schedule.follow_up_sweep);
             assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
             assert_eq!(schedule.idle_interval, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
             assert!(!schedule.jitter_next_delay);
 
-            schedule.record_success(&idle, StdDuration::ZERO);
+            schedule.record_success(&idle);
             assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
             assert_eq!(schedule.idle_interval, StdDuration::from_secs(120));
         }
@@ -5823,17 +5784,17 @@ mod tests {
             free_version_recovery_stats(0, 1, false),
         ] {
             let mut schedule = TierFreeVersionRecoverySchedule::default();
-            schedule.record_success(&idle, StdDuration::ZERO);
-            schedule.record_success(&idle, StdDuration::ZERO);
+            schedule.record_success(&idle);
+            schedule.record_success(&idle);
             assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
 
-            schedule.record_success(&work, StdDuration::ZERO);
+            schedule.record_success(&work);
             assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
             assert_eq!(schedule.idle_interval, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
             assert!(!schedule.follow_up_sweep);
             assert!(!schedule.jitter_next_delay);
 
-            schedule.record_success(&idle, StdDuration::ZERO);
+            schedule.record_success(&idle);
             assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
             assert_eq!(schedule.idle_interval, StdDuration::from_secs(120));
         }
@@ -5972,7 +5933,7 @@ mod tests {
         );
     }
 
-    async fn tier_free_version_recovery_page_call_times(run_duration: StdDuration) -> Vec<StdDuration> {
+    async fn tier_free_version_recovery_page_call_times(run_durations: &[StdDuration]) -> Vec<StdDuration> {
         RECOVERY_JITTER_CALLS.store(0, Ordering::SeqCst);
         let cancel = CancellationToken::new();
         let state = ExpiryState::new();
@@ -5981,6 +5942,7 @@ mod tests {
         let recorded_call_times = Arc::clone(&call_times);
         let call_index = Arc::new(AtomicUsize::new(0));
         let recorded_call_index = Arc::clone(&call_index);
+        let recovery_run_durations = run_durations.to_vec();
         let loop_cancel = cancel.clone();
         let recovery_cancel = cancel.clone();
         let worker = tokio::spawn(async move {
@@ -5991,8 +5953,9 @@ mod tests {
                     .push(tokio::time::Instant::now().duration_since(started_at));
                 let index = recorded_call_index.fetch_add(1, Ordering::SeqCst);
                 let cancel = recovery_cancel.clone();
+                let run_duration = recovery_run_durations.get(index).copied();
                 async move {
-                    if index == 0 {
+                    if let Some(run_duration) = run_duration {
                         tokio::time::sleep(run_duration).await;
                         Ok(free_version_recovery_stats(0, 0, true))
                     } else {
@@ -6005,15 +5968,14 @@ mod tests {
         });
 
         tokio::task::yield_now().await;
-        tokio::time::advance(run_duration).await;
-        tokio::task::yield_now().await;
-        if run_duration < TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL {
-            let remaining = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL - run_duration;
-            tokio::time::advance(remaining - StdDuration::from_secs(1)).await;
-            assert_eq!(call_index.load(Ordering::SeqCst), 1);
+        for (index, run_duration) in run_durations.iter().copied().enumerate() {
+            tokio::time::advance(run_duration).await;
+            tokio::task::yield_now().await;
+            tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL - StdDuration::from_secs(1)).await;
+            assert_eq!(call_index.load(Ordering::SeqCst), index + 1);
             tokio::time::advance(StdDuration::from_secs(1)).await;
+            tokio::task::yield_now().await;
         }
-        tokio::task::yield_now().await;
         worker.await.expect("recovery loop should stop after cancellation");
         assert_eq!(
             RECOVERY_JITTER_CALLS.load(Ordering::SeqCst),
@@ -6029,14 +5991,75 @@ mod tests {
 
     #[tokio::test(start_paused = true)]
     #[serial]
-    async fn tier_free_version_recovery_preserves_start_to_start_page_cadence() {
+    async fn tier_free_version_recovery_waits_after_each_page_completes() {
         assert_eq!(
-            tier_free_version_recovery_page_call_times(StdDuration::from_secs(45)).await,
-            vec![StdDuration::ZERO, StdDuration::from_secs(60)]
+            tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(45)]).await,
+            vec![StdDuration::ZERO, StdDuration::from_secs(105)]
         );
         assert_eq!(
-            tier_free_version_recovery_page_call_times(StdDuration::from_secs(75)).await,
-            vec![StdDuration::ZERO, StdDuration::from_secs(75)]
+            tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(60)]).await,
+            vec![StdDuration::ZERO, StdDuration::from_secs(120)]
+        );
+        assert_eq!(
+            tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(75)]).await,
+            vec![StdDuration::ZERO, StdDuration::from_secs(135)]
+        );
+        assert_eq!(
+            tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(75), StdDuration::from_secs(45)]).await,
+            vec![StdDuration::ZERO, StdDuration::from_secs(135), StdDuration::from_secs(240)]
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn tier_free_version_recovery_notify_during_page_keeps_completion_cooldown() {
+        let cancel = CancellationToken::new();
+        let state = ExpiryState::new();
+        let recovery_notify = Arc::clone(&state.read().await.recovery_notify);
+        let started_at = tokio::time::Instant::now();
+        let call_times = Arc::new(StdMutex::new(Vec::new()));
+        let recorded_call_times = Arc::clone(&call_times);
+        let call_index = Arc::new(AtomicUsize::new(0));
+        let recorded_call_index = Arc::clone(&call_index);
+        let loop_cancel = cancel.clone();
+        let recovery_cancel = cancel.clone();
+        let worker = tokio::spawn(async move {
+            run_tier_free_version_recovery_loop(loop_cancel, state, std::convert::identity, move |_, _, _| {
+                recorded_call_times
+                    .lock()
+                    .expect("recovery call times lock should not be poisoned")
+                    .push(tokio::time::Instant::now().duration_since(started_at));
+                let index = recorded_call_index.fetch_add(1, Ordering::SeqCst);
+                let cancel = recovery_cancel.clone();
+                async move {
+                    if index == 0 {
+                        tokio::time::sleep(StdDuration::from_secs(75)).await;
+                        Ok(free_version_recovery_stats(0, 0, true))
+                    } else {
+                        cancel.cancel();
+                        Ok(free_version_recovery_stats(0, 0, false))
+                    }
+                }
+            })
+            .await;
+        });
+
+        tokio::task::yield_now().await;
+        tokio::time::advance(StdDuration::from_secs(30)).await;
+        recovery_notify.notify_one();
+        tokio::time::advance(StdDuration::from_secs(45)).await;
+        tokio::task::yield_now().await;
+        tokio::time::advance(StdDuration::from_secs(59)).await;
+        assert_eq!(call_index.load(Ordering::SeqCst), 1);
+        tokio::time::advance(StdDuration::from_secs(1)).await;
+        tokio::task::yield_now().await;
+        worker.await.expect("recovery loop should stop after cancellation");
+
+        assert_eq!(
+            call_times
+                .lock()
+                .expect("recovery call times lock should not be poisoned")
+                .as_slice(),
+            &[StdDuration::ZERO, StdDuration::from_secs(135)]
         );
     }
 
@@ -11283,6 +11306,120 @@ mod tests {
         }
     }
 
+    /// backlog#2202: the batch `NewerNoncurrentVersions` path used to delete
+    /// noncurrent versions without telling notification subscribers anything,
+    /// while the current-version path emitted a lifecycle expiration event.
+    /// Only versions this batch actually removed may produce an event.
+    #[tokio::test]
+    #[serial]
+    async fn lifecycle_noncurrent_batch_expiry_emits_events_only_for_committed_deletes() {
+        use crate::services::event_notification::test_recorder;
+        use rustfs_s3_types::EventName;
+
+        let (_disk_paths, ecstore) = setup_test_env().await;
+        let bucket = format!("lifecycle-noncurrent-events-{}", Uuid::new_v4().simple());
+        let object = "logs/object";
+        create_test_bucket(&ecstore, &bucket).await;
+
+        metadata_sys::update_in(
+            &ecstore.ctx,
+            &bucket,
+            BUCKET_VERSIONING_CONFIG,
+            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
+        )
+        .await
+        .expect("bucket versioning should be enabled");
+
+        let now = OffsetDateTime::now_utc();
+        let mut noncurrent_reader = PutObjReader::from_vec(b"noncurrent".to_vec());
+        let noncurrent = ecstore
+            .put_object(
+                &bucket,
+                object,
+                &mut noncurrent_reader,
+                &ObjectOptions {
+                    versioned: true,
+                    mod_time: Some(now - time::Duration::days(40)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the noncurrent version should be created");
+        let noncurrent_version_id = noncurrent.version_id.expect("a versioned PUT has an exact identity");
+        let mut current_reader = PutObjReader::from_vec(b"current".to_vec());
+        let current = ecstore
+            .put_object(
+                &bucket,
+                object,
+                &mut current_reader,
+                &ObjectOptions {
+                    versioned: true,
+                    mod_time: Some(now - time::Duration::days(2)),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the current version should be created");
+        let current_version_id = current.version_id.expect("a versioned PUT has an exact identity");
+
+        let incarnation = ecstore
+            .bucket_incarnation_id_from_disk(&bucket)
+            .await
+            .expect("bucket incarnation should be available");
+
+        test_recorder::install();
+
+        // One version that exists and one that never did: `delete_objects`
+        // suppresses the not-found error, so only the committed delete may be
+        // announced.
+        let missing_version_id = Uuid::new_v4();
+        let targets = vec![
+            ObjectToDelete {
+                object_name: object.to_string(),
+                version_id: Some(noncurrent_version_id),
+                ..Default::default()
+            },
+            ObjectToDelete {
+                object_name: object.to_string(),
+                version_id: Some(missing_version_id),
+                ..Default::default()
+            },
+        ];
+        let failed = crate::bucket::lifecycle::object_handlers_common::delete_object_versions(
+            &ecstore,
+            &bucket,
+            &targets,
+            lifecycle::Event::default(),
+            incarnation,
+        )
+        .await;
+        assert_eq!(failed, 0, "a missing version is not a batch failure");
+
+        let events = test_recorder::recorded_for_bucket(&bucket);
+        let announced = events.iter().map(|event| event.version_id).collect::<Vec<_>>();
+        assert_eq!(
+            announced,
+            vec![Some(noncurrent_version_id)],
+            "only the committed noncurrent delete should be announced \
+             (noncurrent={noncurrent_version_id}, current={current_version_id}, missing={missing_version_id}), got {events:?}"
+        );
+        assert_eq!(events[0].event_name, EventName::LifecycleExpirationDelete.to_string());
+        assert_eq!(events[0].object, object);
+        assert!(!events[0].delete_marker);
+
+        let remaining = ecstore
+            .clone()
+            .list_object_versions(&bucket, object, None, None, None, 10)
+            .await
+            .expect("remaining versions should be listable")
+            .objects
+            .into_iter()
+            .filter(|candidate| candidate.name == object)
+            .collect::<Vec<_>>();
+        assert_eq!(remaining.len(), 1);
+        assert_eq!(remaining[0].version_id, Some(current_version_id));
+    }
+
     #[tokio::test]
     #[serial]
     async fn lifecycle_deletes_only_the_historical_null_version_after_versioning_is_reenabled() {
@@ -12557,6 +12694,181 @@ mod tests {
             .expect("empty recovery test bucket should be removed");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn tier_free_version_recovery_object_budget_is_global_across_buckets() {
+        let (_disk_paths, ecstore) = setup_test_env().await;
+        for budget in [
+            RecoveryWorkBudget {
+                max_objects: 0,
+                max_buckets: 1,
+            },
+            RecoveryWorkBudget {
+                max_objects: 1,
+                max_buckets: 0,
+            },
+        ] {
+            let err = list_tier_free_versions_with_budget(Arc::clone(&ecstore), 1, None, None, CancellationToken::new(), budget)
+                .await
+                .expect_err("a zero recovery work budget must fail before starting an unbounded walk");
+            assert!(matches!(err, Error::Io(_)));
+        }
+
+        let suffix = Uuid::new_v4().simple();
+        let buckets = [
+            format!("zzzz-recovery-budget-{suffix}-a"),
+            format!("zzzz-recovery-budget-{suffix}-b"),
+            format!("zzzz-recovery-budget-{suffix}-c"),
+        ];
+        for bucket in &buckets {
+            create_test_bucket(&ecstore, bucket).await;
+        }
+
+        let first_bucket = buckets[0].clone();
+        let first_bucket_for_hook = first_bucket.clone();
+        let _walk = set_recovery_walk_test_hook(move |bucket| {
+            (bucket == first_bucket_for_hook).then(|| {
+                RecoveryWalkTestAction::SendItems(
+                    ["ordinary-object-0", "ordinary-object-0", "ordinary-object-1"]
+                        .into_iter()
+                        .map(|name| ObjectInfo {
+                            bucket: first_bucket_for_hook.clone(),
+                            name: name.to_string(),
+                            ..Default::default()
+                        })
+                        .collect(),
+                )
+            })
+        });
+        let page = list_tier_free_versions_with_budget(
+            Arc::clone(&ecstore),
+            1,
+            None,
+            None,
+            CancellationToken::new(),
+            RecoveryWorkBudget {
+                max_objects: 1,
+                max_buckets: 10,
+            },
+        )
+        .await
+        .expect("the decoded-object boundary should enforce the recovery budget");
+
+        assert!(page.items.is_empty());
+        assert_eq!(page.scanned_entries, 3);
+        assert_eq!(page.buckets_scanned, 1);
+        assert!(page.truncated);
+        assert_eq!(page.next_bucket_marker.as_deref(), Some(first_bucket.as_str()));
+        assert_eq!(page.next_object_marker.as_deref(), Some("ordinary-object-0"));
+        drop(_walk);
+
+        let walked = Arc::new(StdMutex::new(Vec::new()));
+        let walked_by_hook = Arc::clone(&walked);
+        let test_buckets = buckets.clone();
+        let _walk = set_recovery_walk_test_hook(move |bucket| {
+            test_buckets
+                .iter()
+                .find(|candidate| candidate.as_str() == bucket)
+                .map(|bucket| {
+                    walked_by_hook
+                        .lock()
+                        .expect("recovery walk log should not be poisoned")
+                        .push(bucket.clone());
+                    RecoveryWalkTestAction::SendItems(vec![ObjectInfo {
+                        bucket: bucket.clone(),
+                        name: "ordinary-object".to_string(),
+                        ..Default::default()
+                    }])
+                })
+        });
+
+        let page = list_tier_free_versions_with_budget(
+            Arc::clone(&ecstore),
+            1,
+            None,
+            None,
+            CancellationToken::new(),
+            RecoveryWorkBudget {
+                max_objects: 2,
+                max_buckets: 10,
+            },
+        )
+        .await
+        .expect("the bounded recovery page should be listed");
+
+        assert!(page.items.is_empty());
+        assert_eq!(page.scanned_entries, 2);
+        assert_eq!(page.buckets_scanned, 2);
+        assert!(page.truncated);
+        assert_eq!(page.next_bucket_marker.as_deref(), Some(buckets[1].as_str()));
+        assert_eq!(page.next_object_marker.as_deref(), Some("ordinary-object"));
+        assert_eq!(walked.lock().expect("recovery walk log should not be poisoned").as_slice(), &buckets[..2]);
+
+        for bucket in &buckets {
+            ecstore
+                .delete_bucket(bucket, &DeleteBucketOptions::default())
+                .await
+                .expect("empty recovery test bucket should be removed");
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn tier_free_version_recovery_bucket_budget_resumes_at_unscanned_bucket() {
+        let (_disk_paths, ecstore) = setup_test_env().await;
+        let suffix = Uuid::new_v4().simple();
+        let buckets = [
+            format!("zzzz-recovery-buckets-{suffix}-a"),
+            format!("zzzz-recovery-buckets-{suffix}-b"),
+        ];
+        for bucket in &buckets {
+            create_test_bucket(&ecstore, bucket).await;
+        }
+
+        let test_buckets = buckets.clone();
+        let _walk = set_recovery_walk_test_hook(move |bucket| {
+            test_buckets
+                .iter()
+                .any(|candidate| candidate.as_str() == bucket)
+                .then(|| RecoveryWalkTestAction::SendItems(Vec::new()))
+        });
+        let budget = RecoveryWorkBudget {
+            max_objects: 10,
+            max_buckets: 1,
+        };
+        let first = list_tier_free_versions_with_budget(Arc::clone(&ecstore), 1, None, None, CancellationToken::new(), budget)
+            .await
+            .expect("the first bucket-bounded page should be listed");
+
+        assert_eq!(first.buckets_scanned, 1);
+        assert!(first.truncated);
+        assert_eq!(first.next_bucket_marker.as_deref(), Some(buckets[1].as_str()));
+        assert!(first.next_object_marker.is_none());
+
+        let second = list_tier_free_versions_with_budget(
+            Arc::clone(&ecstore),
+            1,
+            first.next_bucket_marker,
+            first.next_object_marker,
+            CancellationToken::new(),
+            budget,
+        )
+        .await
+        .expect("the second bucket-bounded page should resume");
+
+        assert_eq!(second.buckets_scanned, 1);
+        assert!(!second.truncated);
+        assert!(second.next_bucket_marker.is_none());
+        assert!(second.next_object_marker.is_none());
+
+        for bucket in &buckets {
+            ecstore
+                .delete_bucket(bucket, &DeleteBucketOptions::default())
+                .await
+                .expect("empty recovery test bucket should be removed");
+        }
+    }
+
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_propagates_walk_item_error() {

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -182,6 +182,16 @@ pub(crate) enum DurableIlmRecordCheckpoint {
         identity_sha256: String,
         state: tier_delete_journal::TierDeleteDispatchManifestState,
     },
+    TierDeleteDispatchParent {
+        content_sha256: String,
+        identity_sha256: String,
+        revision: u64,
+        next_chunk_sequence: u64,
+        completed_journal_count: u64,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        active_chunk_identity_sha256: Option<String>,
+        completed: bool,
+    },
     TransitionTransaction {
         content_sha256: String,
         identity_sha256: String,
@@ -220,6 +230,7 @@ impl DurableIlmRecordCheckpoint {
         match self {
             Self::TierDeleteJournal { content_sha256, .. }
             | Self::TierDeleteDispatchManifest { content_sha256, .. }
+            | Self::TierDeleteDispatchParent { content_sha256, .. }
             | Self::TransitionTransaction { content_sha256, .. }
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
@@ -341,6 +352,52 @@ impl DurableIlmRecordCheckpoint {
                         (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
                     )
             }
+            (
+                Self::TierDeleteDispatchParent {
+                    identity_sha256: previous_identity,
+                    revision: previous_revision,
+                    next_chunk_sequence: previous_sequence,
+                    completed_journal_count: previous_completed_journals,
+                    active_chunk_identity_sha256: previous_active,
+                    completed: previous_completed,
+                    ..
+                },
+                Self::TierDeleteDispatchParent {
+                    identity_sha256: next_identity,
+                    revision: next_revision,
+                    next_chunk_sequence: next_sequence,
+                    completed_journal_count: next_completed_journals,
+                    active_chunk_identity_sha256: next_active,
+                    completed: next_completed,
+                    ..
+                },
+            ) => {
+                let Some((sequence_delta, completed_journal_delta)) = tier_delete_dispatch_parent_progress_delta(
+                    *previous_sequence,
+                    *previous_completed_journals,
+                    *next_sequence,
+                    *next_completed_journals,
+                ) else {
+                    return Err(Error::other("durable ILM record generation is not a monotonic successor"));
+                };
+                let same_position_transition = sequence_delta == 0
+                    && completed_journal_delta == 0
+                    && matches!(
+                        (previous_active.as_ref(), next_active.as_ref(), previous_completed, next_completed),
+                        (None, Some(_), false, false) | (Some(_), None, false, false) | (None, None, false, true)
+                    );
+                let progress_transition = sequence_delta > 0
+                    && completed_journal_delta > 0
+                    && !matches!(
+                        (previous_active.as_ref(), next_active.as_ref()),
+                        (Some(previous), Some(next)) if previous == next
+                    );
+                previous_identity == next_identity
+                    && !previous_completed
+                    && next_revision > previous_revision
+                    && (!next_completed || next_active.is_none())
+                    && (same_position_transition || progress_transition)
+            }
             (
                 Self::TransitionTransaction {
                     identity_sha256: previous_identity,
@@ -475,11 +532,58 @@ impl DurableIlmRecordCheckpoint {
                     ..
                 },
             ) => previous_identity == terminal_identity,
+            (
+                Self::TierDeleteDispatchParent {
+                    identity_sha256: previous_identity,
+                    revision: previous_revision,
+                    next_chunk_sequence: previous_sequence,
+                    completed_journal_count: previous_completed_journals,
+                    active_chunk_identity_sha256: previous_active,
+                    completed: false,
+                    ..
+                },
+                Self::TierDeleteDispatchParent {
+                    identity_sha256: terminal_identity,
+                    revision: terminal_revision,
+                    next_chunk_sequence: terminal_sequence,
+                    completed_journal_count: terminal_completed_journals,
+                    active_chunk_identity_sha256: None,
+                    completed: true,
+                    ..
+                },
+            ) => {
+                previous_identity == terminal_identity
+                    && terminal_revision > previous_revision
+                    && tier_delete_dispatch_parent_progress_delta(
+                        *previous_sequence,
+                        *previous_completed_journals,
+                        *terminal_sequence,
+                        *terminal_completed_journals,
+                    )
+                    .is_some_and(|(sequence_delta, completed_journal_delta)| {
+                        if sequence_delta == 0 && completed_journal_delta == 0 {
+                            previous_active.is_none()
+                        } else {
+                            sequence_delta > 0 && completed_journal_delta > 0
+                        }
+                    })
+            }
             _ => false,
         }
     }
 }
 
+fn tier_delete_dispatch_parent_progress_delta(
+    previous_sequence: u64,
+    previous_completed_journals: u64,
+    next_sequence: u64,
+    next_completed_journals: u64,
+) -> Option<(u64, u64)> {
+    let sequence_delta = next_sequence.checked_sub(previous_sequence)?;
+    let completed_journal_delta = next_completed_journals.checked_sub(previous_completed_journals)?;
+    (sequence_delta <= completed_journal_delta).then_some((sequence_delta, completed_journal_delta))
+}
+
 fn transition_state_distance(
     from: transition_transaction::TransitionTransactionState,
     to: transition_transaction::TransitionTransactionState,
@@ -913,17 +1017,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
             )
         }
         DurableIlmRecordKind::TierDeleteDispatchManifest => {
-            let (operation_id, identity_sha256, state) =
-                tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
-            (
-                "operation_id",
-                hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
-                DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
-                    content_sha256,
+            match tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)? {
+                tier_delete_journal::TierDeleteDispatchDurableRecord::Manifest {
+                    operation_id,
                     identity_sha256,
                     state,
-                },
-            )
+                } => (
+                    "operation_id",
+                    hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
+                    DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
+                        content_sha256,
+                        identity_sha256,
+                        state,
+                    },
+                ),
+                tier_delete_journal::TierDeleteDispatchDurableRecord::Parent {
+                    operation_id,
+                    identity_sha256,
+                    revision,
+                    next_chunk_sequence,
+                    completed_journal_count,
+                    active_chunk_identity_sha256,
+                    completed,
+                } => (
+                    "operation_id",
+                    hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
+                    DurableIlmRecordCheckpoint::TierDeleteDispatchParent {
+                        content_sha256,
+                        identity_sha256,
+                        revision,
+                        next_chunk_sequence,
+                        completed_journal_count,
+                        active_chunk_identity_sha256,
+                        completed,
+                    },
+                ),
+            }
         }
         DurableIlmRecordKind::TransitionTransaction => {
             let transaction = transition_transaction::decode_transition_transaction_record(path, data)
@@ -1143,6 +1272,85 @@ mod tests {
         assert!(aborted.validate_successor(&preparing).is_err());
     }
 
+    #[test]
+    fn tier_delete_dispatch_parent_checkpoint_is_monotonic_across_chunks() {
+        let identity = "a".repeat(64);
+        let checkpoint = |revision, sequence, completed_journals, active: Option<&str>, completed| {
+            DurableIlmRecordCheckpoint::TierDeleteDispatchParent {
+                content_sha256: format!("{revision:064x}"),
+                identity_sha256: identity.clone(),
+                revision,
+                next_chunk_sequence: sequence,
+                completed_journal_count: completed_journals,
+                active_chunk_identity_sha256: active.map(ToOwned::to_owned),
+                completed,
+            }
+        };
+        let idle = checkpoint(0, 0, 0, None, false);
+        let first_child = "b".repeat(64);
+        let second_child = "c".repeat(64);
+        let bound = checkpoint(1, 0, 0, Some(&first_child), false);
+        let advanced = checkpoint(2, 1, 2, None, false);
+        let next_bound = checkpoint(3, 1, 2, Some(&second_child), false);
+        let completed = checkpoint(4, 2, 3, None, true);
+        let terminal_after_more_chunks = checkpoint(6, 4, 7, None, true);
+
+        idle.validate_successor(&bound).expect("an idle parent may bind one child");
+        bound
+            .validate_successor(&advanced)
+            .expect("a completed child may advance the parent sequence");
+        advanced
+            .validate_successor(&next_bound)
+            .expect("the next sequence may bind a new immutable child");
+        next_bound
+            .validate_successor(&completed)
+            .expect("receipt progress may skip directly to a later terminal checkpoint");
+        assert!(
+            bound.is_predecessor_of_terminal(&terminal_after_more_chunks),
+            "terminal cleanup may still recognize a valid multi-chunk predecessor"
+        );
+        assert!(
+            advanced.is_predecessor_of_terminal(&terminal_after_more_chunks),
+            "terminal cleanup may still skip over later valid parent generations"
+        );
+        assert!(
+            idle.validate_successor(&checkpoint(1, 0, 1, Some(&first_child), false))
+                .is_err()
+        );
+        assert!(bound.validate_successor(&checkpoint(2, 1, 0, None, false)).is_err());
+        assert!(
+            bound.validate_successor(&checkpoint(2, 2, 1, None, false)).is_err(),
+            "sequence cannot advance beyond completed journal evidence"
+        );
+        assert!(
+            advanced.validate_successor(&checkpoint(3, 1, 3, None, false)).is_err(),
+            "completed journal count cannot grow without a completed child sequence"
+        );
+        assert!(
+            bound.validate_successor(&checkpoint(2, 0, 0, None, true)).is_err(),
+            "an active child cannot be marked completed without completion evidence"
+        );
+        assert!(
+            bound
+                .validate_successor(&checkpoint(2, 0, 0, Some(&second_child), false))
+                .is_err(),
+            "an active child cannot be replaced at the same parent sequence"
+        );
+        assert!(
+            bound
+                .validate_successor(&checkpoint(2, 1, 1, Some(&first_child), false))
+                .is_err(),
+            "sequence growth cannot retain the same active child identity"
+        );
+        assert!(
+            !bound.is_predecessor_of_terminal(&checkpoint(2, 0, 0, None, true)),
+            "terminal cleanup must not treat an active child as completed without count evidence"
+        );
+        assert!(completed.validate_successor(&checkpoint(5, 3, 4, None, true)).is_err());
+        assert!(completed.validate_successor(&advanced).is_err());
+        assert!(advanced.validate_successor(&idle).is_err());
+    }
+
     #[test]
     fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
         use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};

crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs

@@ -20,6 +20,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
 const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped";
 const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed";
 
+use crate::bucket::lifecycle::bucket_lifecycle_audit::emit_noncurrent_expiration_event;
 use crate::bucket::lifecycle::lifecycle;
 use crate::bucket::lifecycle::replication_sink::{self, ReplicationObjectBridge};
 use crate::object_api::ObjectOptions;
@@ -98,6 +99,12 @@ pub async fn delete_object_versions(
             // version so it does not sit resident until TTL (ODC-26).
             if let Some(target) = to_del.get(i) {
                 crate::object_api::notify_object_mutation(bucket, &target.object_name).await;
+                // Announce the version this batch actually removed. Cache
+                // eviction and replication scheduling keep their existing
+                // order and admission; the event is derived from the committed
+                // result, and a send failure never rolls back a delete that
+                // already happened (backlog#2202).
+                emit_noncurrent_expiration_event(bucket, target, deleted_obj, false);
             }
             if deleted_obj.replication_state.is_none() {
                 continue;

crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs

@@ -32,7 +32,10 @@ use crate::store::ECStore;
 use rustfs_filemeta::FileInfo;
 
 pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
+// These are page-wide repair budgets. Applying them per bucket would still let
+// one recovery pass walk an unbounded namespace before the scheduler can cool down.
 const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
+const DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT: usize = 100;
 #[cfg(not(test))]
 const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
 #[cfg(test)]
@@ -41,6 +44,12 @@ const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100);
 type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
 type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct RecoveryWorkBudget {
+    pub(super) max_objects: usize,
+    pub(super) max_buckets: usize,
+}
+
 fn recovery_walk_options(limit: usize, marker: Option<String>) -> WalkOptions {
     WalkOptions {
         include_free_versions: true,
@@ -58,6 +67,7 @@ fn recovery_walk_options(limit: usize, marker: Option<String>) -> WalkOptions {
 
 #[cfg(test)]
 pub(super) enum RecoveryWalkTestAction {
+    SendItems(Vec<ObjectInfo>),
     SendItemsThenError(Vec<ObjectInfo>, crate::error::Error),
     SendItemsThenHang(Vec<ObjectInfo>, Arc<tokio::sync::Notify>),
     SendItemsUntilReceiverCloses(Arc<tokio::sync::Notify>),
@@ -277,6 +287,17 @@ pub(super) async fn list_tier_free_versions(
     bucket_marker: Option<String>,
     object_marker: Option<String>,
     cancel_token: CancellationToken,
+) -> Result<FreeVersionRecoveryPage> {
+    list_tier_free_versions_with_budget(api, limit, bucket_marker, object_marker, cancel_token, recovery_work_budget(limit)).await
+}
+
+pub(super) async fn list_tier_free_versions_with_budget(
+    api: Arc<ECStore>,
+    limit: usize,
+    bucket_marker: Option<String>,
+    object_marker: Option<String>,
+    cancel_token: CancellationToken,
+    work_budget: RecoveryWorkBudget,
 ) -> Result<FreeVersionRecoveryPage> {
     let mut page = FreeVersionRecoveryPage {
         items: Vec::new(),
@@ -290,6 +311,9 @@ pub(super) async fn list_tier_free_versions(
     if limit == 0 {
         return Ok(page);
     }
+    if work_budget.max_objects == 0 || work_budget.max_buckets == 0 {
+        return Err(std::io::Error::other("free-version recovery work budget must be greater than zero").into());
+    }
 
     let bucket_options = BucketOptions::default();
     let list_buckets = async {
@@ -313,7 +337,7 @@ pub(super) async fn list_tier_free_versions(
     };
     let mut bucket_seen = bucket_marker.is_none();
     let mut truncated_after: Option<RecoveryCursor> = None;
-    let walk_scan_limit = recovery_walk_scan_limit(limit);
+    let mut remaining_scan_objects = work_budget.max_objects;
 
     for bucket in buckets {
         if cancel_token.is_cancelled() {
@@ -329,12 +353,20 @@ pub(super) async fn list_tier_free_versions(
             bucket_seen = true;
         }
 
+        if page.buckets_scanned >= work_budget.max_buckets {
+            page.truncated = true;
+            page.next_bucket_marker = Some(bucket.name);
+            page.next_object_marker = None;
+            break;
+        }
+
         page.buckets_scanned += 1;
         let bucket_object_marker = if bucket_marker.as_deref() == Some(bucket.name.as_str()) {
             object_marker.clone()
         } else {
             None
         };
+        let bucket_walk_limit = remaining_scan_objects;
 
         let (tx, mut rx) = mpsc::channel::<ObjectInfoOrErr>(100);
         let cancel = cancel_token.child_token();
@@ -358,6 +390,21 @@ pub(super) async fn list_tier_free_versions(
                 #[cfg(test)]
                 if let Some(action) = test_action {
                     match action {
+                        RecoveryWalkTestAction::SendItems(items) => {
+                            for item in items {
+                                if tx
+                                    .send(ObjectInfoOrErr {
+                                        item: Some(item),
+                                        err: None,
+                                    })
+                                    .await
+                                    .is_err()
+                                {
+                                    return Ok(());
+                                }
+                            }
+                            return Ok(());
+                        }
                         RecoveryWalkTestAction::SendItemsThenError(items, err) => {
                             for item in items {
                                 if tx
@@ -425,7 +472,7 @@ pub(super) async fn list_tier_free_versions(
                     }
                 }
 
-                api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(walk_scan_limit, object_marker))
+                api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(bucket_walk_limit, object_marker))
                     .await
             }
         });
@@ -466,6 +513,18 @@ pub(super) async fn list_tier_free_versions(
             let Some(oi) = item.item else {
                 continue;
             };
+            if last_seen_object.as_deref() != Some(oi.name.as_str()) && scanned_objects >= bucket_walk_limit {
+                // The disk listing limit follows S3-visible counting rules and
+                // does not charge metadata containing only hidden/free versions.
+                // Enforce the repair budget again at the decoded-object boundary.
+                page.truncated = true;
+                page.next_bucket_marker = Some(bucket.name.clone());
+                page.next_object_marker = last_seen_object.clone();
+                cancel.cancel();
+                draining_after_truncation = true;
+                drain_deadline = Some(tokio::time::Instant::now() + BACKGROUND_WALK_SHUTDOWN_TIMEOUT);
+                continue;
+            }
             record_scanned_object(&mut last_seen_object, &mut scanned_objects, &oi.name);
             if let Some(cursor) = &truncated_after
                 && (cursor.bucket.as_str() != bucket.name.as_str() || cursor.object.as_str() != oi.name.as_str())
@@ -509,7 +568,8 @@ pub(super) async fn list_tier_free_versions(
             return Err(err);
         }
         walk_result?;
-        mark_scan_truncated_if_needed(&mut page, scanned_objects, walk_scan_limit, &bucket.name, last_seen_object.as_deref());
+        remaining_scan_objects = remaining_scan_objects.saturating_sub(scanned_objects);
+        mark_scan_truncated_if_needed(&mut page, scanned_objects, bucket_walk_limit, &bucket.name, last_seen_object.as_deref());
 
         if page.truncated {
             break;
@@ -528,6 +588,13 @@ fn recovery_walk_scan_limit(limit: usize) -> usize {
     DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT.max(limit.saturating_add(1))
 }
 
+fn recovery_work_budget(limit: usize) -> RecoveryWorkBudget {
+    RecoveryWorkBudget {
+        max_objects: recovery_walk_scan_limit(limit),
+        max_buckets: DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT,
+    }
+}
+
 fn record_scanned_object(last_seen_object: &mut Option<String>, scanned_objects: &mut usize, object: &str) {
     if last_seen_object.as_deref() == Some(object) {
         return;
@@ -700,6 +767,13 @@ mod tests {
             recovery_walk_scan_limit(DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT),
             DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT + 1
         );
+        assert_eq!(
+            recovery_work_budget(DEFAULT_FREE_VERSION_RECOVERY_LIMIT),
+            RecoveryWorkBudget {
+                max_objects: DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT,
+                max_buckets: DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT,
+            }
+        );
     }
 
     #[test]

crates/ecstore/src/bucket/mod.rs

@@ -31,6 +31,7 @@ pub mod policy_sys;
 pub mod quota;
 pub mod remote_s3_client;
 pub mod replication;
+pub mod sealed_credentials;
 pub mod tagging;
 pub mod target;
 pub mod utils;

crates/ecstore/src/bucket/on_demand_migration/source_client.rs

@@ -14,9 +14,9 @@
 
 //! Outbound client for an on-demand migration source bucket.
 //!
-//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared
-//! remote builder and exposes the read-only surface the migration path
-//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
+//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The
+//! S3 backend uses the shared remote builder and exposes the surface the
+//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
 //! probe for admin validation). Every request carries the
 //! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and
 //! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of
@@ -511,7 +511,7 @@ pub struct SourceObject {
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct SourcePage {
     pub objects: Vec<SourceObject>,
-    /// Rolled-up prefixes, in the local namespace; always empty when the
+    /// Rolled-up prefixes, in the same namespace as `objects`; always empty when the
     /// request carried no delimiter.
     pub common_prefixes: Vec<String>,
     pub is_truncated: bool,
@@ -519,7 +519,8 @@ pub struct SourcePage {
 }
 
 /// One `ListObjectsV2` page request against the source. Keys are given in the
-/// local namespace; `SourceClient` maps them through `source_prefix`.
+/// local namespace at `SourceClient`, and in the source namespace at
+/// `SourceBackend`; `SourceClient` maps them through `source_prefix`.
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct SourceListRequest<'a> {
     pub prefix: Option<&'a str>,
@@ -575,8 +576,30 @@ impl Intercept for SourceProxyMarkerInterceptor {
     }
 }
 
-pub struct SourceClient {
+/// Read-only provider operations in the source bucket namespace.
+///
+/// Implementations must preserve streaming, honor the requested range and
+/// pagination cursor, and classify failures without including credentials.
+/// `SourceClient` owns prefix mapping so every provider shares the same local
+/// namespace. Continuation tokens are opaque and must never be prefix-mapped.
+#[async_trait::async_trait]
+pub trait SourceBackend: Send + Sync {
+    async fn head(&self, key: &str) -> Result<SourceHead, SourceError>;
+    async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError>;
+    async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError>;
+    async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError>;
+    /// Verify bucket access; `SourceClient` separately probes a filtered listing.
+    async fn probe(&self) -> Result<(), SourceError>;
+}
+
+/// S3-compatible implementation, including request signing and anti-loop headers.
+pub struct S3SourceBackend {
     client: S3Client,
+    bucket: String,
+}
+
+pub struct SourceClient {
+    backend: Box<dyn SourceBackend>,
     endpoint: String,
     bucket: String,
     source_prefix: Option<String>,
@@ -609,7 +632,10 @@ impl SourceClient {
     fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
         let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
         Self {
-            client,
+            backend: Box::new(S3SourceBackend {
+                client,
+                bucket: spec.bucket.clone(),
+            }),
             endpoint,
             bucket: spec.bucket.clone(),
             source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
@@ -652,35 +678,15 @@ impl SourceClient {
     }
 
     pub async fn head_object(&self, key: &str) -> Result<SourceHead, SourceError> {
-        let output = self
-            .client
-            .head_object()
-            .bucket(&self.bucket)
-            .key(self.source_key(key))
-            .send()
-            .await
-            .map_err(classify_sdk_error)?;
-        source_head_from_head_output(output)
+        self.backend.head(&self.source_key(key)).await
     }
 
-    /// Streams the object; `range` is passed through as an HTTP `Range`
-    /// header and omitted entirely when `None`.
+    /// Streams the object, preserving an optional HTTP byte range.
     pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
-        let range = range.map(range_header_value).transpose()?;
-        let output = self
-            .client
-            .get_object()
-            .bucket(&self.bucket)
-            .key(self.source_key(key))
-            .set_range(range)
-            .send()
-            .await
-            .map_err(classify_sdk_error)?;
-        source_get_from_output(output)
+        self.backend.get(&self.source_key(key), range).await
     }
 
-    /// Lists one page under the local `prefix`. Keys are returned in the
-    /// local namespace; entries outside `source_prefix` are skipped.
+    /// Lists one page under the local prefix.
     pub async fn list_objects_v2(
         &self,
         prefix: Option<&str>,
@@ -696,9 +702,81 @@ impl SourceClient {
         .await
     }
 
-    /// [`Self::list_objects_v2`] with the delimiter and start-after the
-    /// list-through merge needs (rustfs/backlog#2164).
+    /// Maps keys and common prefixes while leaving opaque cursors untouched.
     pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
+        let prefix = self.source_key(request.prefix.unwrap_or_default());
+        let start_after = request.start_after.map(|key| self.source_key(key));
+        let mut page = self
+            .backend
+            .list(&SourceListRequest {
+                prefix: Some(&prefix),
+                start_after: start_after.as_deref(),
+                ..*request
+            })
+            .await?;
+        page.objects = page
+            .objects
+            .into_iter()
+            .filter_map(|object| self.local_object(object))
+            .collect();
+        page.common_prefixes = page
+            .common_prefixes
+            .into_iter()
+            .filter_map(|prefix| self.local_key(&prefix).map(str::to_string))
+            .collect();
+        Ok(page)
+    }
+
+    fn local_object(&self, mut object: SourceObject) -> Option<SourceObject> {
+        object.key = self.local_key(&object.key)?.to_string();
+        Some(object)
+    }
+
+    pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
+        self.backend.tagging(&self.source_key(key)).await
+    }
+
+    pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
+        self.backend.probe().await?;
+        let page = self.list_objects_v2(None, None, 1).await?;
+        Ok(SourceProbe {
+            sample_object: page.objects.into_iter().next(),
+            has_more_objects: page.is_truncated,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl SourceBackend for S3SourceBackend {
+    async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
+        let output = self
+            .client
+            .head_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .send()
+            .await
+            .map_err(classify_sdk_error)?;
+        source_head_from_head_output(output)
+    }
+
+    /// Streams the object; `range` is passed through as an HTTP `Range`
+    /// header and omitted entirely when `None`.
+    async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
+        let range = range.map(range_header_value).transpose()?;
+        let output = self
+            .client
+            .get_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .set_range(range)
+            .send()
+            .await
+            .map_err(classify_sdk_error)?;
+        source_get_from_output(output)
+    }
+
+    async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
         // `start_after` is silently ignored by S3 once a continuation token is
         // present; refuse the ambiguous pair rather than list from the wrong
         // position.
@@ -711,9 +789,9 @@ impl SourceClient {
             .client
             .list_objects_v2()
             .bucket(&self.bucket)
-            .prefix(self.source_key(request.prefix.unwrap_or_default()))
+            .prefix(request.prefix.unwrap_or_default())
             .set_delimiter(request.delimiter.map(str::to_string))
-            .set_start_after(request.start_after.map(|after| self.source_key(after)))
+            .set_start_after(request.start_after.map(str::to_string))
             .set_continuation_token(request.continuation_token.map(str::to_string))
             .max_keys(request.max_keys)
             .send()
@@ -731,13 +809,13 @@ impl SourceClient {
             .contents
             .unwrap_or_default()
             .into_iter()
-            .filter_map(|object| self.source_object(object))
+            .filter_map(s3_source_object)
             .collect();
         let common_prefixes = output
             .common_prefixes
             .unwrap_or_default()
             .into_iter()
-            .filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
+            .filter_map(|prefix| prefix.prefix)
             .collect();
 
         Ok(SourcePage {
@@ -748,48 +826,43 @@ impl SourceClient {
         })
     }
 
-    fn source_object(&self, object: SdkObject) -> Option<SourceObject> {
-        let key = self.local_key(object.key.as_deref()?)?.to_string();
-        let etag = normalize_etag(object.e_tag);
-        let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
-        Some(SourceObject {
-            key,
-            etag,
-            size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
-            last_modified: system_time(object.last_modified),
-            storage_class: object.storage_class.map(|class| class.as_str().to_string()),
-            is_multipart_etag,
-        })
-    }
-
-    pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
+    async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
         let output = self
             .client
             .get_object_tagging()
             .bucket(&self.bucket)
-            .key(self.source_key(key))
+            .key(key)
             .send()
             .await
             .map_err(classify_sdk_error)?;
         Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect())
     }
 
-    /// Admin validation: HeadBucket plus a one-key listing under the prefix.
-    pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
+    async fn probe(&self) -> Result<(), SourceError> {
         self.client
             .head_bucket()
             .bucket(&self.bucket)
             .send()
             .await
             .map_err(classify_sdk_error)?;
-        let page = self.list_objects_v2(None, None, 1).await?;
-        Ok(SourceProbe {
-            sample_object: page.objects.into_iter().next(),
-            has_more_objects: page.is_truncated,
-        })
+        Ok(())
     }
 }
 
+fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
+    let key = object.key?;
+    let etag = normalize_etag(object.e_tag);
+    let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
+    Some(SourceObject {
+        key,
+        etag,
+        size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
+        last_modified: system_time(object.last_modified),
+        storage_class: object.storage_class.map(|class| class.as_str().to_string()),
+        is_multipart_etag,
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1194,6 +1267,68 @@ mod tests {
         assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri);
     }
 
+    #[tokio::test]
+    async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() {
+        let body = r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<IsTruncated>true</IsTruncated><NextContinuationToken>data/opaque</NextContinuationToken>
+<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
+<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
+</ListBucketResult>"#;
+        let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
+        let first = client
+            .list_page(&SourceListRequest {
+                prefix: Some("photos/"),
+                delimiter: Some("/"),
+                start_after: Some("photos/a"),
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("delimiter listing should succeed");
+        assert_eq!(first.common_prefixes, vec!["photos/"]);
+        assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque"));
+        let second = client
+            .list_page(&SourceListRequest {
+                continuation_token: first.next_continuation_token.as_deref(),
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("opaque continuation should succeed");
+        assert_eq!(second.common_prefixes, first.common_prefixes);
+        let requests = recorded(&requests);
+        let query = |request: &RecordedRequest| {
+            Url::parse(&request.uri)
+                .expect("request URI")
+                .query_pairs()
+                .into_owned()
+                .collect::<HashMap<_, _>>()
+        };
+        let first_query = query(&requests[0]);
+        assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/"));
+        assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a"));
+        assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/"));
+        let second_query = query(&requests[1]);
+        assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque"));
+        assert!(!second_query.contains_key("start-after"));
+    }
+
+    #[tokio::test]
+    async fn list_page_rejects_ambiguous_cursor_before_sending() {
+        let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await;
+        let err = client
+            .list_page(&SourceListRequest {
+                start_after: Some("a"),
+                continuation_token: Some("opaque"),
+                max_keys: 1,
+                ..Default::default()
+            })
+            .await
+            .expect_err("ambiguous list position must fail");
+        assert!(matches!(err, SourceError::Other(_)));
+        assert!(recorded(&requests).is_empty(), "invalid request must never reach the source");
+    }
+
     #[tokio::test]
     async fn list_objects_v2_rejects_truncated_page_without_token() {
         let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await;
@@ -1357,11 +1492,14 @@ mod tests {
 
     fn prefix_client(prefix: Option<String>) -> SourceClient {
         SourceClient {
-            client: S3Client::from_conf(
-                aws_sdk_s3::Config::builder()
-                    .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-                    .build(),
-            ),
+            backend: Box::new(S3SourceBackend {
+                client: S3Client::from_conf(
+                    aws_sdk_s3::Config::builder()
+                        .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+                        .build(),
+                ),
+                bucket: "bucket".to_string(),
+            }),
             endpoint: "https://source.example.com".to_string(),
             bucket: "bucket".to_string(),
             source_prefix: prefix.filter(|prefix| !prefix.is_empty()),

crates/ecstore/src/bucket/sealed_credentials.rs

@@ -0,0 +1,351 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Sealed remote credentials shared by the three stores that persist one
+//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote
+//! tiers (`tier-config.bin`) and on-demand migration sources
+//! (`on-demand-migration.json`).
+//!
+//! The design record is `docs/architecture/remote-credential-sealing-adr.md`.
+//! What this module owns: the versioned envelope, the encryption context that
+//! binds a ciphertext to the record owning it, the sealer registration point,
+//! and the fail-closed error type. What it deliberately does not own: any KMS
+//! call (ECStore does not depend on `rustfs-kms`; the binary installs a
+//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event
+//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and
+//! any decision about which stored field a consumer writes.
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::fmt;
+use std::sync::{Arc, OnceLock};
+
+/// Envelope format this build writes. A reader accepts only versions it
+/// knows; an unknown version is a typed error, never a fallback.
+pub const SEALED_CREDENTIAL_VERSION: u8 = 1;
+
+/// Which store a sealed value belongs to. Part of the encryption context, so
+/// a ciphertext cannot be replayed into a different store.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum SealedCredentialStore {
+    /// `bucket-targets.json` (replication and other bucket targets).
+    BucketTargets,
+    /// `tier-config.bin` (remote tiers).
+    TierConfig,
+    /// `on-demand-migration.json` (migration sources).
+    OnDemandMigration,
+}
+
+impl SealedCredentialStore {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            SealedCredentialStore::BucketTargets => "bucket-targets",
+            SealedCredentialStore::TierConfig => "tier-config",
+            SealedCredentialStore::OnDemandMigration => "on-demand-migration",
+        }
+    }
+}
+
+/// Identity of the record a secret belongs to: the store, its owner (bucket
+/// name, tier name, or target ARN) and the field name. Rendered into the KMS
+/// encryption context so a ciphertext moved between buckets, tiers or fields
+/// fails to decrypt instead of silently authorizing a different remote.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SealScope {
+    pub store: SealedCredentialStore,
+    pub owner: String,
+    pub field: &'static str,
+}
+
+impl SealScope {
+    pub fn new(store: SealedCredentialStore, owner: impl Into<String>, field: &'static str) -> Self {
+        Self {
+            store,
+            owner: owner.into(),
+            field,
+        }
+    }
+
+    /// The encryption context handed to the sealer. Keys are stable: they are
+    /// part of the on-disk contract, because a ciphertext only decrypts under
+    /// the same context.
+    pub fn encryption_context(&self) -> HashMap<String, String> {
+        HashMap::from([
+            ("rustfs:store".to_string(), self.store.as_str().to_string()),
+            ("rustfs:owner".to_string(), self.owner.clone()),
+            ("rustfs:field".to_string(), self.field.to_string()),
+        ])
+    }
+}
+
+/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value
+/// is not a secret, but it is noise in a log line and an operator reading one
+/// should see the key it is wrapped under, not the bytes.
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SealedCredential {
+    /// Envelope version; see [`SEALED_CREDENTIAL_VERSION`].
+    pub v: u8,
+    /// KMS master key id the data key is wrapped under.
+    pub key_id: String,
+    /// Master key version, when the backend reports one. Carried so the KMS
+    /// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell
+    /// stale envelopes apart; nothing here rotates on its own.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub key_version: Option<String>,
+    /// Algorithm label reported by the sealer, for forensics and migration.
+    pub alg: String,
+    /// Ciphertext blob as produced by the sealer, base64 (standard, padded)
+    /// in the JSON stores and raw inside the tier msgpack payload.
+    pub ct: String,
+}
+
+impl fmt::Debug for SealedCredential {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SealedCredential")
+            .field("v", &self.v)
+            .field("key_id", &self.key_id)
+            .field("key_version", &self.key_version)
+            .field("alg", &self.alg)
+            .field("ct", &format_args!("<{} bytes sealed>", self.ct.len()))
+            .finish()
+    }
+}
+
+impl SealedCredential {
+    /// Rejects an envelope this build cannot read. Called before every
+    /// unseal so an unknown version fails here rather than inside a backend.
+    pub fn check_version(&self) -> Result<(), SealedCredentialError> {
+        if self.v == SEALED_CREDENTIAL_VERSION {
+            Ok(())
+        } else {
+            Err(SealedCredentialError::UnsupportedVersion(self.v))
+        }
+    }
+}
+
+/// Why a seal or unseal did not produce a usable value. Every variant is
+/// terminal for the record that carried it: a caller reports the remote as
+/// unusable, and never substitutes a default or empty credential.
+#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum SealedCredentialError {
+    /// No sealer is installed: KMS is not configured, or the process has not
+    /// finished startup. Reading a sealed record is impossible here.
+    #[error("no credential sealer is installed")]
+    NoSealer,
+    /// The stored envelope is from a newer (or otherwise unknown) format.
+    #[error("unsupported sealed credential envelope version {0}")]
+    UnsupportedVersion(u8),
+    /// The stored bytes are not a well-formed envelope.
+    #[error("malformed sealed credential: {0}")]
+    Malformed(String),
+    /// The sealer refused: wrong encryption context, missing key, revoked
+    /// access, or a failed authentication tag.
+    #[error("sealed credential could not be unsealed: {0}")]
+    Kms(String),
+}
+
+/// The KMS-backed half, installed by the binary at startup.
+#[async_trait]
+pub trait CredentialSealer: Send + Sync + 'static {
+    /// Wraps `plaintext` under the scope's encryption context.
+    async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError>;
+
+    /// Unwraps a stored envelope. Must fail when the envelope was sealed
+    /// under a different scope.
+    async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError>;
+}
+
+static CREDENTIAL_SEALER: OnceLock<Arc<dyn CredentialSealer>> = OnceLock::new();
+
+/// Installs the process-wide sealer. Returns `false` when one is already
+/// installed, matching the other ECStore hooks.
+pub fn install_credential_sealer(sealer: Arc<dyn CredentialSealer>) -> bool {
+    CREDENTIAL_SEALER.set(sealer).is_ok()
+}
+
+/// The installed sealer, or `None` when KMS is not wired. Callers that only
+/// need to know whether sealing is possible use this; callers that must have
+/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error.
+pub fn credential_sealer() -> Option<Arc<dyn CredentialSealer>> {
+    CREDENTIAL_SEALER.get().cloned()
+}
+
+/// Seals one secret field. Fails closed: without a sealer the caller must
+/// reject the write rather than persist the secret in clear text after the
+/// operator asked for sealing.
+pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
+    let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
+    sealer.seal(plaintext, scope).await
+}
+
+/// Unseals one secret field, rejecting an unknown envelope version first.
+pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
+    sealed.check_version()?;
+    let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
+    sealer.unseal(sealed, scope).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use parking_lot::Mutex;
+
+    /// Stands in for the KMS-backed sealer: records the context it was called
+    /// with, and refuses a ciphertext presented under a different one.
+    #[derive(Default)]
+    struct FakeSealer {
+        sealed_contexts: Mutex<Vec<HashMap<String, String>>>,
+    }
+
+    #[async_trait]
+    impl CredentialSealer for FakeSealer {
+        async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
+            let context = scope.encryption_context();
+            self.sealed_contexts.lock().push(context.clone());
+            let mut bound = serde_json::to_string(&context).expect("context serializes");
+            bound.push('|');
+            bound.push_str(plaintext);
+            Ok(SealedCredential {
+                v: SEALED_CREDENTIAL_VERSION,
+                key_id: "key-1".to_string(),
+                key_version: Some("3".to_string()),
+                alg: "AES-256-GCM".to_string(),
+                ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()),
+            })
+        }
+
+        async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
+            let raw = base64_simd::STANDARD
+                .decode_to_vec(sealed.ct.as_bytes())
+                .map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
+            let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
+            let expected = serde_json::to_string(&scope.encryption_context()).expect("context serializes");
+            bound
+                .strip_prefix(&expected)
+                .and_then(|rest| rest.strip_prefix('|'))
+                .map(str::to_string)
+                .ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string()))
+        }
+    }
+
+    fn scope(owner: &str) -> SealScope {
+        SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key")
+    }
+
+    #[tokio::test]
+    async fn seal_round_trips_and_binds_the_scope() {
+        let sealer = Arc::new(FakeSealer::default());
+        let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal");
+        assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION);
+        assert_eq!(sealed.key_version.as_deref(), Some("3"));
+        assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret");
+
+        // The same ciphertext under another bucket must not unseal.
+        let err = sealer
+            .unseal(&sealed, &scope("other-bucket"))
+            .await
+            .expect_err("a ciphertext must not move between owners");
+        assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
+
+        // Nor under another field of the same record.
+        let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token");
+        let err = sealer
+            .unseal(&sealed, &other_field)
+            .await
+            .expect_err("a ciphertext must not move between fields");
+        assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
+
+        let contexts = sealer.sealed_contexts.lock();
+        assert_eq!(contexts.len(), 1);
+        assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration");
+        assert_eq!(contexts[0]["rustfs:owner"], "photos");
+        assert_eq!(contexts[0]["rustfs:field"], "secret_key");
+    }
+
+    #[tokio::test]
+    async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() {
+        let sealed = SealedCredential {
+            v: SEALED_CREDENTIAL_VERSION + 1,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        assert_eq!(
+            sealed.check_version().expect_err("a newer envelope must not be read"),
+            SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
+        );
+        // The global helper reports the version, not "no sealer", even in a
+        // process where none is installed.
+        assert_eq!(
+            unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"),
+            SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
+        );
+    }
+
+    #[tokio::test]
+    async fn without_a_sealer_both_directions_fail_closed() {
+        // This test binary installs no sealer, so the global helpers must
+        // report NoSealer rather than fall back to clear text.
+        assert!(credential_sealer().is_none(), "no sealer is installed in unit tests");
+        assert_eq!(
+            seal_secret("super-secret", &scope("photos")).await.expect_err("seal"),
+            SealedCredentialError::NoSealer
+        );
+        let sealed = SealedCredential {
+            v: SEALED_CREDENTIAL_VERSION,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        assert_eq!(
+            unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"),
+            SealedCredentialError::NoSealer
+        );
+    }
+
+    #[test]
+    fn debug_and_serde_keep_the_on_disk_shape_stable() {
+        let sealed = SealedCredential {
+            v: 1,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        // key_version is omitted when absent, so an envelope from a backend
+        // without version history stays compact.
+        assert_eq!(
+            serde_json::to_string(&sealed).expect("serialize"),
+            r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#
+        );
+        let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#)
+            .expect("an envelope without key_version parses");
+        assert_eq!(parsed, sealed);
+
+        let rendered = format!("{sealed:?}");
+        assert!(rendered.contains("key-1"), "{rendered}");
+        assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}");
+    }
+
+    #[test]
+    fn a_malformed_envelope_is_a_typed_error() {
+        let err = serde_json::from_str::<SealedCredential>(r#"{"v":1,"key_id":"key-1"}"#)
+            .map_err(|err| SealedCredentialError::Malformed(err.to_string()))
+            .expect_err("a truncated envelope must not parse");
+        assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}");
+    }
+}

crates/ecstore/src/cluster/rpc/mod.rs

@@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
 pub(crate) use peer_rest_client::TierConfigReloadOutcome;
 pub use peer_rest_client::{
     KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
-    SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
+    SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
 pub use peer_s3_client::{

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -20,7 +20,8 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
 use crate::error::{Error, Result};
 use crate::storage_api_contracts::internode::{
     SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
-    SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
+    SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
+    SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
 };
 use crate::{
     bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats, TierDailyStatsWire},
@@ -47,18 +48,19 @@ use rustfs_protos::proto_gen::node_service::{
     HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
     LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
-    ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
-    ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
-    StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest,
-    TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
-    node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
+    ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
+    ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest,
+    SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
+    TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
+    TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
 use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
 use rustfs_utils::XHost;
 use serde::{Deserialize, Serialize as _};
 use std::{
-    collections::HashMap,
+    collections::{BTreeMap, HashMap},
     io::Cursor,
     sync::{
         Arc, Weak,
@@ -185,18 +187,31 @@ pub struct ScannerPeerActivity {
     pub publication_blocked: Option<bool>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ScannerPeerDirtyUsageSnapshot {
+    pub instance_id: String,
+    pub generation: u64,
+    pub pending_bucket_count: u64,
+    pub protocol_version: u32,
+    pub complete: bool,
+    pub buckets: BTreeMap<String, u64>,
+}
+
+fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
+    instance_id.len() == 32
+        && instance_id
+            .as_bytes()
+            .iter()
+            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte))
+}
+
 fn decode_scanner_activity_with_verifier(
     response: ScannerActivityResponse,
     challenge: &[u8; 16],
     verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
 ) -> Result<ScannerPeerActivity> {
     let instance_id = &response.instance_id;
-    if instance_id.len() != 32
-        || !instance_id
-            .as_bytes()
-            .iter()
-            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte))
-    {
+    if !scanner_instance_id_is_valid(instance_id) {
         return Err(Error::other("peer returned an invalid scanner activity instance ID"));
     }
     let (
@@ -318,6 +333,82 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
     })
 }
 
+fn decode_scanner_dirty_usage_snapshot_with_verifier(
+    response: ScannerDirtyUsageSnapshotResponse,
+    challenge: &[u8; 16],
+    verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
+) -> Result<ScannerPeerDirtyUsageSnapshot> {
+    let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(challenge, &response)
+        .map_err(|_| Error::other("peer scanner dirty usage snapshot is too large to authenticate"))?;
+    verify_proof(&canonical, &response.response_proof)?;
+
+    if response.protocol_version != SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION {
+        return Err(Error::other("peer returned unsupported scanner dirty usage snapshot protocol"));
+    }
+    if !scanner_instance_id_is_valid(&response.instance_id) {
+        return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID"));
+    }
+    if response.generation == u64::MAX {
+        return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation"));
+    }
+    if response.pending_bucket_count > 0 && response.generation == 0 {
+        return Err(Error::other("peer scanner dirty usage snapshot has pending buckets without a generation"));
+    }
+    if response.buckets.len() > SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
+        return Err(Error::other("peer scanner dirty usage snapshot exceeds the entry limit"));
+    }
+    let bucket_count = u64::try_from(response.buckets.len())
+        .map_err(|_| Error::other("peer scanner dirty usage snapshot entry count cannot be represented"))?;
+    let max_entries = u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES).unwrap_or(u64::MAX);
+    if response.complete {
+        if response.pending_bucket_count != bucket_count {
+            return Err(Error::other(
+                "complete peer scanner dirty usage snapshot has an inconsistent bucket count",
+            ));
+        }
+    } else if !response.buckets.is_empty() || response.pending_bucket_count <= max_entries {
+        return Err(Error::other(
+            "incomplete peer scanner dirty usage snapshot must represent an entry-limit overflow",
+        ));
+    }
+    for pair in response.buckets.windows(2) {
+        if pair[0].bucket >= pair[1].bucket {
+            return Err(Error::other("peer scanner dirty usage snapshot buckets are not strictly ordered"));
+        }
+    }
+    for bucket in &response.buckets {
+        if bucket.bucket.is_empty() {
+            return Err(Error::other("peer scanner dirty usage snapshot contains an empty bucket name"));
+        }
+        if bucket.generation == 0 || bucket.generation > response.generation {
+            return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation"));
+        }
+    }
+
+    Ok(ScannerPeerDirtyUsageSnapshot {
+        instance_id: response.instance_id,
+        generation: response.generation,
+        pending_bucket_count: response.pending_bucket_count,
+        protocol_version: response.protocol_version,
+        complete: response.complete,
+        buckets: response
+            .buckets
+            .into_iter()
+            .map(|bucket| (bucket.bucket, bucket.generation))
+            .collect(),
+    })
+}
+
+fn decode_scanner_dirty_usage_snapshot(
+    response: ScannerDirtyUsageSnapshotResponse,
+    challenge: &[u8; 16],
+) -> Result<ScannerPeerDirtyUsageSnapshot> {
+    decode_scanner_dirty_usage_snapshot_with_verifier(response, challenge, |canonical, proof| {
+        verify_tonic_rpc_response_proof(canonical, proof)
+            .map_err(|_| Error::other("peer returned an invalid scanner dirty usage snapshot response proof"))
+    })
+}
+
 fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
     matches!(
         err,
@@ -1935,6 +2026,30 @@ impl PeerRestClient {
         }
     }
 
+    pub async fn scanner_dirty_usage_snapshot(&self) -> Result<ScannerPeerDirtyUsageSnapshot> {
+        self.finalize_result(
+            async {
+                let challenge = Uuid::new_v4();
+                let mut client = self
+                    .get_client()
+                    .await?
+                    .max_decoding_message_size(SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE)
+                    .max_encoding_message_size(SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE);
+                let mut request = Request::new(ScannerDirtyUsageSnapshotRequest {
+                    challenge: challenge.as_bytes().to_vec().into(),
+                    protocol_version: SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
+                });
+                let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(request.get_ref())
+                    .map_err(|_| Error::other("scanner dirty usage snapshot request is too large to authenticate"))?;
+                set_tonic_canonical_body_digest(&mut request, &canonical)?;
+                let response = client.scanner_dirty_usage_snapshot(request).await?.into_inner();
+                decode_scanner_dirty_usage_snapshot(response, challenge.as_bytes())
+            }
+            .await,
+        )
+        .await
+    }
+
     pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
         let result = self
             .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
@@ -2640,6 +2755,141 @@ mod tests {
         })
     }
 
+    fn decode_test_scanner_dirty_usage_snapshot(
+        response: ScannerDirtyUsageSnapshotResponse,
+    ) -> Result<ScannerPeerDirtyUsageSnapshot> {
+        decode_scanner_dirty_usage_snapshot_with_verifier(response, &[9; 16], |_canonical, proof| {
+            (proof == b"proof")
+                .then_some(())
+                .ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot response proof"))
+        })
+    }
+
+    fn test_scanner_dirty_usage_snapshot_response() -> ScannerDirtyUsageSnapshotResponse {
+        ScannerDirtyUsageSnapshotResponse {
+            instance_id: "0123456789abcdef0123456789abcdef".to_string(),
+            generation: 7,
+            pending_bucket_count: 2,
+            protocol_version: SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
+            complete: true,
+            buckets: vec![
+                rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
+                    bucket: "archive".to_string(),
+                    generation: 3,
+                },
+                rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
+                    bucket: "photos".to_string(),
+                    generation: 7,
+                },
+            ],
+            response_proof: b"proof".to_vec().into(),
+        }
+    }
+
+    #[test]
+    fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() {
+        let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response())
+            .expect("a complete authenticated dirty usage snapshot should decode");
+        assert_eq!(decoded.instance_id, "0123456789abcdef0123456789abcdef");
+        assert_eq!(decoded.generation, 7);
+        assert_eq!(decoded.pending_bucket_count, 2);
+        assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
+        assert!(decoded.complete);
+        assert_eq!(decoded.buckets.get("archive"), Some(&3));
+        assert_eq!(decoded.buckets.get("photos"), Some(&7));
+
+        let overflow_count =
+            u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
+        let overflow = decode_test_scanner_dirty_usage_snapshot(ScannerDirtyUsageSnapshotResponse {
+            pending_bucket_count: overflow_count,
+            complete: false,
+            buckets: Vec::new(),
+            ..test_scanner_dirty_usage_snapshot_response()
+        })
+        .expect("an explicit all-or-nothing overflow snapshot should decode");
+        assert!(!overflow.complete);
+        assert!(overflow.buckets.is_empty());
+    }
+
+    #[test]
+    fn scanner_dirty_usage_snapshot_rejects_inconsistent_or_partial_peer_data() {
+        let mut cases = Vec::new();
+
+        let mut invalid_instance = test_scanner_dirty_usage_snapshot_response();
+        invalid_instance.instance_id = "ABCDEF0123456789ABCDEF0123456789".to_string();
+        cases.push((invalid_instance, "instance ID"));
+
+        let mut unsupported = test_scanner_dirty_usage_snapshot_response();
+        unsupported.protocol_version = SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1;
+        cases.push((unsupported, "unsupported"));
+
+        let mut exhausted = test_scanner_dirty_usage_snapshot_response();
+        exhausted.generation = u64::MAX;
+        cases.push((exhausted, "exhausted"));
+
+        let mut inconsistent_count = test_scanner_dirty_usage_snapshot_response();
+        inconsistent_count.pending_bucket_count = 3;
+        cases.push((inconsistent_count, "bucket count"));
+
+        let mut unordered = test_scanner_dirty_usage_snapshot_response();
+        unordered.buckets.reverse();
+        cases.push((unordered, "strictly ordered"));
+
+        let mut future_bucket = test_scanner_dirty_usage_snapshot_response();
+        future_bucket.buckets[0].generation = 8;
+        cases.push((future_bucket, "bucket generation"));
+
+        let mut zero_generation = test_scanner_dirty_usage_snapshot_response();
+        zero_generation.buckets[0].generation = 0;
+        cases.push((zero_generation, "bucket generation"));
+
+        let mut empty_bucket = test_scanner_dirty_usage_snapshot_response();
+        empty_bucket.buckets[0].bucket.clear();
+        cases.push((empty_bucket, "empty bucket name"));
+
+        let mut partial = test_scanner_dirty_usage_snapshot_response();
+        partial.complete = false;
+        cases.push((partial, "entry-limit overflow"));
+
+        let too_many_buckets = ScannerDirtyUsageSnapshotResponse {
+            generation: 1,
+            pending_bucket_count: u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1)
+                .expect("the test snapshot entry limit should fit in u64"),
+            buckets: (0..=SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES)
+                .map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
+                    bucket: format!("bucket-{index:04}"),
+                    generation: 1,
+                })
+                .collect(),
+            ..test_scanner_dirty_usage_snapshot_response()
+        };
+        cases.push((too_many_buckets, "exceeds the entry limit"));
+
+        let overflow_count =
+            u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
+        let invalid_overflow = ScannerDirtyUsageSnapshotResponse {
+            generation: 0,
+            pending_bucket_count: overflow_count,
+            complete: false,
+            buckets: Vec::new(),
+            ..test_scanner_dirty_usage_snapshot_response()
+        };
+        cases.push((invalid_overflow, "without a generation"));
+
+        for (response, expected) in cases {
+            let err =
+                decode_test_scanner_dirty_usage_snapshot(response).expect_err("malformed dirty usage snapshots must fail closed");
+            assert!(err.to_string().contains(expected), "expected {expected:?} in {err}");
+        }
+
+        let mut invalid_proof = test_scanner_dirty_usage_snapshot_response();
+        invalid_proof.protocol_version = SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1;
+        invalid_proof.response_proof = b"invalid".to_vec().into();
+        let err = decode_test_scanner_dirty_usage_snapshot(invalid_proof)
+            .expect_err("an invalid response proof must fail before peer fields are trusted");
+        assert!(err.to_string().contains("response proof"));
+    }
+
     #[test]
     fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
         let slots = vec![

crates/ecstore/src/runtime/instance.rs

@@ -213,7 +213,6 @@ pub struct InstanceContext {
     object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
-    #[cfg(test)]
     tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
 }
 
@@ -260,7 +259,6 @@ impl InstanceContext {
             object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
-            #[cfg(test)]
             tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
         }
     }
@@ -655,16 +653,10 @@ impl InstanceContext {
             .insert(store_id)
     }
 
-    #[cfg(test)]
-    #[allow(
-        dead_code,
-        reason = "driven by the tier-delete-journal recovery test behind `--features test-util` (backlog#1823)"
-    )]
     pub(crate) fn wake_tier_delete_journal_recovery(&self) {
         self.tier_delete_journal_recovery_wakeup.notify_one();
     }
 
-    #[cfg(test)]
     pub(crate) async fn wait_for_tier_delete_journal_recovery(&self) {
         self.tier_delete_journal_recovery_wakeup.notified().await;
     }

crates/ecstore/src/services/event_notification.rs

@@ -117,26 +117,85 @@ pub fn send_event(args: EventArgs) {
     );
 }
 
+/// Shared event recorder for this crate's tests.
+///
+/// [`register_event_dispatch_hook`] backs a `OnceLock`, so only the first
+/// caller in a test binary can install a hook. Every test that needs to
+/// observe dispatched events must therefore go through this single recorder
+/// instead of registering its own.
+#[cfg(test)]
+pub(crate) mod test_recorder {
+    use super::register_event_dispatch_hook;
+    use std::sync::{Mutex, OnceLock};
+    use uuid::Uuid;
+
+    /// The fields a test needs from a dispatched event. `EventArgs` itself is
+    /// not `Clone`, and recording a reduced shape keeps this test seam from
+    /// constraining the production type.
+    #[derive(Clone, Debug)]
+    pub(crate) struct RecordedEvent {
+        pub(crate) event_name: String,
+        pub(crate) bucket: String,
+        pub(crate) object: String,
+        pub(crate) version_id: Option<Uuid>,
+        pub(crate) delete_marker: bool,
+    }
+
+    static RECORDED: OnceLock<Mutex<Vec<RecordedEvent>>> = OnceLock::new();
+
+    fn recorded() -> &'static Mutex<Vec<RecordedEvent>> {
+        RECORDED.get_or_init(|| Mutex::new(Vec::new()))
+    }
+
+    pub(crate) fn install() {
+        static INSTALLED: OnceLock<()> = OnceLock::new();
+        INSTALLED.get_or_init(|| {
+            assert!(
+                register_event_dispatch_hook(|args| {
+                    recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent {
+                        event_name: args.event_name,
+                        bucket: args.bucket_name,
+                        object: args.object.name,
+                        version_id: args.object.version_id,
+                        delete_marker: args.object.delete_marker,
+                    });
+                }),
+                "the test event recorder must own this binary's dispatch hook"
+            );
+        });
+    }
+
+    /// Everything recorded for one bucket. Tests select by their own unique
+    /// bucket name rather than draining, because tests that are not
+    /// `#[serial]` may dispatch events concurrently.
+    pub(crate) fn recorded_for_bucket(bucket: &str) -> Vec<RecordedEvent> {
+        recorded()
+            .lock()
+            .unwrap_or_else(|err| err.into_inner())
+            .iter()
+            .filter(|event| event.bucket == bucket)
+            .cloned()
+            .collect()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-
-    static DISPATCH_COUNT: AtomicUsize = AtomicUsize::new(0);
 
     #[test]
     fn send_event_dispatches_to_registered_hook() {
-        let _ = register_event_dispatch_hook(|_args| {
-            DISPATCH_COUNT.fetch_add(1, Ordering::Relaxed);
-        });
-        let before = DISPATCH_COUNT.load(Ordering::Relaxed);
+        test_recorder::install();
+        let bucket = format!("event-dispatch-{}", uuid::Uuid::new_v4().simple());
 
         send_event(EventArgs {
             event_name: "s3:ObjectCreated:Put".to_string(),
-            bucket_name: "demo".to_string(),
+            bucket_name: bucket.clone(),
             ..Default::default()
         });
 
-        assert_eq!(DISPATCH_COUNT.load(Ordering::Relaxed), before + 1);
+        let dispatched = test_recorder::recorded_for_bucket(&bucket);
+        assert_eq!(dispatched.len(), 1);
+        assert_eq!(dispatched[0].event_name, "s3:ObjectCreated:Put");
     }
 }

crates/ecstore/src/services/notification_sys.rs

@@ -13,7 +13,9 @@
 // limitations under the License.
 
 use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
-use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
+use crate::cluster::rpc::{
+    PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome,
+};
 use crate::diagnostics::admin_server_info::get_commit_id;
 use crate::disk::DiskAPI;
 use crate::error::{Error, Result};
@@ -2080,6 +2082,32 @@ impl NotificationSys {
         Ok(generations)
     }
 
+    pub async fn scanner_dirty_usage_snapshots(&self) -> Result<Vec<(String, ScannerPeerDirtyUsageSnapshot)>> {
+        if self.peer_clients.is_empty() {
+            return Err(Error::other("scanner dirty usage snapshot probe has no remote peers"));
+        }
+        if self.all_peer_clients.len() != self.peer_clients.len() + 1 {
+            return Err(Error::other("scanner dirty usage snapshot peer topology is incomplete"));
+        }
+
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter().cloned() {
+            futures.push(async move {
+                let client = client.ok_or_else(|| Error::other("scanner dirty usage snapshot peer is unreachable"))?;
+                let host = client.grid_host.clone();
+                scanner_dirty_usage_snapshot_with_retry(&client, &host)
+                    .await
+                    .map(|snapshot| (host, snapshot))
+            });
+        }
+
+        let mut snapshots = Vec::with_capacity(futures.len());
+        for result in join_all(futures).await {
+            snapshots.push(result?);
+        }
+        Ok(snapshots)
+    }
+
     pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
         let mut by_host = HashMap::with_capacity(acknowledgements.len());
         for (host, instance_id, generation) in acknowledgements {
@@ -2591,6 +2619,54 @@ async fn scanner_activity_with_retry(client: &PeerRestClient, host: &str) -> Res
     }
 }
 
+async fn scanner_dirty_usage_snapshot_with_retry(client: &PeerRestClient, host: &str) -> Result<ScannerPeerDirtyUsageSnapshot> {
+    let first = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_dirty_usage_snapshot()).await;
+    let should_retry = match &first {
+        Ok(Ok(_)) => false,
+        Ok(Err(err)) => scanner_activity_should_retry(Some(err), false),
+        Err(_) => scanner_activity_should_retry(None, true),
+    };
+
+    match first {
+        Ok(Ok(snapshot)) => return Ok(snapshot),
+        Ok(Err(err)) if !should_retry => return Err(err),
+        Ok(Err(err)) => {
+            debug!(
+                event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                result = "retrying",
+                capability = "scanner_dirty_usage_snapshot",
+                peer = host,
+                error = %err,
+                "notification capability probe retrying"
+            );
+            client.prepare_retry().await;
+        }
+        Err(_) => {
+            debug!(
+                event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                result = "retrying",
+                capability = "scanner_dirty_usage_snapshot",
+                peer = host,
+                timeout = ?SCANNER_ACTIVITY_PROBE_TIMEOUT,
+                "notification capability probe retrying"
+            );
+            client.prepare_retry().await;
+        }
+    }
+
+    match timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_dirty_usage_snapshot()).await {
+        Ok(result) => result,
+        Err(_) => {
+            client.evict_connection().await;
+            Err(Error::Timeout)
+        }
+    }
+}
+
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 async fn call_peer_with_timeout<F, Fut>(
     timeout_dur: Duration,
@@ -3688,6 +3764,52 @@ mod tests {
         assert!(err.to_string().contains("peer topology is incomplete"));
     }
 
+    #[tokio::test]
+    async fn scanner_dirty_usage_snapshot_probe_rejects_unusable_peer_topologies() {
+        let unreachable = NotificationSys {
+            peer_clients: vec![None],
+            all_peer_clients: vec![None, None],
+            peer_topology_hosts: vec!["node-a:9000".to_string()],
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+            tier_config_reload_workers: Default::default(),
+        };
+        let err = unreachable
+            .scanner_dirty_usage_snapshots()
+            .await
+            .expect_err("an unreachable peer must invalidate the distributed dirty usage snapshot");
+        assert!(err.to_string().contains("peer is unreachable"));
+
+        let empty = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: Vec::new(),
+            peer_topology_hosts: Vec::new(),
+            peer_admin_caches: Vec::new(),
+            tier_config_reload_workers: Default::default(),
+        };
+        let err = empty
+            .scanner_dirty_usage_snapshots()
+            .await
+            .expect_err("an empty peer set must not produce a distributed dirty usage snapshot");
+        assert!(err.to_string().contains("no remote peers"));
+
+        let client = PeerRestClient::new(
+            "127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
+            "http://127.0.0.1:9000".to_string(),
+        );
+        let incomplete = NotificationSys {
+            peer_clients: vec![Some(client)],
+            all_peer_clients: vec![None],
+            peer_topology_hosts: vec!["127.0.0.1:9000".to_string()],
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+            tier_config_reload_workers: Default::default(),
+        };
+        let err = incomplete
+            .scanner_dirty_usage_snapshots()
+            .await
+            .expect_err("an incomplete topology must not produce a distributed dirty usage snapshot");
+        assert!(err.to_string().contains("peer topology is incomplete"));
+    }
+
     #[tokio::test]
     async fn server_info_no_client_slot_uses_topology_host_without_counting_rpc_failure() {
         let sys = NotificationSys {

crates/ecstore/src/services/tier/tier_admin.rs

@@ -18,25 +18,203 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]
 
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
 
-#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[derive(Serialize, Deserialize, Default, Clone)]
+#[serde(default)]
+pub struct TierServicePrincipalAuth {
+    #[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")]
+    pub tenant_id: String,
+    #[serde(rename = "ClientID", alias = "clientID", alias = "client_id")]
+    pub client_id: String,
+    #[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")]
+    pub client_secret: String,
+}
+
+impl TierServicePrincipalAuth {
+    pub(crate) fn is_empty(&self) -> bool {
+        self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty()
+    }
+}
+
+impl std::fmt::Debug for TierServicePrincipalAuth {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierServicePrincipalAuth")
+            .field("tenant_id", &self.tenant_id)
+            .field("client_id", &self.client_id)
+            .field("client_secret", &"REDACTED")
+            .finish()
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Clone)]
 #[serde(default)]
 pub struct TierCreds {
-    #[serde(rename = "accessKey")]
+    #[serde(rename = "access", alias = "accessKey")]
     pub access_key: String,
-    #[serde(rename = "secretKey")]
+    #[serde(rename = "secret", alias = "secretKey")]
     pub secret_key: String,
 
-    #[serde(rename = "awsRole")]
+    #[serde(rename = "awsrole", alias = "awsRole")]
     pub aws_role: bool,
-    #[serde(rename = "awsRoleWebIdentityTokenFile")]
+    #[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")]
     pub aws_role_web_identity_token_file: String,
-    #[serde(rename = "awsRoleArn")]
+    #[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")]
     pub aws_role_arn: String,
 
-    //azsp: ServicePrincipalAuth,
+    #[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")]
+    pub azure_service_principal: TierServicePrincipalAuth,
 
-    //#[serde(rename = "credsJson")]
+    #[serde(
+        rename = "creds",
+        alias = "credsJson",
+        alias = "credsJSON",
+        alias = "creds_json",
+        default,
+        skip_serializing_if = "Vec::is_empty",
+        with = "base64_bytes"
+    )]
     pub creds_json: Vec<u8>,
 }
+
+impl std::fmt::Debug for TierCreds {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierCreds")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &"REDACTED")
+            .field("aws_role", &self.aws_role)
+            .field(
+                "aws_role_web_identity_token_file",
+                &(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"),
+            )
+            .field("aws_role_arn", &self.aws_role_arn)
+            .field("azure_service_principal", &self.azure_service_principal)
+            .field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED"))
+            .finish()
+    }
+}
+
+mod base64_bytes {
+    use super::*;
+
+    pub(super) fn serialize<S>(value: &[u8], serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value))
+    }
+
+    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum EncodedBytes {
+            Base64(String),
+            Legacy(Vec<u8>),
+        }
+
+        match EncodedBytes::deserialize(deserializer)? {
+            EncodedBytes::Base64(value) => base64_simd::STANDARD
+                .decode_to_vec(value.as_bytes())
+                .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes()))
+                .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes()))
+                .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes()))
+                .map_err(de::Error::custom),
+            EncodedBytes::Legacy(value) => Ok(value),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() {
+        let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes();
+        let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ==";
+        let creds: TierCreds = serde_json::from_value(serde_json::json!({
+            "access": "access",
+            "secret": "secret",
+            "awsrole": false,
+            "creds": encoded,
+        }))
+        .expect("madmin tier credentials should decode");
+
+        assert_eq!(creds.access_key, "access");
+        assert_eq!(creds.secret_key, "secret");
+        assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
+
+        let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
+        assert_eq!(wire["access"], "access");
+        assert_eq!(wire["secret"], "secret");
+        assert_eq!(wire["creds"], encoded);
+        assert!(wire.get("accessKey").is_none());
+        assert!(wire.get("secretKey").is_none());
+
+        let legacy: TierCreds = serde_json::from_value(serde_json::json!({
+            "accessKey": "legacy-access",
+            "secretKey": "legacy-secret",
+            "credsJson": service_account,
+        }))
+        .expect("the former RustFS field names and byte-array encoding should remain readable");
+        assert_eq!(legacy.access_key, "legacy-access");
+        assert_eq!(legacy.secret_key, "legacy-secret");
+        assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
+    }
+
+    #[test]
+    fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() {
+        let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes();
+        for encoder in [
+            base64_simd::STANDARD,
+            base64_simd::STANDARD_NO_PAD,
+            base64_simd::URL_SAFE,
+            base64_simd::URL_SAFE_NO_PAD,
+        ] {
+            let encoded = encoder.encode_to_string(service_account);
+            let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded }))
+                .expect("all supported madmin base64 forms should decode");
+            assert_eq!(creds.creds_json, service_account);
+        }
+    }
+
+    #[test]
+    fn tier_creds_debug_redacts_secret_payloads() {
+        let creds = TierCreds {
+            access_key: "access".to_string(),
+            secret_key: "tier-secret-value".to_string(),
+            aws_role_web_identity_token_file: "/var/run/private-token".to_string(),
+            creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(),
+            ..Default::default()
+        };
+
+        let rendered = format!("{creds:?}");
+        assert!(!rendered.contains("tier-secret-value"));
+        assert!(!rendered.contains("/var/run/private-token"));
+        assert!(!rendered.contains("gcs-private-key-value"));
+    }
+
+    #[test]
+    fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() {
+        let creds: TierCreds = serde_json::from_value(serde_json::json!({
+            "azSP": {
+                "TenantID": "tenant",
+                "ClientID": "client",
+                "ClientSecret": "service-principal-secret"
+            }
+        }))
+        .expect("canonical madmin azure service principal credentials should decode");
+
+        assert_eq!(creds.azure_service_principal.tenant_id, "tenant");
+        assert_eq!(creds.azure_service_principal.client_id, "client");
+        assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret");
+        let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode");
+        assert_eq!(wire["azSP"]["TenantID"], "tenant");
+        assert_eq!(wire["azSP"]["ClientID"], "client");
+        assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret");
+        assert!(!format!("{creds:?}").contains("service-principal-secret"));
+    }
+}

crates/ecstore/src/services/tier/tier_config.rs

@@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
 pub enum TierType {
     #[default]
     Unsupported,
-    #[serde(rename = "s3")]
+    #[serde(rename = "s3", alias = "S3")]
     S3,
     #[serde(rename = "wasabi")]
     Wasabi,
@@ -58,7 +58,7 @@ pub enum TierType {
     Huaweicloud,
     #[serde(rename = "azure")]
     Azure,
-    #[serde(rename = "gcs")]
+    #[serde(rename = "gcs", alias = "GCS")]
     GCS,
     #[serde(rename = "r2")]
     R2,
@@ -138,16 +138,18 @@ impl TierType {
     }
 }
 
-#[derive(Default, Debug, Serialize, Deserialize)]
+pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
+
+#[derive(Default, Serialize, Deserialize)]
 #[serde(default)]
 pub struct TierConfig {
     #[serde(skip)]
     pub version: String,
-    #[serde(rename = "type")]
+    #[serde(rename = "type", alias = "Type")]
     pub tier_type: TierType,
-    #[serde(skip)]
+    #[serde(rename = "Name", alias = "name", skip_serializing)]
     pub name: String,
-    #[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")]
     pub s3: Option<TierS3>,
     #[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
     pub wasabi: Option<TierWasabi>,
@@ -159,7 +161,7 @@ pub struct TierConfig {
     pub huaweicloud: Option<TierHuaweicloud>,
     #[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
     pub azure: Option<TierAzure>,
-    #[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")]
     pub gcs: Option<TierGCS>,
     #[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
     pub r2: Option<TierR2>,
@@ -170,109 +172,91 @@ pub struct TierConfig {
 }
 
 impl Clone for TierConfig {
-    fn clone(&self) -> TierConfig {
-        let mut s3 = None;
-        let mut wasabi = None;
-        let mut r = None;
-        let mut compatible_backend = None;
-        let mut aliyun = None;
-        let mut tencent = None;
-        let mut huaweicloud = None;
-        let mut azure = None;
-        let mut gcs = None;
-        let mut r2 = None;
-        match self.tier_type {
-            TierType::S3 => {
-                if let Some(s3_) = self.s3.as_ref() {
-                    let mut s3_clone = s3_.clone();
-                    s3_clone.secret_key = "REDACTED".to_string();
-                    s3 = Some(s3_clone);
-                }
-            }
-            TierType::Wasabi => {
-                if let Some(wasabi_) = self.wasabi.as_ref() {
-                    let mut wasabi_clone = wasabi_.clone();
-                    wasabi_clone.secret_key = "REDACTED".to_string();
-                    wasabi = Some(wasabi_clone);
-                }
-            }
-            TierType::RustFS => {
-                if let Some(r_) = self.rustfs.as_ref() {
-                    let mut r_clone = r_.clone();
-                    r_clone.secret_key = "REDACTED".to_string();
-                    r = Some(r_clone);
-                }
-            }
-            TierType::MinIO => {
-                if let Some(compatible_backend_) = self.minio.as_ref() {
-                    let mut compatible_backend_clone = compatible_backend_.clone();
-                    compatible_backend_clone.secret_key = "REDACTED".to_string();
-                    compatible_backend = Some(compatible_backend_clone);
-                }
-            }
-            TierType::Aliyun => {
-                if let Some(aliyun_) = self.aliyun.as_ref() {
-                    let mut aliyun_clone = aliyun_.clone();
-                    aliyun_clone.secret_key = "REDACTED".to_string();
-                    aliyun = Some(aliyun_clone);
-                }
-            }
-            TierType::Tencent => {
-                if let Some(tencent_) = self.tencent.as_ref() {
-                    let mut tencent_clone = tencent_.clone();
-                    tencent_clone.secret_key = "REDACTED".to_string();
-                    tencent = Some(tencent_clone);
-                }
-            }
-            TierType::Huaweicloud => {
-                if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
-                    let mut huaweicloud_clone = huaweicloud_.clone();
-                    huaweicloud_clone.secret_key = "REDACTED".to_string();
-                    huaweicloud = Some(huaweicloud_clone);
-                }
-            }
-            TierType::Azure => {
-                if let Some(azure_) = self.azure.as_ref() {
-                    let mut azure_clone = azure_.clone();
-                    azure_clone.secret_key = "REDACTED".to_string();
-                    azure = Some(azure_clone);
-                }
-            }
-            TierType::GCS => {
-                if let Some(gcs_) = self.gcs.as_ref() {
-                    let mut gcs_clone = gcs_.clone();
-                    gcs_clone.creds = "REDACTED".to_string();
-                    gcs = Some(gcs_clone);
-                }
-            }
-            TierType::R2 => {
-                if let Some(r2_) = self.r2.as_ref() {
-                    let mut r2_clone = r2_.clone();
-                    r2_clone.secret_key = "REDACTED".to_string();
-                    r2 = Some(r2_clone);
-                }
-            }
-            _ => (),
-        }
-        TierConfig {
-            version: self.version.clone(),
-            tier_type: self.tier_type.clone(),
-            name: self.name.clone(),
-            s3,
-            wasabi,
-            rustfs: r,
-            minio: compatible_backend,
-            aliyun,
-            tencent,
-            huaweicloud,
-            azure,
-            gcs,
-            r2,
-        }
+    fn clone(&self) -> Self {
+        self.redacted()
     }
 }
 
 impl TierConfig {
+    pub(crate) fn redacted(&self) -> Self {
+        let mut redacted = Self {
+            version: self.version.clone(),
+            tier_type: self.tier_type.clone(),
+            name: self.name.clone(),
+            ..Default::default()
+        };
+        match self.tier_type {
+            TierType::S3 => {
+                redacted.s3 = self.s3.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    if !backend.aws_role_web_identity_token_file.is_empty() {
+                        backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string();
+                    }
+                    backend
+                });
+            }
+            TierType::Wasabi => {
+                redacted.wasabi = self.wasabi.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::RustFS => {
+                redacted.rustfs = self.rustfs.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::MinIO => {
+                redacted.minio = self.minio.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Aliyun => {
+                redacted.aliyun = self.aliyun.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Tencent => {
+                redacted.tencent = self.tencent.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Huaweicloud => {
+                redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Azure => {
+                redacted.azure = self.azure.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    if !backend.sp_auth.client_secret.is_empty() {
+                        backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string();
+                    }
+                    backend
+                });
+            }
+            TierType::GCS => {
+                redacted.gcs = self.gcs.clone().map(|mut backend| {
+                    backend.creds = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::R2 => {
+                redacted.r2 = self.r2.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Unsupported => {}
+        }
+        redacted
+    }
+
     pub(crate) fn clone_with_credentials(&self) -> Self {
         Self {
             version: self.version.clone(),
@@ -372,31 +356,61 @@ impl TierConfig {
     }
 }
 
+impl std::fmt::Debug for TierConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let redacted = self.redacted();
+        f.debug_struct("TierConfig")
+            .field("version", &redacted.version)
+            .field("tier_type", &redacted.tier_type)
+            .field("name", &redacted.name)
+            .field("s3", &redacted.s3)
+            .field("wasabi", &redacted.wasabi)
+            .field("aliyun", &redacted.aliyun)
+            .field("tencent", &redacted.tencent)
+            .field("huaweicloud", &redacted.huaweicloud)
+            .field("azure", &redacted.azure)
+            .field("gcs", &redacted.gcs)
+            .field("r2", &redacted.r2)
+            .field("rustfs", &redacted.rustfs)
+            .field("minio", &redacted.minio)
+            .finish()
+    }
+}
+
 //type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
 
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct TierS3 {
+    #[serde(alias = "Name")]
     pub name: String,
+    #[serde(alias = "Endpoint")]
     pub endpoint: String,
-    #[serde(rename = "accessKey")]
+    #[serde(rename = "accessKey", alias = "AccessKey")]
     pub access_key: String,
-    #[serde(rename = "secretKey")]
+    #[serde(rename = "secretKey", alias = "SecretKey")]
     pub secret_key: String,
+    #[serde(alias = "Bucket")]
     pub bucket: String,
+    #[serde(alias = "Prefix")]
     pub prefix: String,
+    #[serde(alias = "Region")]
     pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
     pub storage_class: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)]
     pub aws_role: bool,
-    #[serde(skip)]
+    #[serde(
+        rename = "AWSRoleWebIdentityTokenFile",
+        alias = "awsRoleWebIdentityTokenFile",
+        skip_serializing
+    )]
     pub aws_role_web_identity_token_file: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)]
     pub aws_role_arn: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)]
     pub aws_role_session_name: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)]
     pub aws_role_duration_seconds: i32,
 }
 
@@ -623,8 +637,11 @@ pub struct TierHuaweicloud {
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct ServicePrincipalAuth {
+    #[serde(alias = "TenantID")]
     pub tenant_id: String,
+    #[serde(alias = "ClientID")]
     pub client_id: String,
+    #[serde(alias = "ClientSecret")]
     pub client_secret: String,
 }
 
@@ -640,9 +657,9 @@ pub struct TierAzure {
     pub bucket: String,
     pub prefix: String,
     pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
     pub storage_class: String,
-    #[serde(rename = "spAuth")]
+    #[serde(rename = "spAuth", alias = "SPAuth")]
     pub sp_auth: ServicePrincipalAuth,
 }
 
@@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error {
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct TierGCS {
+    #[serde(alias = "Name")]
     pub name: String,
+    #[serde(alias = "Endpoint")]
     pub endpoint: String,
-    #[serde(rename = "creds")]
+    #[serde(rename = "creds", alias = "Creds")]
     pub creds: String,
+    #[serde(alias = "Bucket")]
     pub bucket: String,
+    #[serde(alias = "Prefix")]
     pub prefix: String,
+    #[serde(alias = "Region")]
     pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
     pub storage_class: String,
 }
 
@@ -725,6 +747,43 @@ pub struct TierR2 {
 mod tests {
     use super::*;
 
+    #[test]
+    fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() {
+        let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode");
+        let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode");
+        assert!(matches!(s3, TierType::S3));
+        assert!(matches!(gcs, TierType::GCS));
+        assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#);
+        assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#);
+    }
+
+    #[test]
+    fn azure_service_principal_accepts_canonical_madmin_field_names() {
+        for field in ["TenantID", "ClientID", "ClientSecret"] {
+            let mut sp_auth = serde_json::Map::new();
+            sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string()));
+            let config: TierConfig = serde_json::from_value(serde_json::json!({
+                "type": "azure",
+                "Name": "COLD-AZURE",
+                "azure": {
+                    "name": "COLD-AZURE",
+                    "endpoint": "https://azure.example.invalid",
+                    "accessKey": "account",
+                    "secretKey": "key",
+                    "bucket": "archive",
+                    "SPAuth": sp_auth
+                }
+            }))
+            .expect("mixed RustFS/madmin Azure payload should decode");
+            let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth;
+
+            assert!(
+                !sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(),
+                "canonical {field} must not be silently discarded"
+            );
+        }
+    }
+
     fn wasabi_config() -> TierWasabi {
         TierWasabi {
             name: "COLD-WASABI".to_string(),
@@ -838,9 +897,14 @@ mod tests {
         let config = TierConfig {
             tier_type: TierType::Wasabi,
             wasabi: Some(wasabi_config()),
+            rustfs: Some(TierRustFS {
+                access_key: "inactive-access".to_string(),
+                secret_key: "inactive-secret".to_string(),
+                ..Default::default()
+            }),
             ..Default::default()
         };
-        let redacted = config.clone();
+        let redacted = config.redacted();
         assert_eq!(
             redacted
                 .wasabi
@@ -849,21 +913,46 @@ mod tests {
                 .secret_key,
             "REDACTED"
         );
+        assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider");
+        let cloned = config.clone();
+        assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED");
+        assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics");
+        let preserved = config.clone_with_credentials();
         assert_eq!(
-            config
-                .clone_with_credentials()
+            preserved
                 .wasabi
                 .as_ref()
-                .expect("credential-bearing Wasabi payload should remain")
+                .expect("credential-bearing Wasabi snapshot should remain")
                 .secret_key,
             "secret"
         );
+        assert_eq!(
+            preserved
+                .rustfs
+                .expect("credential-bearing snapshots should preserve inactive provider data")
+                .secret_key,
+            "inactive-secret"
+        );
 
         let mut debug_config = wasabi_config();
         debug_config.secret_key = "wasabi-debug-secret-value".to_string();
         let debug = format!("{debug_config:?}");
         assert!(debug.contains("REDACTED"));
         assert!(!debug.contains("wasabi-debug-secret-value"));
+
+        let debug = format!(
+            "{:?}",
+            TierConfig {
+                tier_type: TierType::RustFS,
+                rustfs: Some(TierRustFS {
+                    secret_key: "rustfs-debug-secret-value".to_string(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }
+        );
+        assert!(debug.contains("REDACTED"));
+        assert!(!debug.contains("rustfs-debug-secret-value"));
     }
 
     #[test]
@@ -894,7 +983,7 @@ mod tests {
         assert_eq!(encoded, expected);
         let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
         assert!(matches!(decoded.tier_type, TierType::Wasabi));
-        let redacted = config.clone();
+        let redacted = config.redacted();
         assert_eq!(
             config
                 .wasabi
@@ -916,4 +1005,87 @@ mod tests {
             "REDACTED"
         );
     }
+
+    #[test]
+    fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() {
+        let cases = [
+            (
+                "s3",
+                TierConfig {
+                    tier_type: TierType::S3,
+                    s3: Some(TierS3 {
+                        secret_key: "s3-secret-bytes".to_string(),
+                        aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["s3-secret-bytes", "/var/run/s3-private-token"],
+            ),
+            (
+                "gcs",
+                TierConfig {
+                    tier_type: TierType::GCS,
+                    gcs: Some(TierGCS {
+                        creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["gcs-private-key-bytes"],
+            ),
+            (
+                "azure",
+                TierConfig {
+                    tier_type: TierType::Azure,
+                    azure: Some(TierAzure {
+                        secret_key: "azure-account-secret-bytes".to_string(),
+                        sp_auth: ServicePrincipalAuth {
+                            client_secret: "azure-client-secret-bytes".to_string(),
+                            ..Default::default()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["azure-account-secret-bytes", "azure-client-secret-bytes"],
+            ),
+        ];
+
+        for (provider, config, secrets) in cases {
+            let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize");
+            let debug = format!("{config:?}");
+            assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted");
+            assert!(
+                debug.contains(TIER_CREDENTIAL_REDACTED),
+                "{provider} Debug output should be visibly redacted"
+            );
+            for secret in secrets {
+                assert!(!api.contains(secret), "{provider} API output exposed credential bytes");
+                assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes");
+            }
+        }
+    }
+
+    #[test]
+    fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() {
+        let config = TierConfig {
+            tier_type: TierType::Azure,
+            azure: Some(TierAzure {
+                secret_key: "azure-account-secret-bytes".to_string(),
+                sp_auth: ServicePrincipalAuth::default(),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize");
+        let debug = format!("{config:?}");
+
+        assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED);
+        assert_eq!(api["azure"]["spAuth"]["client_secret"], "");
+        assert!(debug.contains("client_secret: \"\""));
+        assert!(!debug.contains("client_secret: \"REDACTED\""));
+        assert!(!debug.contains("azure-account-secret-bytes"));
+    }
 }

crates/ecstore/src/services/tier/warm_backend.rs

@@ -20,7 +20,7 @@
 
 use crate::error::is_err_bucket_not_found;
 use crate::services::tier::{
-    tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
+    tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
     tier_config::{TierConfig, TierType},
     tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
     warm_backend_aliyun::WarmBackendAliyun,
@@ -55,18 +55,21 @@ use s3s::header::{
 };
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Duration;
 use time::OffsetDateTime;
 use time::format_description::well_known::{Rfc2822, Rfc3339};
+use tokio::io::AsyncReadExt;
 use tracing::{info, warn};
 
 pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
 
-const PROBE_OBJECT: &str = "probeobject";
-
 /// Largest object the S3-compatible warm backends accept for a multipart put.
 pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
 /// Part-count ceiling S3-compatible services impose on a multipart upload.
 pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
+pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
+const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1);
+const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1);
 
 #[derive(Default)]
 pub struct WarmBackendGetOpts {
@@ -260,6 +263,23 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
     pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
 }
 
+/// Return the authority format accepted by `TransitionClient::new` while
+/// retaining an explicitly configured port. `url::Url::host_str()` omits the
+/// brackets needed when an IPv6 literal is combined with a port.
+pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Error> {
+    let host = url
+        .host_str()
+        .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+    let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
+    if host.starts_with('[') && host.ends_with(']') {
+        Ok(format!("{host}:{port}"))
+    } else if host.contains(':') {
+        Ok(format!("[{host}]:{port}"))
+    } else {
+        Ok(format!("{host}:{port}"))
+    }
+}
+
 /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
 ///
 /// Credential, bucket, and endpoint validation run in this order because the
@@ -298,17 +318,11 @@ pub(crate) async fn new_s3_compatible_warm_backend(
         bucket_lookup: params.bucket_lookup,
         ..Default::default()
     };
-    let scheme = u.scheme();
-    let default_port = if scheme == "https" { 443 } else { 80 };
-    let host = u
-        .host_str()
-        .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
-    // Runs after the host-presence check above (not immediately after Url::parse) so a
-    // host-less endpoint still reports this constructor's own "missing host" text instead of
-    // validate_endpoint's differently-worded rejection for the same input.
+    let endpoint = endpoint_authority(&u)?;
+    // Run the SSRF guard after the host-presence check so a host-less endpoint
+    // keeps this constructor's stable error text.
     (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
-    let client =
-        TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?;
+    let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
 
     let client = Arc::new(client);
     let core = TransitionCore(Arc::clone(&client));
@@ -451,25 +465,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
     }
 }
 
-pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
-    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
-    w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
-    let remote_version_id = w
-        .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
-        .await
-        .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+async fn remove_discovered_probe_candidate(
+    w: &WarmBackendImpl,
+    probe_object: &str,
+    candidate: TransitionCandidateProbe,
+) -> Result<bool, std::io::Error> {
+    match candidate {
+        TransitionCandidateProbe::Missing => Ok(false),
+        TransitionCandidateProbe::VersionedPresent(remote_version_id) => {
+            w.remove_exact(probe_object, &remote_version_id).await?;
+            Ok(true)
+        }
+        TransitionCandidateProbe::UnversionedPresent => {
+            w.remove(probe_object, "").await?;
+            Ok(true)
+        }
+        TransitionCandidateProbe::Ambiguous => {
+            Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions"))
+        }
+        TransitionCandidateProbe::Unsupported => Err(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            "remote tier cannot discover the outcome of a probe PUT",
+        )),
+    }
+}
 
-    if w.validate_remote_version_id(&remote_version_id).is_err() {
-        w.remove_exact(PROBE_OBJECT, &remote_version_id)
+async fn compensate_uncertain_probe_put(
+    w: &WarmBackendImpl,
+    probe_object: &str,
+    settle_deadline: tokio::time::Instant,
+) -> Result<(), std::io::Error> {
+    let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT;
+    let mut removed_any = false;
+    while tokio::time::Instant::now() < settle_deadline {
+        let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await {
+            Ok(candidate) => candidate?,
+            Err(_) => break,
+        };
+        if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any {
+            break;
+        }
+        removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
             .await
-            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
-        return Err(ERR_TIER_INVALID_CONFIG.clone());
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??;
+
+        let now = tokio::time::Instant::now();
+        if now >= settle_deadline {
+            break;
+        }
+        tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await;
     }
 
-    let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
-    let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
-    //xhttp.DrainBody(r);
-    if read_result.is_err() || remove_result.is_err() {
+    let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
+        .await
+        .map_err(|_| {
+            std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state")
+        })??;
+    if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
+        .await
+        .map_err(|_| {
+            std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate")
+        })??
+    {
+        return Ok(());
+    }
+
+    let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
+        .await
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??;
+    match final_candidate {
+        TransitionCandidateProbe::Missing => Ok(()),
+        _ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")),
+    }
+}
+
+fn probe_cleanup_incomplete_error() -> AdminError {
+    let mut err = ERR_TIER_PERM_ERR.clone();
+    err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string();
+    err
+}
+
+async fn check_warm_backend_with_deadlines(
+    w: Option<&WarmBackendImpl>,
+    deadline: tokio::time::Instant,
+    cleanup_deadline: tokio::time::Instant,
+) -> Result<(), AdminError> {
+    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
+    let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4());
+    let timeout_error = || {
+        let mut err = ERR_TIER_BACKEND_IN_USE.clone();
+        err.message = "Timed out validating the remote tier mutation".to_string();
+        err
+    };
+    tokio::time::timeout_at(deadline, w.validate())
+        .await
+        .map_err(|_| timeout_error())?
+        .map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
+    let put_result =
+        tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await;
+    let remote_version_id = match put_result {
+        Ok(Ok(remote_version_id)) => remote_version_id,
+        Ok(Err(_)) => {
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => ERR_TIER_PERM_ERR.clone(),
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+        Err(_) => {
+            let err = timeout_error();
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => err,
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+    };
+
+    // S3-family backends do not replay a failed request before returning `Ok`,
+    // while GCS discovers every matching generation. The authoritative probe
+    // below therefore closes the acknowledged-PUT path; only an error or
+    // timeout needs the longer visibility reconciliation above.
+    let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await {
+        Ok(Ok(candidate)) => candidate,
+        Ok(Err(_)) | Err(_) => {
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+    };
+    let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok();
+    let response_matches_candidate = match &authoritative_candidate {
+        TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(),
+        TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id,
+        TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false,
+    };
+    if !response_version_is_valid || !response_matches_candidate {
+        return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+            Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
+            Err(_) => probe_cleanup_incomplete_error(),
+        });
+    }
+
+    let read_result = tokio::time::timeout_at(deadline, async {
+        let mut reader = w
+            .get(
+                &probe_object,
+                &remote_version_id,
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 7,
+                },
+            )
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        let mut body = Vec::new();
+        reader
+            .take(7)
+            .read_to_end(&mut body)
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        if body != b"RustFS" {
+            return Err(ERR_TIER_PERM_ERR.clone());
+        }
+        Ok(())
+    })
+    .await
+    .map_err(|_| timeout_error())
+    .and_then(|result| result);
+    let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async {
+        if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? {
+            return Err(std::io::Error::other("remote tier probe disappeared before cleanup"));
+        }
+        match w.probe_transition_candidate(&probe_object).await? {
+            TransitionCandidateProbe::Missing => Ok(()),
+            _ => Err(std::io::Error::other("remote tier probe remained after cleanup")),
+        }
+    })
+    .await;
+    if !matches!(cleanup_result, Ok(Ok(()))) {
+        return Err(probe_cleanup_incomplete_error());
+    }
+    if let Err(err) = read_result {
         //if is_err_bucket_not_found(&err) {
         //    return Err(ERR_TIER_BUCKET_NOT_FOUND);
         //}
@@ -477,12 +653,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
             return Err(ERR_TIER_MISSING_CREDENTIALS);
         }*/
         //else {
-        return Err(ERR_TIER_PERM_ERR.clone());
+        return Err(err);
         //}
     }
     Ok(())
 }
 
+/// Validate a backend using a caller-owned deadline while retaining a bounded
+/// reconciliation window for an uncertain probe PUT. The validation future is
+/// kept alive through cleanup so an outer timeout cannot abandon the remote
+/// probe object.
+pub(crate) async fn check_warm_backend_until(
+    w: Option<&WarmBackendImpl>,
+    deadline: tokio::time::Instant,
+) -> Result<(), AdminError> {
+    check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await
+}
+
+pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
+    let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT;
+    check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await
+}
+
 pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
     let mut d: Option<WarmBackendImpl> = None;
     match tier.tier_type {
@@ -701,7 +893,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
     let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
 
     if probe {
-        d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
+        check_warm_backend(Some(&d)).await?;
     }
     Ok(d)
 }
@@ -754,6 +946,7 @@ pub(crate) async fn new_transition_candidate_reconciler(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::services::tier::test_util::{MockWarmBackend, MockWarmOp};
     use crate::services::tier::tier_config::TierWasabi;
     use std::sync::{
         Arc,
@@ -920,13 +1113,38 @@ mod tests {
 
     struct RejectingProbeVersionBackend {
         gets: Arc<AtomicUsize>,
+        present: Arc<std::sync::atomic::AtomicBool>,
         removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        returned_version: String,
     }
 
     struct RecordingProbeBackend {
         get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        present: Arc<std::sync::atomic::AtomicBool>,
         removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        remove_clears_candidate: bool,
         fail_get: bool,
+        body: ProbeBody,
+    }
+
+    struct HangingProbePutBackend {
+        put_started: Arc<tokio::sync::Notify>,
+        present: Arc<std::sync::atomic::AtomicBool>,
+        probes: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    struct LateVisibleProbeBackend {
+        visible_at: tokio::time::Instant,
+        removed: Arc<std::sync::atomic::AtomicBool>,
+        probes: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    #[derive(Clone, Copy)]
+    enum ProbeBody {
+        Exact,
+        Mismatch,
     }
 
     #[async_trait::async_trait]
@@ -976,7 +1194,7 @@ mod tests {
         }
 
         async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
-            Ok(uuid::Uuid::nil().to_string())
+            Ok(self.returned_version.clone())
         }
 
         async fn put_with_meta(
@@ -999,10 +1217,19 @@ mod tests {
         }
 
         async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.present.store(false, Ordering::SeqCst);
             self.removed_versions.lock().await.push(rv.to_string());
             Ok(())
         }
 
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
         async fn in_use(&self) -> Result<bool, std::io::Error> {
             Ok(false)
         }
@@ -1029,15 +1256,121 @@ mod tests {
             if self.fail_get {
                 Err(std::io::Error::other("probe GET failed"))
             } else {
-                Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
+                match self.body {
+                    ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))),
+                    ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))),
+                }
             }
         }
 
         async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            if self.remove_clears_candidate {
+                self.present.store(false, Ordering::SeqCst);
+            }
             self.removed_versions.lock().await.push(rv.to_string());
             Ok(())
         }
 
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for HangingProbePutBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            self.put_started.notify_one();
+            std::future::pending().await
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("GET must not run after a timed out probe PUT"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.present.store(false, Ordering::SeqCst);
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            self.probes.fetch_add(1, Ordering::SeqCst);
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for LateVisibleProbeBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            Err(std::io::Error::new(
+                std::io::ErrorKind::ConnectionReset,
+                "probe PUT response was lost before the object became visible",
+            ))
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("GET must not run after a lost probe PUT response"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.removed.store(true, Ordering::SeqCst);
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            self.probes.fetch_add(1, Ordering::SeqCst);
+            if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
         async fn in_use(&self) -> Result<bool, std::io::Error> {
             Ok(false)
         }
@@ -1098,13 +1431,15 @@ mod tests {
         assert_eq!(probe, TransitionCandidateProbe::Unsupported);
     }
 
-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
     async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
         let gets = Arc::new(AtomicUsize::new(0));
         let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
         let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
             gets: gets.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
             removed_versions: removed_versions.clone(),
+            returned_version: uuid::Uuid::nil().to_string(),
         });
 
         let err = check_warm_backend(Some(&backend))
@@ -1113,7 +1448,27 @@ mod tests {
 
         assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
         assert_eq!(gets.load(Ordering::SeqCst), 0);
-        assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() {
+        let gets = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
+            gets: gets.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            returned_version: String::new(),
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("an empty PUT version must not read or generically delete a versioned object");
+
+        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
+        assert_eq!(gets.load(Ordering::SeqCst), 0);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
     }
 
     #[tokio::test]
@@ -1122,8 +1477,11 @@ mod tests {
         let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
         let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
             get_versions: get_versions.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
             removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
             fail_get: false,
+            body: ProbeBody::Exact,
         });
 
         check_warm_backend(Some(&backend))
@@ -1140,8 +1498,11 @@ mod tests {
         let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
         let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
             get_versions: get_versions.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
             removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
             fail_get: true,
+            body: ProbeBody::Exact,
         });
 
         let err = check_warm_backend(Some(&backend))
@@ -1153,6 +1514,169 @@ mod tests {
         assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
     }
 
+    #[tokio::test]
+    async fn check_warm_backend_removes_probe_after_body_mismatch() {
+        let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions,
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
+            fail_get: false,
+            body: ProbeBody::Mismatch,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("a mismatched body should fail after cleanup");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() {
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: false,
+            fail_get: false,
+            body: ProbeBody::Exact,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("cleanup must not succeed while the deleted candidate remains visible");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(err.message.contains("cleanup is incomplete"));
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_reconciles_a_lost_put_response() {
+        let backend = MockWarmBackend::new();
+        backend.lose_next_put_response();
+        let driver: WarmBackendImpl = Box::new(backend.clone());
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("a lost probe PUT response must fail after compensation");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(backend.object_count().await, 0);
+        assert_eq!(backend.exact_remove_count(), 1);
+        let operations = backend.op_log().await;
+        let put = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Put { object } => Some(object),
+            _ => None,
+        });
+        let probe = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Probe { object } => Some(object),
+            _ => None,
+        });
+        let remove = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Remove { object } => Some(object),
+            _ => None,
+        });
+        let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else {
+            panic!("lost-response compensation should PUT, probe, and remove");
+        };
+        assert_eq!(put, probe);
+        assert_eq!(probe, remove);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_retries_until_a_late_put_becomes_visible() {
+        let probes = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend {
+            visible_at: tokio::time::Instant::now() + Duration::from_secs(5),
+            removed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            probes: probes.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("a late-visible probe PUT must still report the lost response");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(
+            probes.load(Ordering::SeqCst) > 5,
+            "reconciliation must not stop at the first Missing result"
+        );
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() {
+        for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] {
+            let backend = MockWarmBackend::new();
+            backend.set_transition_candidate_probe_override(Some(candidate)).await;
+            backend.lose_next_put_response();
+            let driver: WarmBackendImpl = Box::new(backend.clone());
+
+            let err = check_warm_backend(Some(&driver))
+                .await
+                .expect_err("an uncertain candidate must fail without a guessed delete");
+
+            assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+            assert!(err.message.contains("cleanup is incomplete"));
+            assert_eq!(backend.remove_count().await, 0);
+            assert_eq!(backend.object_count().await, 1);
+        }
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_reports_an_exact_cleanup_failure() {
+        let backend = MockWarmBackend::new();
+        backend.set_remove_failure(true);
+        backend.lose_next_put_response();
+        let driver: WarmBackendImpl = Box::new(backend.clone());
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("an exact cleanup failure must replace the ambiguous PUT error");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(err.message.contains("cleanup is incomplete"));
+        assert_eq!(backend.exact_remove_count(), 1);
+        assert_eq!(backend.object_count().await, 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_reconciles_a_timed_out_put() {
+        let put_started = Arc::new(tokio::sync::Notify::new());
+        let probes = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let driver: WarmBackendImpl = Box::new(HangingProbePutBackend {
+            put_started: put_started.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            probes: probes.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+        let check = check_warm_backend(Some(&driver));
+        tokio::pin!(check);
+        tokio::select! {
+            _ = put_started.notified() => {}
+            result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"),
+        }
+
+        tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await;
+        let err = check.await.expect_err("a timed out probe PUT must fail after compensation");
+
+        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
+        assert!(
+            probes.load(Ordering::SeqCst) > 1,
+            "timed-out PUT reconciliation must keep checking through the visibility window"
+        );
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
     #[tokio::test]
     async fn new_wasabi_backend_honors_probe_flag() {
         let tier = TierConfig {
@@ -1298,6 +1822,15 @@ mod tests {
         assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
     }
 
+    #[test]
+    fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() {
+        let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse");
+        assert_eq!(
+            endpoint_authority(&url).expect("the endpoint should have an authority"),
+            "[2001:db8::1]:9443"
+        );
+    }
+
     #[tokio::test]
     async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
         let mut params = s3_compatible_params("http://tier.example.com:9000");

crates/ecstore/src/services/tier/warm_backend_aliyun.rs

@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
     tier_config::TierAliyun,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendAliyun {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }

crates/ecstore/src/services/tier/warm_backend_azure.rs

@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
     tier_config::TierAzure,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,16 @@ impl WarmBackend for WarmBackendAzure {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        // Azure currently uses the shared S3/SigV4 transport, but its normal
+        // object path cannot persist exact remote versions across mixed
+        // RustFS releases. The mutation probe may still detect and precisely
+        // remove a versioned test object before rejecting that configuration.
+        self.0
+            .probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id")
+            .await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }
@@ -98,6 +108,23 @@ impl WarmBackend for WarmBackendAzure {
 mod tests {
     use super::*;
     use crate::services::tier::tier_config::TierAzure;
+    use rustfs_s3_client::{
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{Options, TransitionClient, TransitionCore},
+    };
+    use std::sync::Arc;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+            let read = stream.read(&mut buffer).await.expect("fixture request should be readable");
+            assert_ne!(read, 0, "connection closed before request headers were received");
+            request.extend_from_slice(&buffer[..read]);
+        }
+        String::from_utf8_lossy(&request).into_owned()
+    }
 
     /// The SSRF guard itself is exercised once, generically, in
     /// `warm_backend::tests` (see backlog#2040/backlog#2041 and
@@ -119,4 +146,85 @@ mod tests {
             Err(err) => assert!(err.to_string().contains("not allowed")),
         }
     }
+
+    #[tokio::test]
+    async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
+            let get_request = read_request_head(&mut get_stream).await;
+            get_stream
+                .write_all(
+                    b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx",
+                )
+                .await
+                .expect("fixture should write candidate GET response");
+
+            let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE");
+            let delete_request = read_request_head(&mut delete_stream).await;
+            delete_stream
+                .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .await
+                .expect("fixture should write exact DELETE response");
+            (get_request, delete_request)
+        });
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "azure",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        let backend = WarmBackendAzure(WarmBackendS3 {
+            core: TransitionCore(Arc::clone(&client)),
+            client,
+            bucket: "bucket".to_string(),
+            prefix: String::new(),
+            storage_class: String::new(),
+        });
+        assert!(
+            !backend.0.client.provider_version_capabilities().exact_get_delete,
+            "probe-only version discovery must not change Azure's persisted data-path contract"
+        );
+
+        let candidate = backend
+            .probe_transition_candidate("probe")
+            .await
+            .expect("Azure candidate should be discovered");
+        assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string()));
+        backend
+            .remove_exact("probe", "azure-version")
+            .await
+            .expect("Azure candidate should be deleted by exact version");
+
+        let (get_request, delete_request) = fixture.await.expect("fixture should join");
+        assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n"));
+        assert!(
+            delete_request
+                .lines()
+                .next()
+                .is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version "))
+        );
+    }
 }

crates/ecstore/src/services/tier/warm_backend_gcs.rs

@@ -18,13 +18,13 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
 use std::io::{Error, ErrorKind};
 use std::sync::Arc;
 
 use bytes::Bytes;
-use google_cloud_auth::credentials::Credentials;
-use google_cloud_auth::credentials::user_account::Builder;
+use google_cloud_auth::credentials::service_account::Builder;
 use google_cloud_storage as gcs;
 use google_cloud_storage::client::Storage;
 use google_cloud_storage::client::StorageControl;
@@ -32,7 +32,7 @@ use std::convert::TryFrom;
 
 use crate::services::tier::{
     tier_config::TierGCS,
-    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
 };
 use rustfs_s3_client::{
     admin_handler_utils::AdminError,
@@ -43,6 +43,7 @@ use rustfs_utils::egress::validate_outbound_url;
 use tracing::warn;
 
 const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MAX_GCS_CANDIDATE_PAGES: usize = 64;
 
 fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
     if remote_version.is_empty() {
@@ -57,6 +58,85 @@ fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
     Ok(Some(generation))
 }
 
+fn append_gcs_chunk<E: std::fmt::Display>(
+    contents: &mut Vec<u8>,
+    chunk: Result<Bytes, E>,
+    max_response_bytes: Option<usize>,
+) -> std::io::Result<()> {
+    let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?;
+    if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidData,
+            "GCS object response exceeded the configured byte limit",
+        ));
+    }
+    contents.extend_from_slice(&chunk);
+    Ok(())
+}
+
+fn gcs_bucket_resource_name(bucket: &str) -> String {
+    format!("projects/_/buckets/{bucket}")
+}
+
+struct GcsCandidateObject {
+    name: String,
+    generation: i64,
+}
+
+struct GcsCandidatePage {
+    objects: Vec<GcsCandidateObject>,
+    next_page_token: String,
+}
+
+async fn probe_exact_gcs_candidate<F, Fut>(
+    remote_object: &str,
+    mut fetch_page: F,
+) -> Result<TransitionCandidateProbe, std::io::Error>
+where
+    F: FnMut(String) -> Fut,
+    Fut: Future<Output = Result<GcsCandidatePage, std::io::Error>>,
+{
+    let mut page_token = String::new();
+    let mut seen_page_tokens = HashSet::new();
+    let mut generation = None;
+    let mut pages_seen = 0_usize;
+
+    loop {
+        pages_seen += 1;
+        if pages_seen > MAX_GCS_CANDIDATE_PAGES {
+            return Err(std::io::Error::new(
+                ErrorKind::InvalidData,
+                "GCS candidate listing exceeded the page limit",
+            ));
+        }
+        let response = fetch_page(page_token.clone()).await?;
+        for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) {
+            if candidate.generation <= 0 {
+                return Err(std::io::Error::new(
+                    ErrorKind::InvalidData,
+                    "GCS candidate listing returned a non-positive generation",
+                ));
+            }
+            if generation.replace(candidate.generation).is_some() {
+                return Ok(TransitionCandidateProbe::Ambiguous);
+            }
+        }
+
+        if response.next_page_token.is_empty() {
+            break;
+        }
+        if !seen_page_tokens.insert(response.next_page_token.clone()) {
+            return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token"));
+        }
+        page_token = response.next_page_token;
+    }
+
+    Ok(match generation {
+        Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()),
+        None => TransitionCandidateProbe::Missing,
+    })
+}
+
 pub struct WarmBackendGCS {
     pub client: Arc<Storage>,
     pub control: Arc<StorageControl>,
@@ -80,8 +160,8 @@ impl WarmBackendGCS {
                 .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
         }
 
-        let authorized_user = serde_json::from_str(&conf.creds)?;
-        let credentials = Builder::new(authorized_user)
+        let service_account = serde_json::from_str(&conf.creds)?;
+        let credentials = Builder::new(service_account)
             //.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
             //.with_backoff_policy(backoff)
             .build()
@@ -98,7 +178,11 @@ impl WarmBackendGCS {
         let client = Arc::new(client);
         // Control-plane client: the data-plane `Storage` client cannot delete or list objects;
         // delete_object/list_objects live on StorageControl.
-        let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else {
+        let mut control_builder = StorageControl::builder().with_credentials(credentials);
+        if !conf.endpoint.is_empty() {
+            control_builder = control_builder.with_endpoint(conf.endpoint.clone());
+        }
+        let Ok(control) = control_builder.build().await else {
             return Err(std::io::Error::other("StorageControl::builder error"));
         };
         let control = Arc::new(control);
@@ -136,9 +220,10 @@ impl WarmBackend for WarmBackendGCS {
             ReaderImpl::Body(content_body) => content_body.to_vec(),
             ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
         };
+        let bucket = gcs_bucket_resource_name(&self.bucket);
         let Ok(res) = Box::pin(
             self.client
-                .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
+                .write_object(&bucket, &self.get_dest(object), Bytes::from(d))
                 .send_buffered(),
         )
         .await
@@ -154,7 +239,9 @@ impl WarmBackend for WarmBackendGCS {
     }
 
     async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
-        let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
+        let bucket = gcs_bucket_resource_name(&self.bucket);
+        let mut req = self.client.read_object(&bucket, &self.get_dest(object));
+        let mut max_response_bytes = None;
         if let Some(generation) = parse_generation(rv)? {
             req = req.set_generation(generation);
         }
@@ -170,6 +257,11 @@ impl WarmBackend for WarmBackendGCS {
                 .length
                 .try_into()
                 .map_err(|_| std::io::Error::other("invalid range: negative length"))?;
+            max_response_bytes = Some(
+                opts.length
+                    .try_into()
+                    .map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?,
+            );
             req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
         }
 
@@ -177,8 +269,8 @@ impl WarmBackend for WarmBackendGCS {
             return Err(std::io::Error::other("read_object error"));
         };
         let mut contents = Vec::new();
-        while let Ok(Some(chunk)) = reader.next().await.transpose() {
-            contents.extend_from_slice(&chunk);
+        while let Some(chunk) = reader.next().await {
+            append_gcs_chunk(&mut contents, chunk, max_response_bytes)?;
         }
         Ok(ReadCloser::new(std::io::Cursor::new(contents)))
     }
@@ -190,7 +282,7 @@ impl WarmBackend for WarmBackendGCS {
         let mut req = self
             .control
             .delete_object()
-            .set_bucket(format!("projects/_/buckets/{}", self.bucket))
+            .set_bucket(gcs_bucket_resource_name(&self.bucket))
             .set_object(self.get_dest(object));
         if let Some(generation) = parse_generation(rv)? {
             req = req.set_generation(generation);
@@ -199,13 +291,47 @@ impl WarmBackend for WarmBackendGCS {
         Ok(())
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
+        let parent = gcs_bucket_resource_name(&self.bucket);
+        probe_exact_gcs_candidate(&remote_object, |page_token| {
+            let control = self.control.clone();
+            let parent = parent.clone();
+            let prefix = remote_object.clone();
+            async move {
+                let response = control
+                    .list_objects()
+                    .set_parent(parent)
+                    .set_prefix(prefix)
+                    .set_versions(true)
+                    .set_page_size(2)
+                    .set_page_token(page_token)
+                    .send()
+                    .await
+                    .map_err(|err| std::io::Error::other(err.to_string()))?;
+                Ok(GcsCandidatePage {
+                    objects: response
+                        .objects
+                        .into_iter()
+                        .map(|candidate| GcsCandidateObject {
+                            name: candidate.name,
+                            generation: candidate.generation,
+                        })
+                        .collect(),
+                    next_page_token: response.next_page_token,
+                })
+            }
+        })
+        .await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         // Scope the listing to this tier's prefix (matching the other warm backends) and only
         // need to know whether a single object exists.
         let resp = self
             .control
             .list_objects()
-            .set_parent(format!("projects/_/buckets/{}", self.bucket))
+            .set_parent(gcs_bucket_resource_name(&self.bucket))
             .set_prefix(self.prefix.clone())
             .set_page_size(1)
             .send()
@@ -218,10 +344,126 @@ impl WarmBackend for WarmBackendGCS {
 
 #[cfg(test)]
 mod tests {
+    use super::GcsCandidateObject;
+    use super::GcsCandidatePage;
+    use super::MAX_GCS_CANDIDATE_PAGES;
     use super::WarmBackendGCS;
+    use super::append_gcs_chunk;
+    use super::gcs_bucket_resource_name;
     use super::parse_generation;
+    use super::probe_exact_gcs_candidate;
     use crate::services::tier::tier_config::TierGCS;
+    use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts};
+    use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
+    use google_cloud_storage::client::{Storage, StorageControl};
     use std::io::ErrorKind;
+    use std::sync::Arc;
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    async fn serve_data_plane_fixture(listener: TcpListener) -> Vec<String> {
+        let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#;
+        let responses = [
+            format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}",
+                upload_body.len()
+            ),
+            "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!"
+                .to_string(),
+            "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!"
+                .to_string(),
+        ];
+        let mut requests = Vec::new();
+
+        for response in responses {
+            let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request");
+            let mut request = Vec::new();
+            loop {
+                let mut chunk = [0_u8; 1024];
+                let count = stream
+                    .read(&mut chunk)
+                    .await
+                    .expect("the GCS fixture should read request headers");
+                if count == 0 {
+                    break;
+                }
+                request.extend_from_slice(&chunk[..count]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let header_end = request
+                .windows(4)
+                .position(|window| window == b"\r\n\r\n")
+                .map(|position| position + 4)
+                .expect("the GCS fixture should receive complete request headers");
+            let headers = String::from_utf8_lossy(&request[..header_end]);
+            if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) {
+                stream
+                    .write_all(b"HTTP/1.1 100 Continue\r\n\r\n")
+                    .await
+                    .expect("the GCS fixture should acknowledge 100-continue");
+            }
+            let content_length = headers
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().expect("content-length should be numeric"))
+                })
+                .unwrap_or_default();
+            while request.len() < header_end.saturating_add(content_length) {
+                let mut chunk = [0_u8; 1024];
+                let count = stream
+                    .read(&mut chunk)
+                    .await
+                    .expect("the GCS fixture should read the request body");
+                if count == 0 {
+                    break;
+                }
+                request.extend_from_slice(&chunk[..count]);
+            }
+            requests.push(String::from_utf8_lossy(&request).into_owned());
+            stream
+                .write_all(response.as_bytes())
+                .await
+                .expect("the GCS fixture should write its response");
+        }
+
+        requests
+    }
+
+    fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage {
+        GcsCandidatePage {
+            objects: objects
+                .iter()
+                .map(|(name, generation)| GcsCandidateObject {
+                    name: (*name).to_string(),
+                    generation: *generation,
+                })
+                .collect(),
+            next_page_token: next_page_token.to_string(),
+        }
+    }
+
+    async fn probe_candidate_pages(
+        remote_object: &str,
+        pages: Vec<GcsCandidatePage>,
+    ) -> (Result<TransitionCandidateProbe, std::io::Error>, Vec<String>) {
+        let mut pages = pages.into_iter();
+        let mut requested_tokens = Vec::new();
+        let result = probe_exact_gcs_candidate(remote_object, |page_token| {
+            requested_tokens.push(page_token);
+            std::future::ready(
+                pages
+                    .next()
+                    .ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")),
+            )
+        })
+        .await;
+        (result, requested_tokens)
+    }
 
     #[test]
     fn generation_parser_preserves_exact_numeric_versions() {
@@ -241,6 +483,243 @@ mod tests {
         }
     }
 
+    #[test]
+    fn body_collection_propagates_an_error_after_a_complete_prefix() {
+        let mut contents = Vec::new();
+        append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7))
+            .expect("the prefix chunk should be collected");
+        let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7))
+            .expect_err("a trailing stream error must not be mistaken for EOF");
+
+        assert_eq!(contents, b"RustFS");
+        assert!(err.to_string().contains("trailing stream failure"));
+    }
+
+    #[test]
+    fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() {
+        let mut contents = Vec::new();
+        let err = append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7))
+            .expect_err("the GCS collection layer must reject an oversized probe response");
+
+        assert!(contents.is_empty());
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_finds_exact_name_on_first_or_later_page() {
+        let (first, first_tokens) =
+            probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await;
+        assert_eq!(
+            first.expect("an exact first-page object should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("7".to_string())
+        );
+        assert_eq!(first_tokens, [""]);
+
+        let (later, later_tokens) = probe_candidate_pages(
+            "prefix/object",
+            vec![
+                candidate_page(&[("prefix/object-shadow", 8)], "next"),
+                candidate_page(&[("prefix/object", 9)], ""),
+            ],
+        )
+        .await;
+        assert_eq!(
+            later.expect("an exact later-page object should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("9".to_string())
+        );
+        assert_eq!(later_tokens, ["", "next"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_ignores_non_exact_prefix_matches() {
+        let (probe, _) = probe_candidate_pages(
+            "prefix/object",
+            vec![candidate_page(
+                &[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)],
+                "",
+            )],
+        )
+        .await;
+
+        assert_eq!(
+            probe.expect("prefix-only matches should not hide the exact object"),
+            TransitionCandidateProbe::VersionedPresent("7".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() {
+        let (probe, _) = probe_candidate_pages(
+            "prefix/object",
+            vec![
+                candidate_page(&[("prefix/object", 7)], "next"),
+                candidate_page(&[("prefix/object", 8)], ""),
+            ],
+        )
+        .await;
+
+        assert_eq!(
+            probe.expect("multiple exact generations should produce a conservative result"),
+            TransitionCandidateProbe::Ambiguous
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_reports_missing_without_an_exact_name() {
+        let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await;
+
+        assert_eq!(
+            probe.expect("a complete listing without an exact name should be definitive"),
+            TransitionCandidateProbe::Missing
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_non_positive_generations() {
+        for generation in [0, -1] {
+            let (probe, _) =
+                probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await;
+            let err = probe.expect_err("a non-positive GCS generation must fail closed");
+            assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}");
+        }
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_a_page_token_that_does_not_advance() {
+        let (probe, requested_tokens) =
+            probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await;
+
+        let err = probe.expect_err("a repeated GCS page token must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens, ["", "next"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() {
+        let (probe, requested_tokens) = probe_candidate_pages(
+            "prefix/object",
+            vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")],
+        )
+        .await;
+
+        let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens, ["", "a", "b"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_an_unbounded_unique_token_chain() {
+        let pages = (0..MAX_GCS_CANDIDATE_PAGES)
+            .map(|index| candidate_page(&[], &format!("token-{index}")))
+            .collect();
+        let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await;
+
+        let err = probe.expect_err("an unbounded unique page-token chain must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES);
+    }
+
+    #[tokio::test]
+    async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() {
+        let listener = TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("the GCS fixture should bind a loopback port");
+        let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address"));
+        let fixture = tokio::spawn(serve_data_plane_fixture(listener));
+        let credentials = Anonymous::new().build();
+        let client = Storage::builder()
+            .with_endpoint(endpoint.clone())
+            .with_credentials(credentials.clone())
+            .build()
+            .await
+            .expect("the GCS data client should build");
+        let control = StorageControl::builder()
+            .with_endpoint(endpoint)
+            .with_credentials(credentials)
+            .build()
+            .await
+            .expect("the GCS control client should build");
+        let backend = WarmBackendGCS {
+            client: Arc::new(client),
+            control: Arc::new(control),
+            bucket: "tier-bucket".to_string(),
+            prefix: String::new(),
+        };
+
+        let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async {
+            let version = backend
+                .put(
+                    "probe",
+                    rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")),
+                    6,
+                )
+                .await
+                .expect("a plain configured bucket should reach the GCS upload transport");
+            let mut reader = backend
+                .get(
+                    "probe",
+                    &version,
+                    WarmBackendGetOpts {
+                        start_offset: 0,
+                        length: 7,
+                    },
+                )
+                .await
+                .expect("a plain configured bucket should reach the GCS read transport");
+            let mut body = Vec::new();
+            reader
+                .read_to_end(&mut body)
+                .await
+                .expect("the fixture body should be readable");
+            let oversized_error = match backend
+                .get(
+                    "probe",
+                    &version,
+                    WarmBackendGetOpts {
+                        start_offset: 0,
+                        length: 7,
+                    },
+                )
+                .await
+            {
+                Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"),
+                Err(err) => err,
+            };
+            let requests = fixture.await.expect("the GCS fixture task should finish");
+            (version, body, oversized_error.kind(), requests)
+        })
+        .await
+        .expect("the GCS data-plane requests should not be rejected before transport");
+
+        assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket");
+        assert_eq!(version, "123");
+        assert_eq!(body, b"RustFS!");
+        assert_eq!(oversized_error_kind, ErrorKind::InvalidData);
+        assert!(
+            requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"),
+            "unexpected upload request line: {}",
+            requests[0].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
+            "unexpected read request line: {}",
+            requests[1].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
+            "the GCS probe read must preserve its seven-byte range"
+        );
+        assert!(
+            requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
+            "unexpected oversized read request line: {}",
+            requests[2].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
+            "the oversized response must be fetched under the same seven-byte request boundary"
+        );
+    }
+
     #[tokio::test]
     async fn new_rejects_loopback_endpoint_before_credential_setup() {
         let conf = TierGCS {

crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs

@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
     tier_config::TierHuaweicloud,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendHuaweicloud {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -26,11 +26,12 @@ use crate::services::tier::{
     tier_config::TierS3,
     warm_backend::{
         TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
-        build_transition_put_options,
+        build_transition_put_options, endpoint_authority,
     },
 };
 use http::HeaderMap;
 use rustfs_s3_client::{
+    api_error_response::to_error_response,
     api_get_options::GetObjectOptions,
     api_list::ListObjectsOptions,
     api_put_object::PutObjectOptions,
@@ -43,7 +44,7 @@ use rustfs_s3_client::{
 };
 use rustfs_utils::egress::validate_outbound_url;
 use rustfs_utils::path::SLASH_SEPARATOR;
-use s3s::dto::BucketVersioningStatus;
+use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
 
 pub struct WarmBackendS3 {
     pub client: Arc<TransitionClient>,
@@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBu
     })
 }
 
+fn bounded_get_range(opts: &WarmBackendGetOpts) -> Result<Option<(i64, i64)>, std::io::Error> {
+    if opts.start_offset < 0 || opts.length <= 0 {
+        return Ok(None);
+    }
+    usize::try_from(opts.length)
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?;
+    let end_offset = opts
+        .start_offset
+        .checked_add(opts.length - 1)
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?;
+    Ok(Some((opts.start_offset, end_offset)))
+}
+
 impl WarmBackendS3 {
     pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
         Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
@@ -132,10 +146,8 @@ impl WarmBackendS3 {
             bucket_lookup,
             ..Default::default()
         };
-        let host = u
-            .host()
-            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
-        let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
+        let endpoint = endpoint_authority(&u)?;
+        let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
 
         let client = Arc::new(client);
         let core = TransitionCore(Arc::clone(&client));
@@ -177,10 +189,8 @@ impl WarmBackendS3 {
         if !rv.is_empty() {
             gopts.version_id = rv.to_string();
         }
-        if opts.start_offset >= 0 && opts.length > 0 {
-            gopts
-                .set_range(opts.start_offset, opts.start_offset + opts.length - 1)
-                .map_err(std::io::Error::other)?;
+        if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? {
+            gopts.set_range(start_offset, end_offset)?;
         }
         let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
         Ok((headers, reader))
@@ -191,36 +201,64 @@ impl WarmBackendS3 {
         remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
     }
 
-    async fn probe_transition_candidate_versions(
+    async fn probe_current_transition_candidate_with_header(
         &self,
         object: &str,
-        bucket_versioning: RemoteBucketVersioning,
+        raw_version_header: Option<&'static str>,
     ) -> Result<TransitionCandidateProbe, std::io::Error> {
-        let remote_object = self.get_dest(object);
-        let mut opts = ListObjectsOptions::default();
-        opts.set("prefix", &remote_object);
-        opts.set("max-keys", "1000");
-
-        let mut key_marker = String::new();
-        let mut version_id_marker = String::new();
-        let mut candidates = TransitionCandidateVersions::default();
-        loop {
-            let versions = self
-                .client
-                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
-                .await?;
-            candidates.extend(&remote_object, &versions);
-            if candidates.is_ambiguous() {
-                return Ok(TransitionCandidateProbe::Ambiguous);
+        match self
+            .get_with_headers(
+                object,
+                "",
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 1,
+                },
+            )
+            .await
+        {
+            Ok((headers, _)) => {
+                let version_id = match raw_version_header {
+                    Some(header_name) => match headers.get(header_name) {
+                        Some(value) => {
+                            let version_id = value.to_str().map_err(|_| {
+                                std::io::Error::new(
+                                    std::io::ErrorKind::InvalidData,
+                                    "remote object version id is not valid ASCII",
+                                )
+                            })?;
+                            validate_remote_version_id(version_id)?;
+                            Some(version_id)
+                        }
+                        None => None,
+                    },
+                    None => self.client.raw_version_id(&headers)?,
+                };
+                Ok(match version_id {
+                    Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()),
+                    None => TransitionCandidateProbe::UnversionedPresent,
+                })
             }
-            if !versions.is_truncated {
-                return classify_transition_candidates(candidates, bucket_versioning);
+            Err(err) => {
+                let response = to_error_response(&err);
+                if response.code == S3ErrorCode::NoSuchKey {
+                    Ok(TransitionCandidateProbe::Missing)
+                } else {
+                    Err(err)
+                }
             }
-
-            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
         }
     }
 
+    pub(crate) async fn probe_transition_candidate_with_raw_version_header(
+        &self,
+        object: &str,
+        raw_version_header: &'static str,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.probe_current_transition_candidate_with_header(object, Some(raw_version_header))
+            .await
+    }
+
     async fn probe_transition_candidate_identity(
         &self,
         object: &str,
@@ -343,6 +381,7 @@ struct TransitionCandidateVersions {
 }
 
 impl TransitionCandidateVersions {
+    #[cfg(test)]
     fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
         for version in versions.versions.iter().filter(|version| version.key == remote_object) {
             if self.version_id.is_some() {
@@ -353,10 +392,6 @@ impl TransitionCandidateVersions {
         }
     }
 
-    fn is_ambiguous(&self) -> bool {
-        self.ambiguous
-    }
-
     fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
         if self.ambiguous {
             return TransitionCandidateProbe::Ambiguous;
@@ -380,6 +415,8 @@ impl TransitionCandidateVersions {
 mod tests {
     use super::*;
     use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
     #[tokio::test]
     async fn new_rejects_loopback_endpoint_before_network_setup() {
@@ -398,6 +435,204 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn new_preserves_an_explicit_endpoint_port() {
+        let conf = TierS3 {
+            endpoint: "https://tier.example.com:9443".to_string(),
+            bucket: "tier-bucket".to_string(),
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            region: "us-east-1".to_string(),
+            ..Default::default()
+        };
+
+        let backend = WarmBackendS3::new(&conf, "tier")
+            .await
+            .expect("a well-formed S3 endpoint should initialize without network I/O");
+        assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com"));
+        assert_eq!(backend.client.endpoint_url.port(), Some(9443));
+    }
+
+    #[tokio::test]
+    async fn overflowing_get_range_is_rejected_before_network_io() {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "s3",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        let backend = WarmBackendS3 {
+            core: TransitionCore(Arc::clone(&client)),
+            client,
+            bucket: "bucket".to_string(),
+            prefix: String::new(),
+            storage_class: String::new(),
+        };
+
+        let err = backend
+            .get_with_headers(
+                "probe",
+                "",
+                WarmBackendGetOpts {
+                    start_offset: i64::MAX,
+                    length: 2,
+                },
+            )
+            .await
+            .expect_err("an overflowing range must fail before issuing a GET");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), listener.accept())
+                .await
+                .is_err()
+        );
+    }
+
+    async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let responses = [
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
+            ];
+            let mut requests = Vec::new();
+            for response in responses {
+                let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
+                let mut request = Vec::new();
+                let mut buffer = [0; 1024];
+                loop {
+                    let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                    assert_ne!(read, 0, "connection closed before request headers were received");
+                    request.extend_from_slice(&buffer[..read]);
+                    if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                        break;
+                    }
+                }
+                requests.push(String::from_utf8_lossy(&request).into_owned());
+                stream
+                    .write_all(response.as_bytes())
+                    .await
+                    .expect("fixture should write candidate response");
+            }
+            requests
+        });
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "s3",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        Some((
+            WarmBackendS3 {
+                core: TransitionCore(Arc::clone(&client)),
+                client,
+                bucket: "bucket".to_string(),
+                prefix: String::new(),
+                storage_class: String::new(),
+            },
+            fixture,
+        ))
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_uses_only_exact_bounded_get_permissions() {
+        let Some((backend, fixture)) = candidate_probe_fixture().await else {
+            return;
+        };
+
+        assert_eq!(
+            backend
+                .probe_transition_candidate("versioned-probe")
+                .await
+                .expect("versioned candidate should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("opaque-version".to_string())
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("unversioned-probe")
+                .await
+                .expect("unversioned candidate should be discovered"),
+            TransitionCandidateProbe::UnversionedPresent
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("missing-probe")
+                .await
+                .expect("a missing key should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("provider-missing-probe")
+                .await
+                .expect("a provider-specific missing code should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        let err = backend
+            .probe_transition_candidate("forbidden-probe")
+            .await
+            .expect_err("an authorization failure must not be mistaken for a missing key");
+        assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
+
+        let requests = fixture.await.expect("candidate fixture should join");
+        for request in requests {
+            let request = request.to_ascii_lowercase();
+            assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
+            assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
+            assert!(!request.contains("?versioning"));
+            assert!(!request.contains("?versions"));
+        }
+    }
+
     fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
         ListVersionsResult {
             versions: versions
@@ -631,8 +866,7 @@ impl WarmBackend for WarmBackendS3 {
     }
 
     async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
-        let bucket_versioning = self.remote_bucket_versioning().await?;
-        self.probe_transition_candidate_versions(object, bucket_versioning).await
+        self.probe_current_transition_candidate_with_header(object, None).await
     }
 
     async fn in_use(&self) -> Result<bool, std::io::Error> {

crates/ecstore/src/services/tier/warm_backend_tencent.rs

@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
     tier_config::TierTencent,
     warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
         new_s3_compatible_warm_backend, optimal_part_size,
     },
     warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendTencent {
         self.0.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> Result<bool, std::io::Error> {
         self.0.in_use().await
     }

crates/ecstore/src/services/tier/warm_backend_wasabi.rs

@@ -23,7 +23,7 @@ use uuid::Uuid;
 
 use crate::services::tier::{
     tier_config::{TierS3, TierWasabi},
-    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
     warm_backend_s3::WarmBackendS3,
 };
 use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
@@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi {
         self.s3.remove(object, rv).await
     }
 
+    async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
+        self.s3.probe_transition_candidate(object).await
+    }
+
     async fn in_use(&self) -> io::Result<bool> {
         self.check_remote_bucket_unversioned().await?;
         let in_use = self.s3.in_use().await?;

crates/ecstore/src/set_disk/ops/object.rs

@@ -7935,6 +7935,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     delete_marker_mtime: vr.mod_time.or(goi.mod_time),
                     object_name: vr.name.clone(),
                     replication_state: vr.replication_state_internal.clone(),
+                    source_missing,
                     ..Default::default()
                 }
             } else {
@@ -7946,6 +7947,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                         vr.version_id
                     },
                     replication_state: vr.replication_state_internal.clone(),
+                    source_missing,
                     ..Default::default()
                 };
                 accounting[i] = Some(DeleteAccounting {

crates/ecstore/src/storage_api_contracts/mod.rs

@@ -32,8 +32,9 @@ pub(crate) mod internode {
         PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
         PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
         SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
-        SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
-        WALK_DIR_STREAM_COMPLETION_V1,
+        SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
+        SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
+        WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
     };
 }
 

crates/ecstore/src/store/object.rs

@@ -18,7 +18,9 @@ use crate::bucket::lifecycle::{
     get_expiry_configs,
     tier_delete_journal::{
         ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
-        complete_tier_delete_dispatch, prepare_tier_delete_dispatch, record_tier_delete_journal_backend_identity,
+        TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch,
+        inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch,
+        record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit,
         tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope,
     },
     tier_sweeper::{
@@ -41,7 +43,10 @@ use crate::object_api::{
     NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
     TierFreeVersionReceiptSink,
 };
-use crate::services::notification_sys::acquire_tier_delete_journal_fleet_proof;
+use crate::services::notification_sys::{
+    TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches,
+    tier_delete_journal_topology_generation,
+};
 use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
 use crate::set_disk::{
     SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
@@ -52,6 +57,7 @@ use crate::storage_api_contracts::{
     namespace::NamespaceLocking as _,
     object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
 };
+use futures::StreamExt as _;
 use parking_lot::Mutex as ParkingMutex;
 use rustfs_filemeta::ObjectPartInfo;
 use rustfs_io_metrics::{
@@ -73,6 +79,7 @@ const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
 const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
 const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks";
 const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50);
+const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16;
 
 fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
     if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
@@ -147,14 +154,96 @@ async fn prepare_prefix_tier_delete_journal_entries(
     Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await
 }
 
+type TierDeleteLeaseReference = (String, Option<TierDestinationId>);
+
+fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool {
+    truncated && limit_cancellation && matches!(error, Error::OperationCanceled)
+}
+
+fn combine_tier_delete_walk_results(results: impl IntoIterator<Item = Result<()>>) -> Result<()> {
+    let mut cancelled = false;
+    for result in results {
+        match result {
+            Ok(()) => {}
+            Err(Error::OperationCanceled) => cancelled = true,
+            Err(err) => return Err(err),
+        }
+    }
+    if cancelled { Err(Error::OperationCanceled) } else { Ok(()) }
+}
+
+async fn acquire_prefix_tier_delete_reference_leases(
+    api: &Arc<ECStore>,
+    tier_references: &std::collections::HashSet<TierDeleteLeaseReference>,
+) -> Result<Vec<TierOperationLease>> {
+    let mut tier_references = tier_references.iter().cloned().collect::<Vec<_>>();
+    tier_references.sort_unstable();
+    let mut leases = Vec::with_capacity(tier_references.len());
+    for (tier_name, backend_identity) in tier_references {
+        let lease = match backend_identity {
+            Some(backend_identity) => {
+                TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
+                    .await
+            }
+            None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
+        }
+        .map_err(Error::other)?;
+        leases.push(lease);
+    }
+    Ok(leases)
+}
+
+async fn acquire_prefix_tier_delete_leases(api: &Arc<ECStore>, entries: &[Jentry]) -> Result<Vec<TierOperationLease>> {
+    let tier_references = entries
+        .iter()
+        .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
+        .collect::<std::collections::HashSet<_>>();
+    acquire_prefix_tier_delete_reference_leases(api, &tier_references).await
+}
+
 async fn prepare_prefix_tier_delete_journal_entries_inner(
     api: &Arc<ECStore>,
     bucket: &str,
     prefix: &str,
     opts: &ObjectOptions,
 ) -> Result<PreparedPrefixTierDelete> {
+    let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) {
+        (false, false, None)
+    } else {
+        let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
+        let bucket_fence = opts
+            .bucket_lifecycle_lock_fence
+            .as_ref()
+            .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
+        match Box::pin(inspect_tier_delete_chunk_parent(
+            Arc::clone(api),
+            bucket,
+            bucket_incarnation,
+            prefix,
+            bucket_fence,
+        ))
+        .await?
+        {
+            TierDeleteChunkParentInspection::NoParent => (false, false, None),
+            TierDeleteChunkParentInspection::LegacyManifest => (false, true, None),
+            TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)),
+            TierDeleteChunkParentInspection::Resume(dispatch) => {
+                let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?;
+                return Ok(PreparedPrefixTierDelete {
+                    dispatch: Some(*dispatch),
+                    chunk_parent_active: true,
+                    chunk_parent_fleet_proof: None,
+                    _leases: leases,
+                });
+            }
+            TierDeleteChunkParentInspection::RetryRequired => {
+                return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch"));
+            }
+        }
+    };
     let mut tier_references = std::collections::HashSet::<(String, Option<TierDestinationId>)>::new();
     let mut entries_by_name = std::collections::BTreeMap::new();
+    let batch_limit = tier_delete_dispatch_batch_limit();
     let logical_prefix = decode_dir_object(prefix);
     let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone());
     let physical_sets = api
@@ -164,12 +253,11 @@ async fn prepare_prefix_tier_delete_journal_entries_inner(
         .collect::<Vec<_>>();
     let (tx, mut rx) = tokio::sync::mpsc::channel::<ObjectInfoOrErr>(100);
     let cancellation = tokio_util::sync::CancellationToken::new();
+    let limit_cancellation = Arc::new(AtomicBool::new(false));
     let walk_cancel = cancellation.clone();
     let bucket_owned = bucket.to_string();
     let prefix_owned = prefix.to_string();
     let walk = async move {
-        use futures::StreamExt as _;
-
         let results = futures::stream::iter(physical_sets.into_iter().map(|set| {
             let tx = tx.clone();
             let cancellation = walk_cancel.clone();
@@ -199,14 +287,28 @@ async fn prepare_prefix_tier_delete_journal_entries_inner(
         .collect::<Vec<_>>()
         .await;
         drop(tx);
-        results.into_iter().collect::<Result<Vec<_>>>().map(|_| ())
+        combine_tier_delete_walk_results(results)
     };
+    let collect_limit_cancellation = limit_cancellation.clone();
     let collect = async {
+        let mut truncated = false;
         while let Some(result) = rx.recv().await {
             if let Some(err) = result.err {
+                // Once limit + 1 has been observed this request can authorize
+                // only the exact retained batch; it cannot infer prefix
+                // absence or run the raw delete. Drain only the explicit
+                // cancellation fallout; a real walker error must still fail
+                // the request even when another set reached the limit first.
+                if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err)
+                {
+                    continue;
+                }
                 cancellation.cancel();
                 return Err(err);
             }
+            if truncated {
+                continue;
+            }
             let Some(source) = result.item else {
                 continue;
             };
@@ -223,62 +325,125 @@ async fn prepare_prefix_tier_delete_journal_entries_inner(
                     "recursive prefix delete cannot discard an existing tier free-version cleanup obligation",
                 ));
             }
-            if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
+            let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
                 let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?;
-                tier_references.insert((source.transitioned_object.tier.clone(), backend_identity));
-            }
+                Some((source.transitioned_object.tier.clone(), backend_identity))
+            } else {
+                None
+            };
             if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? {
-                entries_by_name
-                    .entry(tier_delete_journal_object_name(&entry))
-                    .or_insert(entry);
+                let name = tier_delete_journal_object_name(&entry);
+                let at_limit = entries_by_name.len() == batch_limit;
+                match entries_by_name.entry(name) {
+                    std::collections::btree_map::Entry::Occupied(_) => {}
+                    std::collections::btree_map::Entry::Vacant(_) if at_limit => {
+                        truncated = true;
+                        collect_limit_cancellation.store(true, Ordering::Release);
+                        cancellation.cancel();
+                    }
+                    std::collections::btree_map::Entry::Vacant(slot) => {
+                        if let Some(tier_reference) = tier_reference {
+                            tier_references.insert(tier_reference);
+                        }
+                        slot.insert(entry);
+                    }
+                }
+            } else if let Some(tier_reference) = tier_reference {
+                tier_references.insert(tier_reference);
             }
         }
-        Ok(())
+        Ok(truncated)
     };
     let (walk_result, collect_result) = tokio::join!(walk, collect);
-    collect_result?;
-    walk_result?;
-    let entries = entries_by_name.into_values().collect::<Vec<_>>();
-
-    let mut tier_references = tier_references.into_iter().collect::<Vec<_>>();
-    tier_references.sort_unstable();
-    let mut leases = Vec::with_capacity(tier_references.len());
-    for (tier_name, backend_identity) in tier_references {
-        let lease = match backend_identity {
-            Some(backend_identity) => {
-                TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
-                    .await
-            }
-            None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
-        }
-        .map_err(Error::other)?;
-        leases.push(lease);
+    let truncated = collect_result?;
+    // A truncated walk normally reports OperationCanceled from the physical
+    // walkers. That cancellation is expected; any other result is a genuine
+    // scan failure and cannot be hidden by the bounded batch.
+    if let Err(err) = walk_result
+        && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err)
+    {
+        return Err(err);
     }
+    let entries = entries_by_name.into_values().collect::<Vec<_>>();
+    let mut leased_tier_references = tier_references;
+    let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?;
     if entries.is_empty() {
+        let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() {
+            let fleet_proof = acquire_tier_delete_journal_fleet_proof()
+                .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?;
+            if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology {
+                return Err(Error::other("tier delete chunk parent topology changed during final source scan"));
+            }
+            Some(fleet_proof)
+        } else {
+            None
+        };
         return Ok(PreparedPrefixTierDelete {
             dispatch: None,
+            chunk_parent_active,
+            chunk_parent_fleet_proof,
             _leases: leases,
         });
     }
 
     let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
-    let fleet_proof = acquire_tier_delete_journal_fleet_proof()
-        .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
     let bucket_fence = opts
         .bucket_lifecycle_lock_fence
         .as_ref()
         .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
-    let dispatch =
+    let fleet_proof = acquire_tier_delete_journal_fleet_proof()
+        .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
+    if chunk_parent_topology_generation
+        .as_deref()
+        .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected)
+    {
+        return Err(Error::other("tier delete chunk parent topology changed during source scan"));
+    }
+    let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) {
+        Box::pin(prepare_tier_delete_chunk_dispatch(
+            Arc::clone(api),
+            bucket,
+            bucket_incarnation,
+            prefix,
+            entries,
+            truncated && !chunk_parent_active,
+            fleet_proof,
+            bucket_fence,
+        ))
+        .await?
+    } else if legacy_manifest_active && truncated {
+        resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
+            .await?
+    } else {
         prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
-            .await?;
+            .await?
+    };
+    // A resumed legacy authorization may own predecessors that are absent
+    // from this bounded scan. Pin every backend generation in the actual
+    // permit before any local mutation, while avoiding duplicate leases for
+    // entries already covered by the scan.
+    let additional_tier_references = dispatch
+        .entries()?
+        .iter()
+        .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
+        .filter(|reference| leased_tier_references.insert(reference.clone()))
+        .collect::<std::collections::HashSet<_>>();
+    leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?);
+    if legacy_manifest_active && truncated {
+        dispatch.require_exact_predecessor_replay();
+    }
     Ok(PreparedPrefixTierDelete {
         dispatch: Some(dispatch),
+        chunk_parent_active: chunk_parent_active || truncated,
+        chunk_parent_fleet_proof: None,
         _leases: leases,
     })
 }
 
 struct PreparedPrefixTierDelete {
     dispatch: Option<crate::bucket::lifecycle::tier_delete_journal::PreparedTierDeleteDispatch>,
+    chunk_parent_active: bool,
+    chunk_parent_fleet_proof: Option<TierDeleteJournalFleetProofToken>,
     _leases: Vec<TierOperationLease>,
 }
 
@@ -365,12 +530,16 @@ async fn delete_prefix_with_tier_delete_journal(
     let Some(api) = tier_journal_api else {
         return store.delete_prefix(bucket, object, opts).await;
     };
-    let PreparedPrefixTierDelete { dispatch, _leases } =
-        prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
+    let PreparedPrefixTierDelete {
+        dispatch,
+        chunk_parent_active,
+        chunk_parent_fleet_proof,
+        _leases,
+    } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
     let Some(dispatch) = dispatch else {
-        // There is no remote-cleanup candidate, so no v6 manifest or fleet
-        // proof is required.  Keep any compatibility-path tier leases alive
-        // until the local delete has committed.
+        // There is no new remote-cleanup candidate. Keep compatibility-path
+        // tier leases and, for a chunked final pass, the matching parent fleet
+        // proof alive until local deletion and parent completion both commit.
         let _tier_leases = _leases;
         let mut operation_opts = opts.clone();
         // `tier_delete_journal_api` means a v6 dispatch authorization must be
@@ -379,7 +548,42 @@ async fn delete_prefix_with_tier_delete_journal(
         // transitioned metadata retains its FreeVersion fallback.
         operation_opts.tier_delete_journal_api = None;
         operation_opts.tier_delete_dispatch_authorization = None;
-        return store.delete_prefix(bucket, object, &operation_opts).await;
+        let parent_fleet_proof = if chunk_parent_active {
+            Some(
+                chunk_parent_fleet_proof
+                    .as_ref()
+                    .filter(|proof| tier_delete_journal_fleet_proof_matches(proof))
+                    .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?,
+            )
+        } else {
+            None
+        };
+        store.delete_prefix(bucket, object, &operation_opts).await?;
+        if let Some(parent_fleet_proof) = parent_fleet_proof {
+            #[cfg(all(test, feature = "test-util"))]
+            crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
+                crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
+            )
+            .await;
+            let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
+            let bucket_fence = opts
+                .bucket_lifecycle_lock_fence
+                .as_ref()
+                .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
+            if !Box::pin(complete_tier_delete_chunk_parent(
+                Arc::clone(api),
+                bucket,
+                bucket_incarnation,
+                object,
+                bucket_fence,
+                parent_fleet_proof,
+            ))
+            .await?
+            {
+                return Err(Error::other("tier delete chunk parent disappeared after final local deletion"));
+            }
+        }
+        return Ok(());
     };
     let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
     let bucket_fence = opts
@@ -396,9 +600,19 @@ async fn delete_prefix_with_tier_delete_journal(
     // Keep every backend generation lease until the whole local operation has
     // either committed its journal set or returned an ambiguous mutation.
     let _tier_leases = _leases;
-    if active.predecessor_replay_required() {
+    if active.predecessor_replay_required() || active.is_chunked() {
         replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?;
+        #[cfg(all(test, feature = "test-util"))]
+        if active.is_chunked() {
+            crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
+                crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted,
+            )
+            .await;
+        }
         complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?;
+        if active.is_chunked() {
+            return Err(Error::other("tier delete chunk completed; retry the next durable batch"));
+        }
         return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch"));
     }
     let result = store.delete_prefix(bucket, object, &operation_opts).await;
@@ -443,7 +657,7 @@ async fn replay_authorized_tier_delete_sources(
     let authorization = active.authorization();
     authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?;
 
-    let mut source_objects = std::collections::BTreeSet::new();
+    let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len());
     for entry in active.entries() {
         let source = entry
             .source
@@ -453,61 +667,130 @@ async fn replay_authorized_tier_delete_sources(
         if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) {
             return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope"));
         }
-        source_objects.insert(source.object.clone());
+        source_objects.insert(source.object.as_str());
     }
 
-    let mut deleted = 0;
-    for object in source_objects {
-        if bucket_fence.is_lock_lost() {
-            return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
+    if let Some(scope) = publication_scope {
+        if scope.state() == ScannerPublicationCommitState::Admitted {
+            scope
+                .try_begin()
+                .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
         }
-        let encoded_object = encode_dir_object(&object);
-        let guards = if opts.delete_prefix_object {
-            store
-                .acquire_remaining_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object)
-                .await?
-        } else {
-            store
-                .acquire_all_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object)
-                .await?
-        };
-        authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
-        if let Some(scope) = publication_scope {
-            if scope.state() == ScannerPublicationCommitState::Admitted {
-                scope
-                    .try_begin()
-                    .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
-            }
-            if !scope.can_commit() {
-                let _ = scope.mark_indeterminate();
-                return Err(StorageError::OperationCanceled);
-            }
-        }
-        let mut replay_opts = opts.clone();
-        replay_opts.no_lock = true;
-        replay_opts.delete_prefix = false;
-        replay_opts.delete_prefix_object = false;
-        for guard in &guards {
-            guard.add_namespace_lock_fence(&mut replay_opts);
-        }
-        for pool in &store.pools {
-            for set in &pool.disk_set {
-                authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
-                deleted += set
-                    .replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
-                    .await?;
-            }
-        }
-        if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
-            return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
+        if !scope.can_commit() {
+            let _ = scope.mark_indeterminate();
+            return Err(StorageError::OperationCanceled);
         }
     }
-    if let Some(scope) = publication_scope {
-        let _ = scope.mark_committed();
+
+    let stopped = Arc::new(AtomicBool::new(false));
+    // The caller holds bucket lifecycle WRITE. Each bounded future acquires
+    // only one logical object's physical lock set and releases it before
+    // completion; no future nests locks for two object keys.
+    let make_replay = |object: String| {
+        let stopped = stopped.clone();
+        let authorization = authorization.clone();
+        async move {
+            if stopped.load(Ordering::Acquire) {
+                return Ok::<_, Error>(0usize);
+            }
+            let result = async {
+                if bucket_fence.is_lock_lost() {
+                    return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
+                }
+                let encoded_object = encode_dir_object(&object);
+                let guards = if opts.delete_prefix_object {
+                    store
+                        .acquire_remaining_physical_object_write_locks(
+                            "tier_delete_dispatch_predecessor_replay",
+                            bucket,
+                            &encoded_object,
+                        )
+                        .await?
+                } else {
+                    store
+                        .acquire_all_physical_object_write_locks(
+                            "tier_delete_dispatch_predecessor_replay",
+                            bucket,
+                            &encoded_object,
+                        )
+                        .await?
+                };
+                authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
+                if publication_scope.is_some_and(|scope| !scope.can_commit()) {
+                    return Err(StorageError::OperationCanceled);
+                }
+                let mut replay_opts = opts.clone();
+                replay_opts.no_lock = true;
+                replay_opts.delete_prefix = false;
+                replay_opts.delete_prefix_object = false;
+                for guard in &guards {
+                    guard.add_namespace_lock_fence(&mut replay_opts);
+                }
+                let mut deleted = 0usize;
+                for pool in &store.pools {
+                    for set in &pool.disk_set {
+                        authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
+                        deleted = deleted
+                            .checked_add(
+                                set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
+                                    .await?,
+                            )
+                            .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
+                    }
+                }
+                if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
+                    return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
+                }
+                Ok(deleted)
+            }
+            .await;
+            if result.is_err() {
+                stopped.store(true, Ordering::Release);
+            }
+            result
+        }
+    };
+    let mut objects = source_objects.into_iter();
+    let mut replays = futures::stream::FuturesUnordered::new();
+    for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY {
+        let Some(object) = objects.next().map(ToOwned::to_owned) else {
+            break;
+        };
+        replays.push(make_replay(object));
+    }
+    let mut deleted = 0usize;
+    let mut first_error = None;
+    while let Some(result) = replays.next().await {
+        match result {
+            Ok(count) => {
+                deleted = deleted
+                    .checked_add(count)
+                    .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
+            }
+            Err(err) if first_error.is_none() => first_error = Some(err),
+            Err(_) => {}
+        }
+        if first_error.is_none()
+            && !stopped.load(Ordering::Acquire)
+            && let Some(object) = objects.next().map(ToOwned::to_owned)
+        {
+            replays.push(make_replay(object));
+        }
     }
     if deleted > 0 {
         super::list_objects::observe_list_objects_mutation(store, bucket).await;
     }
+    if let Some(err) = first_error {
+        if publication_scope.is_some_and(|scope| !scope.can_commit())
+            && let Some(scope) = publication_scope
+        {
+            let _ = scope.mark_indeterminate();
+        }
+        return Err(err);
+    }
+    if let Some(scope) = publication_scope {
+        let _ = scope.mark_committed();
+    }
     Ok(())
 }
 
@@ -6518,6 +6801,28 @@ mod tests {
         );
     }
 
+    #[test]
+    fn tier_delete_walk_only_accepts_explicit_limit_cancellation() {
+        let cancelled = Error::OperationCanceled;
+        assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled));
+        assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled));
+        assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled));
+        assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed")));
+    }
+
+    #[test]
+    fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() {
+        let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)])
+            .expect_err("a real walk error must not be hidden by earlier cancellation");
+        assert_eq!(err, StorageError::FileAccessDenied);
+
+        assert_eq!(
+            combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)])
+                .expect_err("cancellation must remain visible when there is no real error"),
+            Error::OperationCanceled
+        );
+    }
+
     impl Drop for BodyCacheHookGuard {
         fn drop(&mut self) {
             clear_get_object_body_cache_hook();

crates/filemeta/src/fileinfo.rs

@@ -28,7 +28,7 @@ use serde::de::{self, MapAccess, SeqAccess, Visitor, value::MapAccessDeserialize
 use serde::ser::SerializeMap;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use time::{OffsetDateTime, PrimitiveDateTime, format_description::well_known::Rfc3339};
 use time::{format_description::FormatItem, macros::format_description};
 use uuid::Uuid;
 
@@ -1417,6 +1417,31 @@ impl RestoreStatusOps for RestoreStatus {
     }
 }
 
+/// Parse the `expiry-date` value carried by a persisted `x-amz-restore` header.
+///
+/// RustFS has always serialized this field as RFC3339, while MinIO (and any
+/// object migrated in place from a MinIO deployment) writes Go's
+/// `http.TimeFormat` — RFC1123 with a literal `GMT` zone. Accept both shapes
+/// so a migrated "restored" object stays readable.
+///
+/// This is not only a HEAD 500: `is_restored_object_on_disk` fails open when
+/// the header cannot be parsed, and `MetaObject::uses_data_dir` falls back to
+/// it, so an unparseable restore header makes a live restored data dir look
+/// unused (backlog#1342).
+///
+/// The write side deliberately stays on RFC3339 for now: an older node only
+/// parses RFC3339, so switching the persisted format before this parser is
+/// deployed everywhere would break rolling upgrades and rollbacks in the
+/// opposite direction.
+fn parse_restore_expiry_date(value: &str) -> Result<OffsetDateTime> {
+    if let Ok(expiry) = OffsetDateTime::parse(value, &Rfc3339) {
+        return Ok(expiry);
+    }
+    PrimitiveDateTime::parse(value, &RFC1123)
+        .map(PrimitiveDateTime::assume_utc)
+        .map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))
+}
+
 pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
     let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect();
     let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect();
@@ -1445,8 +1470,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
             if expiry_tokens[0].trim() != "expiry-date" {
                 return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
             }
-            let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339)
-                .map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))?;
+            let expiry = parse_restore_expiry_date(expiry_tokens[1].trim_matches('"'))?;
             return Ok(RestoreStatus {
                 is_restore_in_progress: Some(false),
                 restore_expiry_date: Some(Timestamp::from(expiry)),
@@ -1471,6 +1495,7 @@ mod tests {
     use super::*;
     use proptest::collection::{hash_map, vec};
     use proptest::prelude::*;
+    use time::macros::datetime;
 
     // backlog#959 / ECA-18: the interleaved per-block bitrot subsystem in
     // rustfs-ecstore (BitrotWriter / bitrot_verify / bitrot_shard_file_size) is
@@ -2685,4 +2710,70 @@ mod tests {
         let empty = FileInfo::default();
         assert!(format!("{empty:?}").contains("data: None"));
     }
+
+    /// backlog#1342: MinIO writes the persisted `x-amz-restore` completion
+    /// header with Go's `http.TimeFormat` (RFC1123/GMT). Before this parser
+    /// accepted that shape, every object migrated in place from MinIO failed
+    /// `parse_restore_obj_status`.
+    #[test]
+    fn parses_minio_rfc1123_restore_expiry() {
+        let status = parse_restore_obj_status("ongoing-request=\"false\", expiry-date=\"Wed, 01 Jan 2025 10:20:30 GMT\"")
+            .expect("RFC1123 expiry-date must parse");
+        assert_eq!(status.is_restore_in_progress, Some(false));
+        let expiry = status.expiry().expect("completed restore carries an expiry");
+        assert_eq!(expiry, datetime!(2025-01-01 10:20:30 UTC));
+    }
+
+    /// The RustFS-written shape must keep parsing unchanged: the persisted
+    /// format stays RFC3339 until every node can read both.
+    #[test]
+    fn parses_rustfs_rfc3339_restore_expiry() {
+        let status = parse_restore_obj_status("ongoing-request=\"false\", expiry-date=\"2025-01-01T10:20:30Z\"")
+            .expect("RFC3339 expiry-date must parse");
+        assert_eq!(status.expiry(), Some(datetime!(2025-01-01 10:20:30 UTC)));
+    }
+
+    /// Round-trip both directions of the migration: what RustFS emits on the
+    /// wire (`to_string2`, RFC1123) and what it persists (`to_string`,
+    /// RFC3339) must both be readable by this parser.
+    #[test]
+    fn restore_status_round_trips_through_both_formats() {
+        let status = RestoreStatus {
+            is_restore_in_progress: Some(false),
+            restore_expiry_date: Some(Timestamp::from(datetime!(2030-06-15 07:08:09 UTC))),
+        };
+        for rendered in [RestoreStatusOps::to_string(&status), status.to_string2()] {
+            let parsed = parse_restore_obj_status(&rendered).unwrap_or_else(|e| panic!("{rendered} must parse: {e}"));
+            assert_eq!(parsed.expiry(), Some(datetime!(2030-06-15 07:08:09 UTC)), "{rendered}");
+        }
+    }
+
+    /// A restored object migrated from MinIO must still be recognised as
+    /// on-disk: `is_restored_object_on_disk` fails open, and
+    /// `MetaObject::uses_data_dir` uses it to decide whether a data dir is
+    /// live, so a parse failure here can make a live data dir look reclaimable.
+    #[test]
+    fn minio_restored_object_is_recognised_as_on_disk() {
+        let mut meta = HashMap::new();
+        meta.insert(
+            X_AMZ_RESTORE.as_str().to_string(),
+            "ongoing-request=\"false\", expiry-date=\"Fri, 01 Jan 9999 00:00:00 GMT\"".to_string(),
+        );
+        assert!(is_restored_object_on_disk(&meta));
+    }
+
+    /// Widening the accepted formats must not weaken malformed-input
+    /// rejection: an unparseable or non-GMT-shaped expiry still fails closed.
+    #[test]
+    fn rejects_malformed_restore_expiry() {
+        for header in [
+            "ongoing-request=\"false\", expiry-date=\"not-a-date\"",
+            "ongoing-request=\"false\", expiry-date=\"Wed, 01 Jan 2025 10:20:30\"",
+            "ongoing-request=\"false\", expiry-date=\"01 Jan 2025 10:20:30 GMT\"",
+            "ongoing-request=\"false\"",
+            "ongoing-request=\"false\", expires=\"2025-01-01T10:20:30Z\"",
+        ] {
+            assert!(parse_restore_obj_status(header).is_err(), "{header} must be rejected");
+        }
+    }
 }

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1251,6 +1251,38 @@ pub struct ScannerActivityResponse {
     #[prost(bool, optional, tag = "11")]
     pub publication_blocked: ::core::option::Option<bool>,
 }
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct ScannerDirtyUsageBucket {
+    #[prost(string, tag = "1")]
+    pub bucket: ::prost::alloc::string::String,
+    #[prost(uint64, tag = "2")]
+    pub generation: u64,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct ScannerDirtyUsageSnapshotRequest {
+    #[prost(bytes = "bytes", tag = "1")]
+    pub challenge: ::prost::bytes::Bytes,
+    #[prost(uint32, tag = "2")]
+    pub protocol_version: u32,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ScannerDirtyUsageSnapshotResponse {
+    #[prost(string, tag = "1")]
+    pub instance_id: ::prost::alloc::string::String,
+    #[prost(uint64, tag = "2")]
+    pub generation: u64,
+    #[prost(uint64, tag = "3")]
+    pub pending_bucket_count: u64,
+    #[prost(uint32, tag = "4")]
+    pub protocol_version: u32,
+    /// Incomplete snapshots are all-or-nothing and carry no bucket entries.
+    #[prost(bool, tag = "5")]
+    pub complete: bool,
+    #[prost(message, repeated, tag = "6")]
+    pub buckets: ::prost::alloc::vec::Vec<ScannerDirtyUsageBucket>,
+    #[prost(bytes = "bytes", tag = "7")]
+    pub response_proof: ::prost::bytes::Bytes,
+}
 /// A short-lived storage-owned read admission used only around a final
 /// scanner metadata publication.  It is intentionally separate from the
 /// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -2899,6 +2931,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn scanner_dirty_usage_snapshot(
+            &mut self,
+            request: impl tonic::IntoRequest<super::ScannerDirtyUsageSnapshotRequest>,
+        ) -> std::result::Result<tonic::Response<super::ScannerDirtyUsageSnapshotResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ScannerDirtyUsageSnapshot");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "ScannerDirtyUsageSnapshot"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn acquire_scanner_publication_lease(
             &mut self,
             request: impl tonic::IntoRequest<super::ScannerPublicationLeaseRequest>,
@@ -3457,6 +3504,10 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::ScannerActivityRequest>,
         ) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
+        async fn scanner_dirty_usage_snapshot(
+            &self,
+            request: tonic::Request<super::ScannerDirtyUsageSnapshotRequest>,
+        ) -> std::result::Result<tonic::Response<super::ScannerDirtyUsageSnapshotResponse>, tonic::Status>;
         async fn acquire_scanner_publication_lease(
             &self,
             request: tonic::Request<super::ScannerPublicationLeaseRequest>,
@@ -5749,6 +5800,34 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/ScannerDirtyUsageSnapshot" => {
+                    #[allow(non_camel_case_types)]
+                    struct ScannerDirtyUsageSnapshotSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::ScannerDirtyUsageSnapshotRequest> for ScannerDirtyUsageSnapshotSvc<T> {
+                        type Response = super::ScannerDirtyUsageSnapshotResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::ScannerDirtyUsageSnapshotRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::scanner_dirty_usage_snapshot(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = ScannerDirtyUsageSnapshotSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/AcquireScannerPublicationLease" => {
                     #[allow(non_camel_case_types)]
                     struct AcquireScannerPublicationLeaseSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/lib.rs

@@ -541,6 +541,34 @@ pub fn canonical_scanner_activity_v7_response_body(
     Ok(body)
 }
 
+pub fn canonical_scanner_dirty_usage_snapshot_request_body(
+    request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-scanner-dirty-usage-snapshot-request-v1\0");
+    body.push_u32(request.protocol_version);
+    body.push_bytes(request.challenge.as_ref())?;
+    Ok(body.finish())
+}
+
+pub fn canonical_scanner_dirty_usage_snapshot_response_body(
+    challenge: &[u8],
+    response: &proto_gen::node_service::ScannerDirtyUsageSnapshotResponse,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-scanner-dirty-usage-snapshot-response-v1\0");
+    body.push_bytes(challenge)?;
+    body.push_str(&response.instance_id)?;
+    body.push_u64(response.generation);
+    body.push_u64(response.pending_bucket_count);
+    body.push_u32(response.protocol_version);
+    body.push_bool(response.complete);
+    body.push_count(response.buckets.len())?;
+    for bucket in &response.buckets {
+        body.push_str(&bucket.bucket)?;
+        body.push_u64(bucket.generation);
+    }
+    Ok(body.finish())
+}
+
 /// Builds the body authenticated by the short-lived remote scanner publication
 /// lease request. This is a separate domain from ScannerActivity so v6/v7
 /// observation proofs remain byte-for-byte compatible.
@@ -1751,13 +1779,112 @@ mod scanner_activity_tests {
     use super::{
         canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
         canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body,
+        canonical_scanner_dirty_usage_snapshot_request_body, canonical_scanner_dirty_usage_snapshot_response_body,
         canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body,
         canonical_scanner_publication_lease_response_body,
         proto_gen::node_service::{
-            ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
+            ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshotRequest,
+            ScannerDirtyUsageSnapshotResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
         },
     };
 
+    #[test]
+    fn canonical_scanner_dirty_usage_snapshot_request_binds_every_field() {
+        let request = ScannerDirtyUsageSnapshotRequest {
+            challenge: vec![1; 16].into(),
+            protocol_version: 1,
+        };
+        let baseline = canonical_scanner_dirty_usage_snapshot_request_body(&request)
+            .expect("scanner dirty usage snapshot request should encode");
+
+        for variant in [
+            ScannerDirtyUsageSnapshotRequest {
+                challenge: vec![2; 16].into(),
+                ..request.clone()
+            },
+            ScannerDirtyUsageSnapshotRequest {
+                protocol_version: 2,
+                ..request
+            },
+        ] {
+            assert_ne!(
+                baseline,
+                canonical_scanner_dirty_usage_snapshot_request_body(&variant)
+                    .expect("scanner dirty usage snapshot request variant should encode")
+            );
+        }
+    }
+
+    #[test]
+    fn canonical_scanner_dirty_usage_snapshot_response_binds_every_field() {
+        let response = ScannerDirtyUsageSnapshotResponse {
+            instance_id: "0123456789abcdef0123456789abcdef".to_string(),
+            generation: 7,
+            pending_bucket_count: 2,
+            protocol_version: 1,
+            complete: true,
+            buckets: vec![
+                ScannerDirtyUsageBucket {
+                    bucket: "archive".to_string(),
+                    generation: 3,
+                },
+                ScannerDirtyUsageBucket {
+                    bucket: "photos".to_string(),
+                    generation: 7,
+                },
+            ],
+            response_proof: vec![9; 32].into(),
+        };
+        let baseline = canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &response)
+            .expect("scanner dirty usage snapshot response should encode");
+        let mut variants = Vec::new();
+        let mut instance = response.clone();
+        instance.instance_id = "fedcba9876543210fedcba9876543210".to_string();
+        variants.push(instance);
+        let mut generation = response.clone();
+        generation.generation = 8;
+        variants.push(generation);
+        let mut count = response.clone();
+        count.pending_bucket_count = 3;
+        variants.push(count);
+        let mut protocol = response.clone();
+        protocol.protocol_version = 2;
+        variants.push(protocol);
+        let mut complete = response.clone();
+        complete.complete = false;
+        variants.push(complete);
+        let mut bucket_name = response.clone();
+        bucket_name.buckets[0].bucket = "backups".to_string();
+        variants.push(bucket_name);
+        let mut bucket_generation = response.clone();
+        bucket_generation.buckets[0].generation = 4;
+        variants.push(bucket_generation);
+        let mut bucket_order = response.clone();
+        bucket_order.buckets.reverse();
+        variants.push(bucket_order);
+
+        for variant in variants {
+            assert_ne!(
+                baseline,
+                canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &variant)
+                    .expect("scanner dirty usage snapshot response variant should encode")
+            );
+        }
+        assert_ne!(
+            baseline,
+            canonical_scanner_dirty_usage_snapshot_response_body(&[2; 16], &response)
+                .expect("scanner dirty usage snapshot response challenge variant should encode")
+        );
+
+        let mut proof_only = response;
+        proof_only.response_proof = vec![8; 32].into();
+        assert_eq!(
+            baseline,
+            canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &proof_only)
+                .expect("response proof must not authenticate itself")
+        );
+    }
+
     #[test]
     fn canonical_scanner_activity_request_binds_every_field() {
         let request = ScannerActivityRequest {

crates/protos/src/node.proto

@@ -882,6 +882,27 @@ message ScannerActivityResponse {
   optional bool publication_blocked = 11;
 }
 
+message ScannerDirtyUsageBucket {
+  string bucket = 1;
+  uint64 generation = 2;
+}
+
+message ScannerDirtyUsageSnapshotRequest {
+  bytes challenge = 1;
+  uint32 protocol_version = 2;
+}
+
+message ScannerDirtyUsageSnapshotResponse {
+  string instance_id = 1;
+  uint64 generation = 2;
+  uint64 pending_bucket_count = 3;
+  uint32 protocol_version = 4;
+  // Incomplete snapshots are all-or-nothing and carry no bucket entries.
+  bool complete = 5;
+  repeated ScannerDirtyUsageBucket buckets = 6;
+  bytes response_proof = 7;
+}
+
 // A short-lived storage-owned read admission used only around a final
 // scanner metadata publication.  It is intentionally separate from the
 // ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -1206,6 +1227,7 @@ service NodeService {
   // rpc CommitBinary() returns () {};
   rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound
   rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound
+  rpc ScannerDirtyUsageSnapshot(ScannerDirtyUsageSnapshotRequest) returns (ScannerDirtyUsageSnapshotResponse) {}; // auth-policy: body-bound
   rpc AcquireScannerPublicationLease(ScannerPublicationLeaseRequest) returns (ScannerPublicationLeaseResponse) {}; // auth-policy: body-bound
   rpc ReleaseScannerPublicationLease(ScannerPublicationLeaseReleaseRequest) returns (ScannerPublicationLeaseReleaseResponse) {}; // auth-policy: body-bound
   rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only

crates/s3-client/src/api_error_response.rs

@@ -57,7 +57,11 @@ fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
 where
     D: Deserializer<'de>,
 {
-    Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
+    let code = String::deserialize(d)?;
+    if code == "NoSuchObject" {
+        return Ok(S3ErrorCode::NoSuchKey);
+    }
+    Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
 }
 
 impl Default for ErrorResponse {
@@ -325,4 +329,21 @@ mod tests {
         assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
         assert_eq!(response.status_code, StatusCode::NOT_FOUND);
     }
+
+    #[test]
+    fn normalizes_provider_specific_missing_object_code() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
+
+        let response = http_resp_to_error_response(
+            StatusCode::NOT_FOUND,
+            &headers,
+            b"<Error><Code>NoSuchObject</Code><Message>remote detail</Message></Error>".to_vec(),
+            "bucket",
+            "object",
+        );
+
+        assert_eq!(response.code, S3ErrorCode::NoSuchKey);
+        assert_eq!(response.status_code, StatusCode::NOT_FOUND);
+    }
 }

crates/s3-client/src/api_get_object.rs

@@ -30,7 +30,9 @@ use tokio_util::io::StreamReader;
 use crate::{
     api_error_response::err_invalid_argument,
     api_get_options::GetObjectOptions,
-    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
+    transition_api::{
+        ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
+    },
 };
 use futures_util::StreamExt;
 use http_body_util::BodyExt;
@@ -39,6 +41,42 @@ use hyper::body::Bytes;
 use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
 use tokio_util::io::ReaderStream;
 
+fn response_limit_from_range(opts: &GetObjectOptions) -> Result<Option<usize>, std::io::Error> {
+    let Some(range) = opts
+        .headers
+        .iter()
+        .find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str()))
+    else {
+        return Ok(None);
+    };
+    let Some((unit, bounds)) = range.split_once('=') else {
+        return Ok(None);
+    };
+    if !unit.eq_ignore_ascii_case("bytes") {
+        return Ok(None);
+    }
+    let Some((start, end)) = bounds.split_once('-') else {
+        return Ok(None);
+    };
+    if start.is_empty() || end.is_empty() || end.contains(',') {
+        return Ok(None);
+    }
+    let start = start
+        .parse::<u64>()
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?;
+    let end = end
+        .parse::<u64>()
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?;
+    let length = end
+        .checked_sub(start)
+        .and_then(|length| length.checked_add(1))
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?;
+    let limit = usize::try_from(length).map_err(|_| {
+        std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory")
+    })?;
+    Ok(Some(limit))
+}
+
 impl TransitionClient {
     pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
         let _ = opts;
@@ -54,6 +92,7 @@ impl TransitionClient {
         object_name: &str,
         opts: &GetObjectOptions,
     ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let max_response_bytes = response_limit_from_range(opts)?;
         let resp = self
             .execute_method(
                 http::Method::GET,
@@ -81,18 +120,214 @@ impl TransitionClient {
 
         let h = resp.headers().clone();
 
-        let mut body_vec = Vec::new();
         let mut body = resp.into_body();
-        while let Some(frame) = body.frame().await {
-            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-            if let Some(data) = frame.data_ref() {
-                body_vec.extend_from_slice(data);
+        let body_vec = if let Some(limit) = max_response_bytes {
+            collect_response_body(body, limit).await?
+        } else {
+            let mut body_vec = Vec::new();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
             }
-        }
+            body_vec
+        };
         Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
     }
 }
 
+#[cfg(test)]
+mod bounded_response_tests {
+    use super::response_limit_from_range;
+    use crate::{
+        api_get_options::GetObjectOptions,
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
+    };
+    use http_body_util::Full;
+    use hyper::body::Bytes;
+    use std::time::Duration;
+    use tokio::{
+        io::{AsyncReadExt, AsyncWriteExt},
+        net::TcpListener,
+    };
+
+    #[test]
+    fn closed_range_derives_a_collection_limit_without_new_public_options() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(5, 11).expect("the closed range should be valid");
+
+        assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7));
+    }
+
+    #[tokio::test]
+    async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+        let max_response_bytes = response_limit_from_range(&opts)
+            .expect("the range should parse")
+            .expect("the closed range should have a limit");
+        let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes)
+            .await
+            .expect_err("the collection layer must reject a response larger than its limit");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
+    async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let request = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let request = String::from_utf8_lossy(&request).into_owned();
+            let response = format!(
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+                body.len()
+            );
+            stream
+                .write_all(response.as_bytes())
+                .await
+                .expect("fixture should write response headers");
+            stream.write_all(body).await.expect("fixture should write response body");
+            request
+        });
+        let client = TransitionClient::new(
+            &endpoint,
+            Options {
+                creds: Credentials::new(Static(Value {
+                    access_key_id: "access-key".to_string(),
+                    secret_access_key: "secret-key".to_string(),
+                    signer_type: SignatureType::SignatureV4,
+                    ..Default::default()
+                })),
+                region: "us-east-1".to_string(),
+                bucket_lookup: BucketLookupType::BucketLookupPath,
+                max_retries: 1,
+                ..Default::default()
+            },
+            "",
+        )
+        .await
+        .expect("fixture client should build");
+        Some((client, request))
+    }
+
+    #[tokio::test]
+    async fn real_transport_accepts_the_exact_closed_range_length() {
+        let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
+            return;
+        };
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let (_, _, mut reader) = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect("a seven-byte response should fit the requested range");
+        let mut body = Vec::new();
+        reader
+            .read_to_end(&mut body)
+            .await
+            .expect("bounded response should be readable");
+
+        assert_eq!(body, b"RustFS!");
+        assert!(
+            request
+                .await
+                .expect("fixture should join")
+                .to_ascii_lowercase()
+                .contains("\r\nrange: bytes=0-6\r\n")
+        );
+    }
+
+    #[tokio::test]
+    async fn real_transport_rejects_a_body_larger_than_the_closed_range() {
+        let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else {
+            return;
+        };
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let err = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect_err("an eight-byte response must exceed the seven-byte range limit");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert!(
+            request
+                .await
+                .expect("fixture should join")
+                .to_ascii_lowercase()
+                .contains("\r\nrange: bytes=0-6\r\n")
+        );
+    }
+
+    #[tokio::test]
+    async fn overflowing_closed_range_is_rejected_before_network_io() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let client = TransitionClient::new(
+            &endpoint,
+            Options {
+                creds: Credentials::new(Static(Value {
+                    access_key_id: "access-key".to_string(),
+                    secret_access_key: "secret-key".to_string(),
+                    signer_type: SignatureType::SignatureV4,
+                    ..Default::default()
+                })),
+                region: "us-east-1".to_string(),
+                bucket_lookup: BucketLookupType::BucketLookupPath,
+                max_retries: 1,
+                ..Default::default()
+            },
+            "",
+        )
+        .await
+        .expect("fixture client should build");
+        let mut opts = GetObjectOptions::default();
+        opts.headers
+            .insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
+
+        let err = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect_err("an overflowing closed range must be rejected locally");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), listener.accept())
+                .await
+                .is_err()
+        );
+    }
+}
+
 #[derive(Default)]
 pub struct GetRequest {
     pub buffer: Vec<u8>,

crates/scanner/src/lib.rs

@@ -89,14 +89,17 @@ pub use scanner::{
     scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest,
 };
 pub use scanner_io::{
-    ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
-    record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
-    scanner_maintenance_generation,
+    ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
+    acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
+    scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
 };
 pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
 use std::sync::atomic::{AtomicU64, Ordering};
 pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
-pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
+pub use storage_api::scan::{
+    SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
+    SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
+};
 pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
 
 static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);

crates/scanner/src/scanner_io.rs

@@ -748,9 +748,9 @@ pub(crate) use cache::{
     current_cache_root_or_prepare_with_generation,
 };
 pub use dirty_usage::{
-    ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
-    record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
-    scanner_maintenance_generation,
+    ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
+    acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
+    scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
 };
 #[cfg(test)]
 pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};

crates/scanner/src/scanner_io/dirty_usage.rs

@@ -27,6 +27,25 @@ pub struct ScannerDirtyUsageState {
     pub pending: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ScannerDirtyUsageBucket {
+    pub bucket: String,
+    pub generation: u64,
+}
+
+/// A point-in-time view of the local dirty bucket generations.
+///
+/// `complete == false` is an all-or-nothing overflow signal: `buckets` is
+/// empty and callers must fall back to the global dirty generation rather than
+/// treating a bounded prefix as authoritative.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ScannerDirtyUsageSnapshot {
+    pub generation: u64,
+    pub pending_bucket_count: u64,
+    pub complete: bool,
+    pub buckets: Vec<ScannerDirtyUsageBucket>,
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
 pub enum ScannerDirtyUsageAckError {
     #[error("scanner process instance changed before dirty usage acknowledgement")]
@@ -98,6 +117,35 @@ pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState {
     }
 }
 
+pub fn scanner_dirty_usage_snapshot(max_entries: usize) -> ScannerDirtyUsageSnapshot {
+    let (generation, pending_bucket_count, complete, mut buckets) = {
+        let dirty_buckets = dirty_usage_buckets();
+        let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
+        let pending_bucket_count = usize_to_u64_saturated(dirty_buckets.len());
+        let complete = dirty_buckets.len() <= max_entries;
+        let buckets = if complete {
+            dirty_buckets
+                .iter()
+                .map(|(bucket, generation)| ScannerDirtyUsageBucket {
+                    bucket: bucket.clone(),
+                    generation: *generation,
+                })
+                .collect::<Vec<_>>()
+        } else {
+            Vec::new()
+        };
+        (generation, pending_bucket_count, complete, buckets)
+    };
+    buckets.sort_unstable_by(|left, right| left.bucket.cmp(&right.bucket));
+
+    ScannerDirtyUsageSnapshot {
+        generation,
+        pending_bucket_count,
+        complete,
+        buckets,
+    }
+}
+
 pub fn acknowledge_dirty_usage_generation(
     instance_id: &str,
     generation: u64,

crates/scanner/src/scanner_io/tests.rs

@@ -500,6 +500,51 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
     clear_dirty_usage_buckets_for_tests();
 }
 
+#[test]
+#[serial]
+fn dirty_usage_snapshot_is_sorted_and_reports_its_cutoff() {
+    clear_dirty_usage_buckets_for_tests();
+    let empty = scanner_dirty_usage_snapshot(0);
+    assert_eq!(empty.pending_bucket_count, 0);
+    assert!(empty.complete);
+    assert!(empty.buckets.is_empty());
+
+    record_dirty_usage_bucket("videos");
+    record_dirty_usage_bucket("photos");
+    let expected_generation = scanner_dirty_usage_state().generation;
+
+    let snapshot = scanner_dirty_usage_snapshot(2);
+
+    assert_eq!(snapshot.generation, expected_generation);
+    assert_eq!(snapshot.pending_bucket_count, 2);
+    assert!(snapshot.complete);
+    assert_eq!(
+        snapshot
+            .buckets
+            .iter()
+            .map(|bucket| bucket.bucket.as_str())
+            .collect::<Vec<_>>(),
+        vec!["photos", "videos"]
+    );
+    assert!(snapshot.buckets.iter().all(|bucket| bucket.generation <= snapshot.generation));
+    clear_dirty_usage_buckets_for_tests();
+}
+
+#[test]
+#[serial]
+fn dirty_usage_snapshot_marks_truncated_results_incomplete() {
+    clear_dirty_usage_buckets_for_tests();
+    record_dirty_usage_bucket("archive");
+    record_dirty_usage_bucket("photos");
+
+    let snapshot = scanner_dirty_usage_snapshot(1);
+
+    assert_eq!(snapshot.pending_bucket_count, 2);
+    assert!(!snapshot.complete);
+    assert!(snapshot.buckets.is_empty(), "incomplete snapshots must not expose a partial bucket list");
+    clear_dirty_usage_buckets_for_tests();
+}
+
 #[test]
 #[serial]
 fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {

crates/scanner/src/storage_api.rs

@@ -304,7 +304,10 @@ pub(crate) mod scan {
     };
     #[cfg(test)]
     pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO};
-    pub use super::storage_contracts::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
+    pub use super::storage_contracts::{
+        SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
+        SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
+    };
 }
 
 pub(crate) mod scanner_io {

crates/storage-api/src/lib.rs

@@ -56,6 +56,9 @@ pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
 /// state is not authenticated by that version.
 pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6;
 pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7;
+pub const SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION: u32 = 1;
+pub const SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES: usize = 4096;
+pub const SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE: usize = 512 * 1024;
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 #[serde(deny_unknown_fields)]

crates/storage-api/src/object.rs

@@ -243,6 +243,16 @@ impl ObjectToDelete {
 #[derive(Debug, Default, Clone)]
 pub struct DeletedObject {
     pub delete_marker: bool,
+    /// True when the delete plan looked the target up and found no such
+    /// object or version.
+    ///
+    /// The lookup only runs when the plan needs the source (Object Lock
+    /// check, replication decision, tier journal, or an expected identity),
+    /// so this proves absence and never proves presence: it stays false when
+    /// no lookup ran. Callers that must not announce a delete that removed
+    /// nothing need this, because the disk layer treats an absent version as
+    /// an idempotent success and reports `found` regardless.
+    pub source_missing: bool,
     pub delete_marker_version_id: Option<Uuid>,
     pub object_name: String,
     pub version_id: Option<Uuid>,