Merge branch 'main' into feat/sse-ecstore-decoupling

crates/common/src/metrics.rs

@@ -1114,6 +1114,8 @@ pub struct ScannerLastMinute {
 pub struct ScannerMetricsReport {
     pub collected_at: DateTime<Utc>,
     pub current_cycle: u64,
+    #[serde(default)]
+    pub current_cycle_active: bool,
     pub current_started: DateTime<Utc>,
     pub cycles_completed_at: Vec<DateTime<Utc>>,
     pub ongoing_buckets: usize,
@@ -2051,7 +2053,7 @@ impl Metrics {
     pub fn record_scanner_transition_failed(&self, count: u64) {
         self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
         self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
-        if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
+        if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
             self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
         }
     }
@@ -2336,6 +2338,21 @@ impl Metrics {
         *self.cycle_info.write().await = cycle;
     }
 
+    /// Publish a scanner cycle and its work-accounting baseline as one state transition.
+    pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
+        let mut current_cycle = self.cycle_info.write().await;
+        let snapshot = self.start_scan_cycle_work();
+        *current_cycle = Some(cycle);
+        snapshot
+    }
+
+    /// Publish the completed work snapshot and idle cycle state as one state transition.
+    pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
+        let mut current_cycle = self.cycle_info.write().await;
+        self.finish_scan_cycle_work(start);
+        *current_cycle = Some(cycle);
+    }
+
     /// Read the current cycle record.
     pub async fn get_cycle(&self) -> Option<CurrentCycle> {
         self.cycle_info.read().await.clone()
@@ -2464,7 +2481,7 @@ impl Metrics {
             &self.current_scan_cycle_replication_repair_work_start,
             &replication_repair_snapshot,
         );
-        self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
+        self.current_scan_cycle_work_active.store(true, Ordering::Release);
         snapshot
     }
 
@@ -2476,11 +2493,11 @@ impl Metrics {
         self.record_scan_cycle_work(work);
         self.record_scan_cycle_source_work(&source_work);
         self.record_scan_cycle_replication_repair_work(&replication_repair_work);
-        self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
+        self.current_scan_cycle_work_active.store(false, Ordering::Release);
     }
 
     pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
-        if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
+        if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
             return false;
         }
 
@@ -2746,13 +2763,41 @@ impl Metrics {
     pub async fn report(&self) -> ScannerMetricsReport {
         let mut m = ScannerMetricsReport::default();
 
-        let has_cycle = if let Some(cycle) = self.get_cycle().await {
-            m.current_cycle = cycle.current;
-            m.cycles_completed_at = cycle.cycle_completed;
-            m.current_started = cycle.started;
-            true
-        } else {
-            false
+        let has_cycle = {
+            let cycle = self.cycle_info.read().await;
+            let has_cycle = if let Some(cycle) = cycle.as_ref() {
+                m.current_cycle = cycle.current;
+                m.cycles_completed_at = cycle.cycle_completed.clone();
+                m.current_started = cycle.started;
+                true
+            } else {
+                false
+            };
+            m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
+            if m.current_cycle_active {
+                let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
+                let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
+                let current_replication_repair_work =
+                    self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
+                m.current_cycle_objects_scanned = current_work.objects_scanned;
+                m.current_cycle_directories_scanned = current_work.directories_scanned;
+                m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
+                m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
+                m.current_cycle_yield_events = current_work.yield_events;
+                m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
+                m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
+                m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
+                m.current_cycle_ilm_actions = current_work.ilm_actions;
+                m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
+                m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
+                m.current_cycle_heal_objects = current_work.heal_objects;
+                m.current_cycle_replication_checks = current_work.replication_checks;
+                m.current_cycle_usage_saves = current_work.usage_saves;
+                m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
+                m.current_cycle_replication_repair =
+                    self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
+            }
+            has_cycle
         };
 
         if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
@@ -2793,28 +2838,6 @@ impl Metrics {
         m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
         m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
         m.current_disk_bucket_scans_active = disk_bucket_scans_active;
-        if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
-            let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
-            let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
-            let current_replication_repair_work =
-                self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
-            m.current_cycle_objects_scanned = current_work.objects_scanned;
-            m.current_cycle_directories_scanned = current_work.directories_scanned;
-            m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
-            m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
-            m.current_cycle_yield_events = current_work.yield_events;
-            m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
-            m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
-            m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
-            m.current_cycle_ilm_actions = current_work.ilm_actions;
-            m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
-            m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
-            m.current_cycle_heal_objects = current_work.heal_objects;
-            m.current_cycle_replication_checks = current_work.replication_checks;
-            m.current_cycle_usage_saves = current_work.usage_saves;
-            m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
-            m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
-        }
         let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
         m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
         m.last_cycle_result_code = last_cycle_result as u64;
@@ -4142,6 +4165,8 @@ mod tests {
 
         let report = metrics.report().await;
 
+        assert!(report.current_cycle_active);
+        assert_eq!(report.current_cycle, 0);
         assert_eq!(report.current_cycle_objects_scanned, 7);
         assert_eq!(report.current_cycle_directories_scanned, 3);
         assert_eq!(report.current_cycle_bucket_drive_scans, 2);
@@ -4158,6 +4183,8 @@ mod tests {
         metrics.finish_scan_cycle_work(start);
         let report = metrics.report().await;
 
+        assert!(!report.current_cycle_active);
+        assert_eq!(report.current_cycle, 0);
         assert_eq!(report.current_cycle_objects_scanned, 0);
         assert_eq!(report.current_cycle_directories_scanned, 0);
         assert_eq!(report.current_cycle_bucket_drive_scans, 0);
@@ -4184,6 +4211,91 @@ mod tests {
         assert_eq!(report.last_cycle_usage_saves, 2);
     }
 
+    #[tokio::test]
+    async fn scan_cycle_activity_and_cycle_state_publish_together() {
+        let metrics = Metrics::new();
+        let cycle_started = Utc::now() - chrono::Duration::seconds(5);
+        let active_cycle = CurrentCycle {
+            current: 12,
+            next: 13,
+            started: cycle_started,
+            ..Default::default()
+        };
+
+        let cycle_state = metrics.cycle_info.read().await;
+        let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
+        let waker = std::task::Waker::noop();
+        let mut context = std::task::Context::from_waker(waker);
+        assert!(start_transition.as_mut().poll(&mut context).is_pending());
+        assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
+        drop(cycle_state);
+
+        let start = start_transition.await;
+        let active = metrics.report().await;
+        assert!(active.current_cycle_active);
+        assert_eq!(active.current_cycle, 12);
+        assert_eq!(active.current_started, cycle_started);
+
+        let idle_cycle = CurrentCycle {
+            current: 0,
+            next: 13,
+            started: cycle_started,
+            ..Default::default()
+        };
+        let cycle_state = metrics.cycle_info.read().await;
+        let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
+        assert!(finish_transition.as_mut().poll(&mut context).is_pending());
+        assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
+        drop(cycle_state);
+
+        finish_transition.await;
+        let idle = metrics.report().await;
+        assert!(!idle.current_cycle_active);
+        assert_eq!(idle.current_cycle, 0);
+    }
+
+    #[tokio::test]
+    async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
+        let metrics = Metrics::new();
+        let cycle_ten = CurrentCycle {
+            current: 10,
+            next: 11,
+            started: Utc::now() - chrono::Duration::seconds(10),
+            ..Default::default()
+        };
+        let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
+        metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
+
+        let paths = metrics.current_paths.write().await;
+        let mut report = Box::pin(metrics.report());
+        let waker = std::task::Waker::noop();
+        let mut context = std::task::Context::from_waker(waker);
+        assert!(report.as_mut().poll(&mut context).is_pending());
+
+        metrics
+            .finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
+            .await;
+        let cycle_eleven_start = metrics
+            .start_scan_cycle_work_with_cycle(CurrentCycle {
+                current: 11,
+                next: 12,
+                started: Utc::now(),
+                ..Default::default()
+            })
+            .await;
+        metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
+
+        drop(paths);
+        let snapshot = report.await;
+
+        assert_eq!(snapshot.current_cycle, 10);
+        assert_eq!(snapshot.current_cycle_objects_scanned, 1);
+
+        metrics
+            .finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
+            .await;
+    }
+
     #[tokio::test]
     async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
         let metrics = Metrics::new();

crates/config/src/constants/internode.rs

@@ -158,17 +158,33 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
 // rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
 const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
 
-/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
-/// one-time consumption of body-bound v2 signatures.
+/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
 ///
-/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
-/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
-/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
-/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
-/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
-/// shared secret) — and increments
+/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
+/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
+/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
+/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
+/// immediately retry with the replay-scoped signature after a peer restart.
+pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
+pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
+
+// Compile-time invariant: mixed-version clusters must remain available until operators make the
+// observed fallback counter an explicit strictness decision.
+const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
+
+/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
+/// one-time consumption of authenticated RPC signatures.
+///
+/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
+/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
+/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
+/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
+/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
+/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
+/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
+/// the shared secret) — and increments
 /// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
-/// counter means this capacity is undersized for the node's peak mutation rate.
+/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
 pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
 pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
 
@@ -354,6 +370,12 @@ mod tests {
         assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
     }
 
+    #[test]
+    fn internode_replay_scope_strict_env_name_is_stable() {
+        // The fail-open default invariant is asserted at compile time next to the definition.
+        assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
+    }
+
     #[test]
     fn internode_replay_cache_capacity_defaults_and_env_name() {
         assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");

crates/data-usage/Cargo.toml

@@ -29,7 +29,6 @@ workspace = true
 
 [dependencies]
 serde = { workspace = true, features = ["derive"] }
-path-clean = { workspace = true }
 rmp-serde = { workspace = true }
 async-trait = { workspace = true }
 rustfs-filemeta = { workspace = true }

crates/data-usage/src/data_usage.rs

@@ -12,12 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use path_clean::PathClean;
 use serde::{Deserialize, Serialize};
 use std::{
     collections::{HashMap, HashSet},
     hash::{DefaultHasher, Hash, Hasher},
-    path::Path,
     time::{Duration, SystemTime},
 };
 
@@ -334,7 +332,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
 impl SizeHistogram {
     pub fn add(&mut self, size: u64) {
         let intervals = [
-            (0, 1024),                                  // LESS_THAN_1024_B
+            (0, 1024 - 1),                              // LESS_THAN_1024_B
             (1024, 64 * 1024 - 1),                      // BETWEEN_1024_B_AND_64_KB
             (64 * 1024, 256 * 1024 - 1),                // BETWEEN_64_KB_AND_256_KB
             (256 * 1024, 512 * 1024 - 1),               // BETWEEN_256_KB_AND_512_KB
@@ -362,7 +360,7 @@ impl SizeHistogram {
         // the sub-ranges in [1 KiB, 512 KiB).
         const ONE_MIB: u64 = 1024 * 1024;
         let intervals = [
-            (0, 1024),                          // LESS_THAN_1024_B
+            (0, 1024 - 1),                      // LESS_THAN_1024_B
             (1024, 64 * 1024 - 1),              // BETWEEN_1024_B_AND_64_KB
             (64 * 1024, 256 * 1024 - 1),        // BETWEEN_64_KB_AND_256_KB
             (256 * 1024, 512 * 1024 - 1),       // BETWEEN_256_KB_AND_512_KB
@@ -1110,9 +1108,39 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
     }
 }
 
-/// Hash a path for data usage caching
+fn clean_data_usage_path(data: &str) -> String {
+    let rooted = data.starts_with('/');
+    let mut parts = Vec::new();
+
+    for part in data.split('/') {
+        match part {
+            "" | "." => {}
+            ".." => {
+                if parts.last().is_some_and(|last| *last != "..") {
+                    parts.pop();
+                } else if !rooted {
+                    parts.push(part);
+                }
+            }
+            _ => parts.push(part),
+        }
+    }
+
+    let clean = parts.join("/");
+    match (rooted, clean.is_empty()) {
+        (true, true) => "/".to_string(),
+        (true, false) => format!("/{clean}"),
+        (false, true) => ".".to_string(),
+        (false, false) => clean,
+    }
+}
+
+/// Hash a slash-separated path for data usage caching.
+///
+/// Cache identifiers are persisted and exchanged across nodes, so their
+/// normalization must not depend on the host operating system.
 pub fn hash_path(data: &str) -> DataUsageHash {
-    DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
+    DataUsageHash(clean_data_usage_path(data))
 }
 
 impl DataUsageInfo {
@@ -1497,6 +1525,23 @@ mod tests {
         buckets_count: u64,
     }
 
+    #[test]
+    fn hash_path_uses_portable_slash_semantics() {
+        for (input, expected) in [
+            ("", "."),
+            (".", "."),
+            ("/", "/"),
+            ("//bucket///prefix/", "/bucket/prefix"),
+            ("bucket/./prefix//object", "bucket/prefix/object"),
+            ("bucket/a/../b", "bucket/b"),
+            ("../bucket/..", ".."),
+            ("/../../bucket", "/bucket"),
+            ("bucket\\prefix/object", "bucket\\prefix/object"),
+        ] {
+            assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
+        }
+    }
+
     #[test]
     fn completeness_marker_is_additive_for_legacy_named_readers() {
         let current = DataUsageInfo {
@@ -1601,6 +1646,49 @@ mod tests {
         assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
     }
 
+    #[test]
+    fn test_size_histogram_classifies_adjacent_boundaries_once() {
+        let cases = [
+            (1023, 0),
+            (1024, 1),
+            (64 * 1024 - 1, 1),
+            (64 * 1024, 2),
+            (256 * 1024 - 1, 2),
+            (256 * 1024, 3),
+            (512 * 1024 - 1, 3),
+            (512 * 1024, 4),
+            (1024 * 1024 - 1, 4),
+            (1024 * 1024, 6),
+            (10 * 1024 * 1024 - 1, 6),
+            (10 * 1024 * 1024, 7),
+            (64 * 1024 * 1024 - 1, 7),
+            (64 * 1024 * 1024, 8),
+            (128 * 1024 * 1024 - 1, 8),
+            (128 * 1024 * 1024, 9),
+            (512 * 1024 * 1024 - 1, 9),
+            (512 * 1024 * 1024, 10),
+        ];
+
+        for (size, expected_bucket) in cases {
+            let mut hist = SizeHistogram::default();
+            hist.add(size);
+
+            assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
+            assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
+        }
+    }
+
+    #[test]
+    fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
+        let mut hist = SizeHistogram::default();
+        hist.add(1024);
+
+        let map = hist.to_map();
+        assert_eq!(map["LESS_THAN_1024_B"], 0);
+        assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
+        assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
+    }
+
     #[test]
     fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
         let mut hist = SizeHistogram::default();

crates/e2e_test/src/internode_rpc_signature_e2e_test.rs

@@ -13,8 +13,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
-//! signature (<https://github.com/rustfs/backlog/issues/1327>).
+//! Cross-process replay / tamper acceptance for internode NodeService RPC
+//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
+//! <https://github.com/rustfs/backlog/issues/1542>).
 //!
 //! # Why this exists on top of the in-process tests
 //!
@@ -78,6 +79,8 @@
 //! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
 //! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
 //! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
+//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
+//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
 //!
 //! Two acceptance items are deliberately left to the in-process tests. A stale
 //! timestamp cannot be forged from outside — it is inside the HMAC — so
@@ -88,18 +91,20 @@
 
 use crate::common::{RustFSTestEnvironment, init_logging};
 use crate::storage_api::internode_rpc_signature::{
-    TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
+    TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+    node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
 };
 use http::{HeaderMap, Method};
 use rustfs_config::{
-    ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
+    ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
+    ENV_INTERNODE_RPC_SIGNATURE_STRICT,
 };
 use rustfs_protos::canonical_make_volume_request_body;
-use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
+use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
 use serial_test::serial;
 use sha2::{Digest, Sha256};
 use std::error::Error;
-use tonic::{Code, Request, Status};
+use tonic::{Code, Request, Response, Status};
 use uuid::Uuid;
 
 type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
@@ -115,12 +120,14 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
 /// clears authentication stops harmlessly at `find_disk`.
 const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
 
-/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
-/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
-/// asserts the header it replaces was actually present, which turns a rename
-/// into a loud failure instead of silently reducing an attack to a no-op.
+/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
+/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
+/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
+/// rename into a loud failure instead of silently reducing an attack to a no-op.
 const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
 const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
+const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
+const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
 
 /// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
 /// without its leading `/`.
@@ -155,19 +162,28 @@ fn align_rpc_secret_with_server() {
 ///
 /// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
 /// to other tests running in the same binary.
-async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
-    let mut env = RustFSTestEnvironment::new().await?;
+fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
     let mut child_env = vec![
         ("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
         (ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
         (ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
+        (ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
         (ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
     ];
     child_env.extend_from_slice(extra_env);
-    env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
+    child_env
+}
+
+async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
     Ok(env)
 }
 
+async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
+    start_server_with_env(&server_env(extra_env)).await
+}
+
 /// Stop the child and drop the cached gRPC channel for its address.
 ///
 /// `node_service_time_out_client_no_auth` memoises channels in a process-global
@@ -238,12 +254,83 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
 /// and nothing else — no interceptor adds or rewrites auth metadata, so the
 /// bytes on the wire are the ones the test chose.
 async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
+    call_make_volume_response(url, request, headers)
+        .await
+        .map(Response::into_inner)
+}
+
+async fn call_make_volume_response(
+    url: &str,
+    request: MakeVolumeRequest,
+    headers: HeaderMap,
+) -> Result<Response<MakeVolumeResponse>, Status> {
     let mut client = node_service_time_out_client_no_auth(&url.to_string())
         .await
         .map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
     let mut rpc_request = Request::new(request);
     rpc_request.metadata_mut().as_mut().extend(headers);
-    client.make_volume(rpc_request).await.map(|response| response.into_inner())
+    client.make_volume(rpc_request).await
+}
+
+async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
+    let mut client = node_service_time_out_client_no_auth(&url.to_string())
+        .await
+        .map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
+    let mut rpc_request = Request::new(PingRequest {
+        version: 1,
+        body: bytes::Bytes::new(),
+    });
+    rpc_request.metadata_mut().as_mut().extend(headers);
+    client.ping(rpc_request).await
+}
+
+fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
+    let challenge = Uuid::new_v4();
+    headers.insert(
+        BOOT_EPOCH_CHALLENGE_HEADER,
+        challenge.to_string().parse().expect("UUID must be a valid header value"),
+    );
+    challenge
+}
+
+fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
+    let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
+    let timestamp = headers
+        .get(TIMESTAMP_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .expect("v2 headers must carry a timestamp")
+        .to_string();
+    headers.extend(
+        gen_tonic_replay_scope_headers(audience, path, &timestamp, content_sha256, boot_epoch)
+            .expect("replay-scope headers must mint with the aligned RPC secret"),
+    );
+    headers
+}
+
+async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
+    let request = make_volume_request("signature-e2e-epoch-bootstrap");
+    let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
+    let challenge = attach_boot_epoch_challenge(&mut headers);
+    let response = call_make_volume_response(url, request, headers)
+        .await
+        .expect("v2 request with epoch challenge must clear default authentication");
+    let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
+        .expect("server must HMAC-authenticate the advertised boot epoch");
+    assert_authenticated(
+        Ok(response.into_inner()),
+        "a v2 epoch-challenge request in the default replay-scope posture",
+    );
+    boot_epoch
+}
+
+async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
+    let mut headers = mint_v2_headers(audience, "Ping", None);
+    let challenge = attach_boot_epoch_challenge(&mut headers);
+    let response = call_ping_response(url, headers)
+        .await
+        .expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
+    verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
+        .expect("strict replay-scope Ping must return a valid boot epoch proof")
 }
 
 /// Assert a call cleared authentication.
@@ -332,6 +419,122 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
     Ok(())
 }
 
+/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
+/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
+/// epoch is learned from a real response, then the same server is restarted in place to prove its
+/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
+#[tokio::test]
+#[serial]
+async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
+    init_logging();
+    align_rpc_secret_with_server();
+    let child_env = server_env(&[]);
+    let mut env = start_server_with_env(&child_env).await?;
+    let url = env.url.clone();
+    let audience = audience_of(&env);
+    let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
+
+    let request = make_volume_request("replay-scope-e2e-once");
+    let captured = mint_replay_scope_headers(
+        &audience,
+        &format!("{TONIC_RPC_PREFIX}/MakeVolume"),
+        &canonical_digest(&request),
+        boot_epoch,
+    );
+    assert_authenticated(
+        call_make_volume(&url, request.clone(), captured.clone()).await,
+        "the first replay-scoped mutation delivery",
+    );
+    assert_rejected(
+        call_make_volume(&url, request.clone(), captured).await,
+        Code::Unauthenticated,
+        None,
+        "the same replay-scoped mutation delivered twice",
+    );
+
+    let transplanted =
+        mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
+    assert_rejected(
+        call_make_volume(&url, request.clone(), transplanted).await,
+        Code::Unauthenticated,
+        None,
+        "a replay-scoped Ping signature transplanted onto MakeVolume",
+    );
+
+    let stale_epoch = mint_replay_scope_headers(
+        &audience,
+        &format!("{TONIC_RPC_PREFIX}/MakeVolume"),
+        &canonical_digest(&request),
+        boot_epoch,
+    );
+    env.restart_server_preserving_data(Vec::new(), &child_env).await?;
+    rustfs_protos::evict_failed_connection(&url).await;
+    assert_rejected(
+        call_make_volume(&url, request.clone(), stale_epoch).await,
+        Code::Unauthenticated,
+        None,
+        "a replay-scoped signature captured before the receiving process restart",
+    );
+
+    let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
+    assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
+    let fresh_epoch = mint_replay_scope_headers(
+        &audience,
+        &format!("{TONIC_RPC_PREFIX}/MakeVolume"),
+        &canonical_digest(&request),
+        restarted_epoch,
+    );
+    assert_authenticated(
+        call_make_volume(&url, request, fresh_epoch).await,
+        "a replay-scoped mutation signed with the replacement process epoch",
+    );
+
+    stop_server(env, &url).await;
+    Ok(())
+}
+
+/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
+/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
+/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
+#[tokio::test]
+#[serial]
+async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
+    init_logging();
+    align_rpc_secret_with_server();
+    let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
+    let url = env.url.clone();
+    let audience = audience_of(&env);
+
+    let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
+    assert_rejected(
+        call_make_volume(
+            &url,
+            v2_request.clone(),
+            mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
+        )
+        .await,
+        Code::Unauthenticated,
+        None,
+        "a v2 mutation after replay-scope strictness is enabled",
+    );
+
+    let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
+    let request = make_volume_request("replay-scope-e2e-strict-v3");
+    let replay_scoped = mint_replay_scope_headers(
+        &audience,
+        &format!("{TONIC_RPC_PREFIX}/MakeVolume"),
+        &canonical_digest(&request),
+        boot_epoch,
+    );
+    assert_authenticated(
+        call_make_volume(&url, request, replay_scoped).await,
+        "a replay-scoped mutation after Ping bootstrap under strict replay scope",
+    );
+
+    stop_server(env, &url).await;
+    Ok(())
+}
+
 /// Baseline: correctly signed mutations are accepted, both with and without a
 /// body digest.
 ///

crates/e2e_test/src/notification_webhook_test.rs

@@ -21,18 +21,22 @@
 //! function, never as an S3 event sink.
 //!
 //! Coverage:
-//!   * PUT / multipart-complete / DELETE each deliver one event with the correct
+//!   * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
 //!     eventName, bucket, key, versionId and eTag.
 //!   * prefix/suffix filters drop non-matching keys (rule-engine gate).
 //!   * an event queued while the target endpoint is unreachable is redelivered
 //!     from the on-disk store once the endpoint recovers (store-and-forward).
+//!   * responseElements and the S3 response use the canonical request ID while
+//!     requestParameters preserve a conflicting client-supplied value.
 
 use crate::common::{RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
+use aws_sdk_s3::operation::RequestId;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
-    NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
+    NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
+    VersioningConfiguration,
 };
 use http::header::{CONTENT_TYPE, HOST};
 use local_ip_address::local_ip;
@@ -40,10 +44,12 @@ use reqwest::StatusCode;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
+use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
 use s3s::Body;
 use serde_json::Value;
 use serial_test::serial;
 use std::error::Error;
+use std::io::Cursor;
 use std::path::Path;
 use std::sync::{
     Arc, Once,
@@ -63,6 +69,8 @@ type BoxError = Box<dyn Error + Send + Sync>;
 /// for target lookup (see `process_queue_configurations`), so the region is
 /// nominal, but it must be present for `ARN::parse` to succeed.
 const NOTIFY_REGION: &str = "us-east-1";
+const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
+const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
 
 /// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
 /// so the ARN a notification rule references is
@@ -579,6 +587,36 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
     value.map(|e| e.trim_matches('"').to_string())
 }
 
+fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
+    assert_eq!(
+        record["requestParameters"][REQUEST_ID_HEADER].as_str(),
+        Some(CLIENT_REQUEST_ID),
+        "notification request parameters should retain the actual client header: {record}"
+    );
+    assert_eq!(
+        record["requestParameters"][AMZ_REQUEST_ID].as_str(),
+        Some(CLIENT_AMZ_REQUEST_ID),
+        "notification request parameters should retain the actual client header: {record}"
+    );
+    assert_eq!(
+        record["responseElements"][AMZ_REQUEST_ID].as_str(),
+        Some(server_request_id),
+        "notification response elements should use the canonical request ID: {record}"
+    );
+}
+
+fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
+    assert!(
+        record["requestParameters"][AMZ_REQUEST_ID].is_null(),
+        "notification request parameters must not invent a client request header: {record}"
+    );
+    assert_eq!(
+        record["responseElements"][AMZ_REQUEST_ID].as_str(),
+        Some(request_id),
+        "notification response elements should match the S3 response request ID: {record}"
+    );
+}
+
 // ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
@@ -671,8 +709,17 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         .bucket(bucket)
         .key(put_key)
         .body(ByteStream::from_static(b"peri-1 put body"))
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
+            request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
+        })
         .send()
         .await?;
+    let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
+    assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
+    assert_ne!(put_request_id, CLIENT_REQUEST_ID);
+    assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
     let put_version = put
         .version_id()
         .ok_or("PUT response missing versionId (versioning not enabled?)")?;
@@ -692,6 +739,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         "record eventName: {record}"
     );
     assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
+    assert_conflicting_request_id_correlation(record, &put_request_id);
     assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
     assert_eq!(
         trimmed_etag(object["eTag"].as_str()),
@@ -744,6 +792,35 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         "multipart eTag in event: {mp_record}"
     );
 
+    // --- Snowball extract: direct notification path keeps response correlation
+    let snowball_key = "uploads/snowball.dat";
+    let snowball_body = b"snowball notification body";
+    let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+    let mut archive_header = tokio_tar::Header::new_gnu();
+    archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
+    archive_header.set_mode(0o644);
+    archive_header.set_cksum();
+    archive_builder
+        .append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
+        .await?;
+    let archive = archive_builder.into_inner().await?.into_inner();
+    let snowball = client
+        .put_object()
+        .bucket(bucket)
+        .key("snowball-fixture.tar")
+        .body(ByteStream::from(archive))
+        .customize()
+        .mutate_request(|request| {
+            request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
+        })
+        .send()
+        .await?;
+    let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
+
+    let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+    let snowball_record = &snowball_event["Records"][0];
+    assert_generated_request_id_correlation(snowball_record, snowball_request_id);
+
     // --- Filter: non-matching prefix and suffix must never be delivered ------
     let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
     let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
@@ -772,7 +849,32 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         "wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
     );
 
-    // --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
+    // --- DeleteObjects: direct notification path keeps response correlation --
+    let delete_many_key = "uploads/delete-many.dat";
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(delete_many_key)
+        .body(ByteStream::from_static(b"delete objects notification body"))
+        .send()
+        .await?;
+    wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+
+    let delete_many = client
+        .delete_objects()
+        .bucket(bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
+    let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
+    assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
+
+    // --- DeleteObject on a versioned bucket: delete-marker version ----------
     let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
     let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
     let removed_record = &removed["Records"][0];

crates/e2e_test/src/reliant/grpc_lock_client.rs

@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
 use tonic::Request;
 use tracing::{info, warn};
 
-use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
+use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 
 /// gRPC lock client without authentication for testing
 /// Similar to RemoteClient but uses no_auth client
@@ -42,7 +42,7 @@ impl GrpcLockClient {
         &self,
     ) -> Result<
         rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
-            tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
+            tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
         >,
     > {
         node_service_time_out_client_no_auth(&self.addr)

crates/e2e_test/src/storage_api.rs

@@ -16,9 +16,12 @@
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
+pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 #[cfg(test)]
-pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
-pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
+pub(crate) use rustfs_ecstore::api::rpc::{
+    TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+    verify_tonic_boot_epoch_response,
+};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
 
@@ -30,7 +33,7 @@ pub(crate) mod node_interact {
 }
 
 pub(crate) mod grpc_lock {
-    pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
+    pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 }
 
 /// Signing/transport surface used by the cross-process internode RPC signature
@@ -40,7 +43,8 @@ pub(crate) mod grpc_lock {
 #[cfg(test)]
 pub(crate) mod internode_rpc_signature {
     pub(crate) use super::{
-        TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
+        TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+        node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
     };
 }
 

crates/ecstore/Cargo.toml

@@ -150,11 +150,18 @@ metrics = { workspace = true }
 [target.'cfg(target_os = "linux")'.dependencies]
 rustfs-uring = "0.2.1"
 
+[target.'cfg(windows)'.dependencies]
+winapi-util.workspace = true
+windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
 criterion = { workspace = true, features = ["html_reports"] }
 temp-env = { workspace = true, features = ["async_closure"] }
 tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
+# Only for `pin_callsite_interest_for_test`, which registers a `NoSubscriber`
+# dispatcher to keep tracing's process-global callsite-interest cache honest.
+tracing-core = { workspace = true }
 serial_test = { workspace = true }
 opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 proptest = "1"

crates/ecstore/src/api/mod.rs

@@ -67,6 +67,14 @@ pub mod bucket {
             };
         }
 
+        pub mod transition_transaction {
+            pub use crate::bucket::lifecycle::transition_transaction::{
+                TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator,
+            };
+        }
+
         pub mod evaluator {
             pub use crate::bucket::lifecycle::evaluator::Evaluator;
         }
@@ -122,13 +130,13 @@ pub mod bucket {
 
     pub mod metadata_sys {
         pub use crate::bucket::metadata_sys::{
-            BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
+            BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
             get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
             get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
             init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
-            update, update_bucket_targets_under_transaction_lock, update_config_with,
+            update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
         };
     }
 
@@ -377,6 +385,7 @@ pub mod metrics {
 pub mod notification {
     pub use crate::services::notification_sys::{
         NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
+        start_remote_version_state_fleet_probe,
     };
 }
 
@@ -409,13 +418,15 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
-        SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
-        TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
-        node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
-        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
-        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature,
+        AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
+        PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
+        ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
+        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
+        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -23,6 +23,7 @@ use uuid::Uuid;
 use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
+    delete_confirmed_transition_candidate_exact_with_lease_idempotent,
     delete_confirmed_transition_candidate_exact_with_manager_and_identity,
     delete_object_from_remote_tier_idempotent_with_manager_and_identity,
 };
@@ -616,6 +617,199 @@ pub enum TransitionTransactionRecoveryOutcome {
     Retained,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TransitionOperatorProbe {
+    Missing,
+    UnversionedPresent,
+    VersionedPresent(String),
+    Ambiguous,
+    Unsupported,
+}
+
+impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
+    fn from(value: TransitionCandidateProbe) -> Self {
+        match value {
+            TransitionCandidateProbe::Missing => Self::Missing,
+            TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
+            TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
+            TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
+            TransitionCandidateProbe::Unsupported => Self::Unsupported,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TransitionOperatorStatus {
+    pub transaction_id: Uuid,
+    pub state: TransitionTransactionState,
+    pub tier_name: String,
+    pub remote_object: String,
+    pub not_after_unix_nanos: i64,
+    pub probe: TransitionOperatorProbe,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TransitionOperatorDeleteResult {
+    pub status: TransitionOperatorStatus,
+    pub journal_observed_after_delete: bool,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum TransitionOperatorError {
+    #[error("transition transaction was not found")]
+    NotFound,
+    #[error("transition transaction is still inside its active ownership window")]
+    NotExpired,
+    #[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
+    InvalidState(TransitionTransactionState),
+    #[error("an exact non-empty remote version is required")]
+    RemoteVersionRequired,
+    #[error("remote candidate is not proven missing: {0:?}")]
+    CandidateNotMissing(TransitionOperatorProbe),
+    #[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
+    CandidateVersionMismatch {
+        expected: String,
+        actual: TransitionOperatorProbe,
+    },
+    #[error("transition transaction store failed: {0}")]
+    Store(#[source] Error),
+    #[error("remote tier reconciliation failed: {0}")]
+    Remote(#[source] std::io::Error),
+}
+
+type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
+
+fn validate_operator_reconcile_transaction(
+    transaction: &TransitionTransaction,
+    now_unix_nanos: i128,
+) -> TransitionOperatorResult<()> {
+    transaction
+        .validate()
+        .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
+    if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
+        return Err(TransitionOperatorError::InvalidState(transaction.state));
+    }
+    if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
+        return Err(TransitionOperatorError::NotExpired);
+    }
+    Ok(())
+}
+
+async fn load_operator_reconcile_transaction(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<TransitionTransaction> {
+    match load_transition_transaction_record(api, transaction_id).await {
+        Ok(transaction) => Ok(transaction),
+        Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
+        Err(err) => Err(TransitionOperatorError::Store(err)),
+    }
+}
+
+async fn operator_probe_transition_candidate(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> TransitionOperatorResult<TransitionOperatorProbe> {
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
+    lease
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
+        .await
+        .map(TransitionOperatorProbe::from)
+        .map_err(TransitionOperatorError::Remote)
+}
+
+pub async fn inspect_transition_transaction_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<TransitionOperatorStatus> {
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let probe = operator_probe_transition_candidate(api, &transaction).await?;
+    Ok(TransitionOperatorStatus {
+        transaction_id,
+        state: transaction.state,
+        tier_name: transaction.tier_name,
+        remote_object: transaction.remote_object,
+        not_after_unix_nanos: transaction.not_after_unix_nanos,
+        probe,
+    })
+}
+
+pub async fn delete_transition_candidate_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+    remote_version_id: &str,
+) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
+    if remote_version_id.is_empty() {
+        return Err(TransitionOperatorError::RemoteVersionRequired);
+    }
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
+    lease
+        .validate_remote_version_id(remote_version_id)
+        .map_err(TransitionOperatorError::Remote)?;
+    let before_delete_probe = lease
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
+        .await
+        .map(TransitionOperatorProbe::from)
+        .map_err(TransitionOperatorError::Remote)?;
+    if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
+        return Err(TransitionOperatorError::CandidateVersionMismatch {
+            expected: remote_version_id.to_string(),
+            actual: before_delete_probe,
+        });
+    }
+    delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
+        .await
+        .map_err(TransitionOperatorError::Remote)?;
+    let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
+    let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
+        Ok(_) => true,
+        Err(Error::ConfigNotFound) => false,
+        Err(err) => return Err(TransitionOperatorError::Store(err)),
+    };
+    Ok(TransitionOperatorDeleteResult {
+        status: TransitionOperatorStatus {
+            transaction_id,
+            state: transaction.state,
+            tier_name: transaction.tier_name,
+            remote_object: transaction.remote_object,
+            not_after_unix_nanos: transaction.not_after_unix_nanos,
+            probe,
+        },
+        journal_observed_after_delete,
+    })
+}
+
+pub async fn finalize_missing_transition_transaction_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<()> {
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
+    if probe != TransitionOperatorProbe::Missing {
+        return Err(TransitionOperatorError::CandidateNotMissing(probe));
+    }
+    delete_transition_transaction_record(api, transaction_id)
+        .await
+        .map_err(TransitionOperatorError::Store)
+}
+
 pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
     let transaction_id = transition_transaction_id_from_record_object_name(object)?;
     TransitionTransaction::decode(transaction_id, data)
@@ -700,7 +894,7 @@ async fn recover_unknown_upload_outcome(
     .map_err(Error::other)?;
 
     match lease
-        .probe_transition_candidate(&transaction.remote_object)
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
         .await
         .map_err(Error::other)?
     {
@@ -1097,6 +1291,27 @@ mod tests {
             .expect("upload state change should succeed")
     }
 
+    #[test]
+    fn operator_reconcile_requires_expired_unknown_upload_outcome() {
+        let mut transaction = new_transaction();
+        let active_deadline = transaction.not_after_unix_nanos;
+
+        assert!(matches!(
+            validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
+            Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
+        ));
+
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("unknown upload outcome should be recorded");
+        assert!(matches!(
+            validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
+            Err(TransitionOperatorError::NotExpired)
+        ));
+        validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
+            .expect("expired unknown upload outcome should be eligible");
+    }
+
     fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
         TransitionCleanupProof {
             transaction_id: transaction.transaction_id,

crates/ecstore/src/client/provider_versions.rs

@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
     }
 }
 
-fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
+pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
     if version_id.is_empty() {
         return Err(Error::new(
             ErrorKind::InvalidData,

crates/ecstore/src/cluster/rpc/client.rs

@@ -12,11 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
-use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
+#[cfg(test)]
+use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
+use crate::cluster::rpc::http_auth::{
+    RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
+    RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
+};
+use crate::cluster::rpc::{
+    gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
+};
+#[cfg(test)]
+use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
 use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
 use crate::runtime::sources as runtime_sources;
-use http::Uri;
+use http::{Request as HttpRequest, Response as HttpResponse, Uri};
 use rustfs_protos::{
     ChannelClass, create_new_channel, get_channel_for_class,
     proto_gen::node_service::{
@@ -24,9 +33,19 @@ use rustfs_protos::{
         tier_mutation_control_service_client::TierMutationControlServiceClient,
     },
 };
-use std::{error::Error, io::ErrorKind};
+use std::{
+    collections::HashMap,
+    error::Error,
+    future::Future,
+    io::ErrorKind,
+    pin::Pin,
+    sync::{LazyLock, Mutex},
+    task::{Context, Poll},
+};
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
+use tower::Service;
 use tracing::debug;
+use uuid::Uuid;
 
 use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
 
@@ -35,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
 pub async fn node_service_time_out_client(
     addr: &String,
     interceptor: TonicInterceptor,
-) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     // Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
     // `_for_class` variant below (grpc-optimization P1).
     node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
@@ -44,13 +63,14 @@ pub async fn node_service_time_out_client(
 pub async fn heal_control_time_out_client(
     addr: &str,
     interceptor: TonicInterceptor,
-) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = match runtime_sources::cached_node_channel(addr).await {
         Some(channel) => channel,
         None => create_new_channel(addr).await?,
     };
     let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
     Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -59,13 +79,14 @@ pub async fn heal_control_time_out_client(
 pub async fn tier_mutation_control_time_out_client(
     addr: &str,
     interceptor: TonicInterceptor,
-) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = match runtime_sources::cached_node_channel(addr).await {
         Some(channel) => channel,
         None => create_new_channel(addr).await?,
     };
     let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
     Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -81,7 +102,7 @@ pub async fn node_service_time_out_client_for_class(
     addr: &String,
     interceptor: TonicInterceptor,
     class: ChannelClass,
-) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = match class {
         ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
@@ -96,6 +117,7 @@ pub async fn node_service_time_out_client_for_class(
     };
 
     let max_message_size = rustfs_protos::internode_rpc_max_message_size();
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
     Ok(NodeServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -103,7 +125,7 @@ pub async fn node_service_time_out_client_for_class(
 
 pub async fn node_service_time_out_client_no_auth(
     addr: &String,
-) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
 }
 
@@ -199,6 +221,104 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
     }
 }
 
+/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
+/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
+/// old servers continue receiving precisely the metadata they understand.
+#[derive(Clone, Debug)]
+pub struct ReplayScopeChannel<S> {
+    inner: S,
+    audience: Option<String>,
+}
+
+/// The channel type used by internode clients after v2 authentication and replay-scope handling.
+pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
+
+static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+impl<S> ReplayScopeChannel<S> {
+    fn new(inner: S, audience: Option<String>) -> Self {
+        Self { inner, audience }
+    }
+}
+
+fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
+    PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
+}
+
+fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
+    if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
+        epochs.insert(audience, epoch);
+    }
+}
+
+impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
+where
+    S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
+    S::Error: Send + 'static,
+    S::Future: Send + 'static,
+    ReqBody: Send + 'static,
+    ResBody: Send + 'static,
+{
+    type Response = HttpResponse<ResBody>;
+    type Error = S::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
+        let authenticated = self.audience.as_ref().is_some_and(|_| {
+            request
+                .headers()
+                .get(RPC_AUTH_VERSION_HEADER)
+                .and_then(|value| value.to_str().ok())
+                == Some(RPC_AUTH_VERSION_V2)
+        });
+        let challenge = authenticated.then(Uuid::new_v4);
+        if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
+            // The challenge is independently HMAC-authenticated by the response proof. It is not
+            // part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
+            request.headers_mut().insert(
+                RPC_BOOT_EPOCH_CHALLENGE_HEADER,
+                challenge.to_string().parse().expect("UUID must be a valid header value"),
+            );
+            if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
+                cached_peer_boot_epoch(audience),
+                request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
+                request
+                    .headers()
+                    .get(RPC_CONTENT_SHA256_HEADER)
+                    .and_then(|value| value.to_str().ok()),
+            ) {
+                match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
+                    Ok(headers) => request.headers_mut().extend(headers),
+                    Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
+                }
+            }
+        }
+
+        let audience = self.audience.clone();
+        let future = self.inner.call(request);
+        Box::pin(async move {
+            let response = future.await?;
+            if let (Some(audience), Some(challenge)) = (audience, challenge) {
+                match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
+                    Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
+                    Err(error)
+                        if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
+                            || response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
+                    {
+                        debug!(error = %error, "peer boot epoch response proof was rejected")
+                    }
+                    Err(_) => {}
+                }
+            }
+            Ok(response)
+        })
+    }
+}
+
 pub struct TonicSignatureInterceptor {
     audience: Option<String>,
 }
@@ -257,6 +377,13 @@ impl TonicInterceptor {
         }
         Ok(self)
     }
+
+    fn replay_scope_audience(&self) -> Option<String> {
+        match self {
+            Self::Signature(interceptor) => interceptor.audience.clone(),
+            Self::NoOp(_) => None,
+        }
+    }
 }
 
 impl tonic::service::Interceptor for TonicInterceptor {
@@ -279,6 +406,38 @@ mod tests {
     use tracing_opentelemetry::OpenTelemetrySpanExt;
     use tracing_subscriber::{Registry, layer::SubscriberExt};
 
+    #[derive(Clone)]
+    struct EpochProofService {
+        audience: String,
+        seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
+    }
+
+    impl Service<HttpRequest<()>> for EpochProofService {
+        type Response = HttpResponse<()>;
+        type Error = std::convert::Infallible;
+        type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
+
+        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
+            self.seen_headers
+                .lock()
+                .expect("test header capture lock must not be poisoned")
+                .push(request.headers().clone());
+            let challenge = tonic_boot_epoch_challenge(request.headers())
+                .expect("client challenge must be syntactically valid")
+                .expect("authenticated client request must carry a boot epoch challenge");
+            let mut response = HttpResponse::new(());
+            response.headers_mut().extend(
+                tonic_boot_epoch_response_headers(&self.audience, challenge)
+                    .expect("test server must be able to sign an epoch proof"),
+            );
+            std::future::ready(Ok(response))
+        }
+    }
+
     fn ensure_test_rpc_secret() {
         runtime_sources::ensure_test_rpc_secret();
     }
@@ -420,6 +579,52 @@ mod tests {
         assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
     }
 
+    #[test]
+    fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
+        ensure_test_rpc_secret();
+        let audience = "replay-scope-client-test:9000";
+        PEER_BOOT_EPOCHS
+            .lock()
+            .expect("peer epoch cache lock must not be poisoned")
+            .remove(audience);
+        let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
+        let service = EpochProofService {
+            audience: audience.to_string(),
+            seen_headers: seen_headers.clone(),
+        };
+        let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
+        let make_request = || {
+            let mut request = HttpRequest::builder()
+                .uri("/node_service.NodeService/Ping")
+                .body(())
+                .expect("test RPC request must build");
+            request.headers_mut().extend(
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
+                    .expect("v2 test headers must mint"),
+            );
+            request
+        };
+
+        futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
+        futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
+
+        let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
+        assert_eq!(headers.len(), 2);
+        assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
+        assert!(
+            !headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
+            "the first request must remain v2-compatible until the peer proves its epoch"
+        );
+        assert!(
+            headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
+            "the second request must carry the replay-scoped v3 signature"
+        );
+        PEER_BOOT_EPOCHS
+            .lock()
+            .expect("peer epoch cache lock must not be poisoned")
+            .remove(audience);
+    }
+
     #[test]
     fn test_signature_interceptor_requires_generated_method_metadata() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -20,8 +20,8 @@
 //! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
 //! below, plus the broader negative-signature suite. The advisory class is: a
 //! node must never accept an RPC whose auth is missing, malformed, or signed
-//! with the default/empty shared secret. Body-bound v2 requests additionally
-//! receive process-local replay protection. See
+//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
+//! requests additionally receive process-local replay protection. See
 //! `docs/testing/security-regressions.md` for the full advisory -> test map.
 //!
 //! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
@@ -50,13 +50,22 @@ use uuid::Uuid;
 type HmacSha256 = Hmac<Sha256>;
 
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
-const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
-const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
+pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
+pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
 const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
 const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
 pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
-const RPC_AUTH_VERSION_V2: &str = "2";
+pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
+pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
+pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
+pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
+pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
+pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
+pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
+const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
 const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
+const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
+const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
 const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
 const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
 const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
@@ -75,9 +84,15 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
         rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
     )
 });
-// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
-// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
-// cannot disable replay protection by rejecting every body-bound request.
+static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
+    get_env_bool(
+        rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
+        rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
+    )
+});
+// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
+// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
+// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
 static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
     rustfs_utils::get_env_usize(
         rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
@@ -86,6 +101,7 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
     .max(1)
 });
 static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
+static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
 
 #[derive(Default)]
 struct RpcNonceCache {
@@ -313,6 +329,189 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
     mac.verify_slice(&signature).is_ok()
 }
 
+#[derive(Clone, Copy)]
+struct ReplayScope<'a> {
+    audience: &'a str,
+    path: &'a str,
+    timestamp: &'a str,
+    nonce: Uuid,
+    content_sha256: &'a str,
+    boot_epoch: Uuid,
+}
+
+fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
+    mac.update(RPC_REPLAY_SCOPE_DOMAIN);
+    for part in [
+        scope.audience.as_bytes(),
+        b"|",
+        scope.path.as_bytes(),
+        b"|POST|",
+        scope.timestamp.as_bytes(),
+        b"|",
+        scope.nonce.as_bytes(),
+        b"|",
+        scope.content_sha256.as_bytes(),
+        b"|",
+        scope.boot_epoch.as_bytes(),
+    ] {
+        mac.update(part);
+    }
+}
+
+fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_replay_scope(&mut mac, scope);
+    Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
+}
+
+fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
+    let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
+        return false;
+    };
+    let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
+        return false;
+    };
+    update_replay_scope(&mut mac, scope);
+    mac.verify_slice(&signature).is_ok()
+}
+
+fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
+    mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
+    mac.update(audience.as_bytes());
+    mac.update(b"|");
+    mac.update(challenge.as_bytes());
+    mac.update(boot_epoch.as_bytes());
+}
+
+fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
+    if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
+        return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
+    }
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
+    Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
+}
+
+fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
+    if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
+        return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
+    }
+    let proof = general_purpose::STANDARD
+        .decode(proof)
+        .map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
+    mac.verify_slice(&proof)
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
+}
+
+fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
+    let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
+    (!value.is_nil())
+        .then_some(value)
+        .ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
+}
+
+fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
+    path.strip_prefix('/')
+        .and_then(|path| path.split_once('/'))
+        .filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
+        .ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
+}
+
+/// The process-unique epoch included in every replay-scoped server verification.
+///
+/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
+/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
+pub fn tonic_rpc_boot_epoch() -> Uuid {
+    *RPC_BOOT_EPOCH
+}
+
+/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
+/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
+/// old servers can continue validating the same request unchanged.
+pub fn gen_tonic_replay_scope_headers(
+    audience: &str,
+    path: &str,
+    timestamp: &str,
+    content_sha256: &str,
+    boot_epoch: Uuid,
+) -> std::io::Result<HeaderMap> {
+    if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
+        return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
+    }
+    parse_tonic_rpc_path(path)?;
+    timestamp
+        .parse::<i64>()
+        .map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
+
+    let nonce = Uuid::new_v4();
+    let signature = generate_replay_scope_signature(
+        &get_shared_secret()?,
+        ReplayScope {
+            audience,
+            path,
+            timestamp,
+            nonce,
+            content_sha256,
+            boot_epoch,
+        },
+    )?;
+    let mut headers = HeaderMap::new();
+    headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
+    headers.insert(
+        RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+        header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
+    );
+    headers.insert(
+        RPC_REPLAY_SCOPE_NONCE_HEADER,
+        header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
+    );
+    headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
+    Ok(headers)
+}
+
+/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
+pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
+    headers
+        .get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
+        .map(|value| {
+            value
+                .to_str()
+                .map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
+                .and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
+        })
+        .transpose()
+}
+
+/// Build the authenticated response headers for a client boot-epoch challenge.
+pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
+    let boot_epoch = tonic_rpc_boot_epoch();
+    let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
+    let mut headers = HeaderMap::new();
+    headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
+    headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
+    Ok(headers)
+}
+
+/// Verify the server boot-epoch response for a challenge generated by this client.
+pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
+    let boot_epoch = headers
+        .get(RPC_BOOT_EPOCH_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
+        .and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
+    let proof = headers
+        .get(RPC_BOOT_EPOCH_PROOF_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
+    verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
+    Ok(boot_epoch)
+}
+
 fn valid_content_sha256(value: &str) -> bool {
     value == UNSIGNED_PAYLOAD
         || (value.len() == 64
@@ -531,6 +730,17 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
     .any(|name| headers.contains_key(*name))
 }
 
+fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
+    [
+        RPC_REPLAY_SCOPE_VERSION_HEADER,
+        RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+        RPC_REPLAY_SCOPE_NONCE_HEADER,
+        RPC_BOOT_EPOCH_HEADER,
+    ]
+    .iter()
+    .any(|name| headers.contains_key(*name))
+}
+
 /// Whether the server requires target-bound v2 authentication on every internode gRPC request,
 /// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
 /// lever gated on the v1-fallback counter reading zero fleet-wide; see
@@ -540,9 +750,127 @@ fn internode_rpc_signature_strict() -> bool {
     *INTERNODE_RPC_SIGNATURE_STRICT
 }
 
+fn internode_rpc_replay_scope_strict() -> bool {
+    *INTERNODE_RPC_REPLAY_SCOPE_STRICT
+}
+
+fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
+    if audience.is_empty() {
+        return Err(std::io::Error::other("Missing RPC audience"));
+    }
+    parse_tonic_rpc_path(path)?;
+
+    let version = headers
+        .get(RPC_REPLAY_SCOPE_VERSION_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
+    if version != RPC_REPLAY_SCOPE_VERSION_V3 {
+        return Err(std::io::Error::other("Unsupported RPC replay scope version"));
+    }
+    let signature = headers
+        .get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
+    let timestamp = headers
+        .get(TIMESTAMP_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
+    let signed_at = timestamp
+        .parse::<i64>()
+        .map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
+    check_timestamp(signed_at)?;
+    let nonce = headers
+        .get(RPC_REPLAY_SCOPE_NONCE_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
+        .and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
+    let content_sha256 = headers
+        .get(RPC_CONTENT_SHA256_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
+    if !valid_content_sha256(content_sha256) {
+        return Err(std::io::Error::other("Invalid RPC content SHA-256"));
+    }
+    let boot_epoch = headers
+        .get(RPC_BOOT_EPOCH_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
+        .and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
+    let secret = get_shared_secret()?;
+    if !verify_replay_scope_signature(
+        &secret,
+        ReplayScope {
+            audience,
+            path,
+            timestamp,
+            nonce,
+            content_sha256,
+            boot_epoch,
+        },
+        signature,
+    ) {
+        return Err(std::io::Error::other("Invalid RPC replay scope signature"));
+    }
+    if boot_epoch != tonic_rpc_boot_epoch() {
+        return Err(std::io::Error::other("RPC boot epoch is stale"));
+    }
+    check_and_record_nonce(nonce, signed_at)
+}
+
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
 pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
-    verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
+    verify_tonic_rpc_signature_with_policy(
+        audience,
+        path,
+        headers,
+        internode_rpc_signature_strict(),
+        internode_rpc_replay_scope_strict(),
+        false,
+    )
+}
+
+/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
+/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
+pub fn verify_tonic_rpc_signature_with_bootstrap(
+    audience: &str,
+    path: &str,
+    headers: &HeaderMap,
+    allow_replay_scope_bootstrap: bool,
+) -> std::io::Result<()> {
+    verify_tonic_rpc_signature_with_policy(
+        audience,
+        path,
+        headers,
+        internode_rpc_signature_strict(),
+        internode_rpc_replay_scope_strict(),
+        allow_replay_scope_bootstrap,
+    )
+}
+
+fn verify_tonic_rpc_signature_with_policy(
+    audience: &str,
+    path: &str,
+    headers: &HeaderMap,
+    signature_strict: bool,
+    replay_scope_strict: bool,
+    allow_replay_scope_bootstrap: bool,
+) -> std::io::Result<()> {
+    if has_replay_scope_headers(headers) {
+        return verify_tonic_replay_scope_signature(audience, path, headers);
+    }
+
+    // Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
+    // client after its peer restarts. Legacy metadata never gets this exception.
+    let bootstrap = allow_replay_scope_bootstrap
+        && has_v2_auth_headers(headers)
+        && tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
+    if replay_scope_strict && !bootstrap {
+        return Err(std::io::Error::other("RPC replay-scoped authentication required"));
+    }
+
+    verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
+    global_internode_metrics().record_replay_scope_fallback();
+    Ok(())
 }
 
 /// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
@@ -1273,6 +1601,104 @@ mod tests {
         assert_eq!(error.to_string(), "Invalid RPC v2 signature");
     }
 
+    #[test]
+    fn replay_scope_binds_path_epoch_and_random_nonce() {
+        ensure_test_rpc_secret();
+        let path = "/node_service.NodeService/Ping";
+        let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("v2 compatibility headers should build");
+        let timestamp = headers
+            .get(TIMESTAMP_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 timestamp")
+            .to_string();
+        let content_sha256 = headers
+            .get(RPC_CONTENT_SHA256_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 content digest")
+            .to_string();
+        headers.extend(
+            gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, tonic_rpc_boot_epoch())
+                .expect("replay-scope headers should build"),
+        );
+
+        assert!(
+            verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
+            "the first replay-scoped request must be accepted"
+        );
+        let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
+            .expect_err("the random replay-scope nonce must be single-use");
+        assert_eq!(replay.to_string(), "RPC request replay detected");
+
+        let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
+            .expect_err("a replay-scoped signature must not move to another method");
+        assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
+    }
+
+    #[test]
+    fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
+        ensure_test_rpc_secret();
+        let path = "/node_service.NodeService/Ping";
+        let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("v2 compatibility headers should build");
+        partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
+        let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
+            .expect_err("partial replay-scope metadata must never downgrade to v2");
+        assert_eq!(error.to_string(), "Missing RPC replay scope signature");
+
+        let timestamp = partial
+            .get(TIMESTAMP_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 timestamp")
+            .to_string();
+        let content_sha256 = partial
+            .get(RPC_CONTENT_SHA256_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 content digest")
+            .to_string();
+        let stale_epoch = Uuid::new_v4();
+        partial.extend(
+            gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, stale_epoch)
+                .expect("replay-scope headers should build"),
+        );
+        let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
+            .expect_err("a signature from a prior server boot epoch must be rejected");
+        assert_eq!(stale.to_string(), "RPC boot epoch is stale");
+    }
+
+    #[test]
+    fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
+        ensure_test_rpc_secret();
+        let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("v2 compatibility headers should build");
+        let rejected =
+            verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
+                .expect_err("strict replay scope must reject stripped new metadata");
+        assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
+
+        headers.insert(
+            RPC_BOOT_EPOCH_CHALLENGE_HEADER,
+            HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
+        );
+        assert!(
+            verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
+                .is_ok(),
+            "only the signed Ping bootstrap may obtain a new server epoch in strict mode"
+        );
+    }
+
+    #[test]
+    fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
+        ensure_test_rpc_secret();
+        let challenge = Uuid::new_v4();
+        let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
+        let epoch =
+            verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
+        assert_eq!(epoch, tonic_rpc_boot_epoch());
+        assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
+        assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
+    }
+
     #[test]
     fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/mod.rs

@@ -26,13 +26,18 @@ pub(crate) mod runtime_sources;
 pub use background_monitor::shutdown_background_monitors;
 pub(crate) use background_monitor::spawn_background_monitor;
 pub use client::{
-    TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
+    node_service_time_out_client_no_auth,
 };
+// Re-exported through `api::rpc`; not every item is consumed inside this crate.
+#[allow(unused_imports)]
 pub use http_auth::{
-    TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
-    set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
-    verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
-    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+    normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
+    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
+    verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+    verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::cluster::rpc::client::{
-    TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
+    AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
     is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
 };
 use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
@@ -66,7 +66,6 @@ use std::{
 use tokio::{net::TcpStream, time::Duration};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
-use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
@@ -222,6 +221,21 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
         .map_err(|_| Error::other("peer returned an invalid heal control response proof"))
 }
 
+fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
+    let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
+    if topology_member != expected_member {
+        return Err(Error::other(
+            "peer returned a remote version state capability for a different topology member",
+        ));
+    }
+    let server_epoch =
+        Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
+    if server_epoch.is_nil() {
+        return Err(Error::other("peer returned a nil remote version state epoch"));
+    }
+    Ok(server_epoch)
+}
+
 #[derive(Clone, Debug)]
 pub struct PeerLiveEventsBatch {
     pub events: Vec<u8>,
@@ -233,6 +247,7 @@ pub struct PeerLiveEventsBatch {
 pub struct PeerRestClient {
     pub host: XHost,
     pub grid_host: String,
+    topology_member: String,
     offline: Arc<AtomicBool>,
     recovery_running: Arc<AtomicBool>,
 }
@@ -325,9 +340,11 @@ impl PeerRestClient {
     }
 
     pub fn new(host: XHost, grid_host: String) -> Self {
+        let topology_member = host.to_string();
         Self {
             host,
             grid_host,
+            topology_member,
             offline: Arc::new(AtomicBool::new(false)),
             recovery_running: Arc::new(AtomicBool::new(false)),
         }
@@ -347,7 +364,11 @@ impl PeerRestClient {
 
             let client = match grid_host {
                 Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
-                    Ok(host) => Some(PeerRestClient::new(host, grid_host)),
+                    Ok(host) => {
+                        let mut client = PeerRestClient::new(host, grid_host);
+                        client.topology_member = peer_host_port.clone();
+                        Some(client)
+                    }
                     Err(err) => {
                         warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
                         None
@@ -390,7 +411,7 @@ impl PeerRestClient {
         (remote, all, remote_topology_hosts)
     }
 
-    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if self.offline.load(Ordering::Acquire) {
             self.mark_offline_and_spawn_recovery();
             return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -411,7 +432,7 @@ impl PeerRestClient {
         &self,
     ) -> Result<
         rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
-            InterceptedService<Channel, TonicInterceptor>,
+            InterceptedService<AuthenticatedChannel, TonicInterceptor>,
         >,
     > {
         if self.offline.load(Ordering::Acquire) {
@@ -432,7 +453,7 @@ impl PeerRestClient {
 
     async fn get_tier_mutation_control_client(
         &self,
-    ) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    ) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if self.offline.load(Ordering::Acquire) {
             self.mark_offline_and_spawn_recovery();
             return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -1154,6 +1175,15 @@ impl PeerRestClient {
         validate_heal_control_capability_proof(&canonical_ack, &proof)
     }
 
+    pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
+        let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
+        let result = self
+            .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
+            .await?;
+        let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
+        Ok((self.topology_member.clone(), epoch))
+    }
+
     pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
         self.finalize_result(
             async {
@@ -2470,6 +2500,22 @@ mod tests {
         }
     }
 
+    #[test]
+    fn remote_version_state_capability_decoder_fails_closed() {
+        let epoch = Uuid::new_v4();
+        let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
+            .expect("small capability response should encode");
+        assert_eq!(
+            decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
+            epoch
+        );
+        assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
+        assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
+        let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
+            .expect("small capability response should encode");
+        assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
+    }
+
     struct TierMutationResponseFixture<'a> {
         version: u32,
         phase: TierMutationRpcPhase,
@@ -2741,6 +2787,11 @@ mod tests {
                 .with_span_list(true),
         );
         let _guard = tracing::subscriber::set_default(subscriber);
+        // The `recovery-monitor` callsite is shared with the production
+        // `mark_offline_and_spawn_recovery` path that sibling tests exercise from
+        // subscriber-less threads; without this the span can be cached as
+        // `Interest::never()` and silently degrade to `Span::none()`.
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
 
         let client = test_peer_client();
         let span = tracing::info_span!("request-span", request_id = "req-peer-rest");

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -14,7 +14,8 @@
 
 use crate::bucket::metadata_sys;
 use crate::cluster::rpc::client::{
-    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
+    node_service_time_out_client,
 };
 use crate::cluster::rpc::set_tonic_mutation_body_digest;
 use crate::disk::error::DiskError;
@@ -40,16 +41,84 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
 use rustfs_protos::proto_gen::node_service::{
     DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
 };
+#[cfg(test)]
+use std::sync::{
+    Mutex as StdMutex,
+    atomic::{AtomicBool, Ordering},
+};
 use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
+#[cfg(test)]
+use tokio::sync::Notify;
 use tokio::{net::TcpStream, sync::RwLock, time};
 use tokio_util::sync::CancellationToken;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
-use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 
 type Client = Arc<Box<dyn PeerS3Client>>;
 
+#[cfg(test)]
+#[derive(Default)]
+pub(crate) struct DeleteBucketEmptyScanBarrier {
+    arrived: AtomicBool,
+    arrived_notify: Notify,
+    released: AtomicBool,
+    release_notify: Notify,
+}
+
+#[cfg(test)]
+impl DeleteBucketEmptyScanBarrier {
+    pub(crate) async fn wait_until_paused(&self) {
+        loop {
+            let notified = self.arrived_notify.notified();
+            if self.arrived.load(Ordering::Acquire) {
+                return;
+            }
+            notified.await;
+        }
+    }
+
+    pub(crate) fn release(&self) {
+        self.released.store(true, Ordering::Release);
+        self.release_notify.notify_waiters();
+    }
+
+    async fn pause(&self) {
+        self.arrived.store(true, Ordering::Release);
+        self.arrived_notify.notify_waiters();
+        loop {
+            let notified = self.release_notify.notified();
+            if self.released.load(Ordering::Acquire) {
+                return;
+            }
+            notified.await;
+        }
+    }
+}
+
+#[cfg(test)]
+static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
+
+#[cfg(test)]
+pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
+    let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
+    *DELETE_BUCKET_EMPTY_SCAN_BARRIER
+        .lock()
+        .expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
+    barrier
+}
+
+#[cfg(test)]
+async fn pause_after_delete_bucket_empty_scan() {
+    let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
+        .lock()
+        .expect("empty scan barrier lock should not be poisoned")
+        .take();
+    if let Some(barrier) = barrier {
+        barrier.pause().await;
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct ScannerBucketListing {
     pub buckets: Vec<BucketInfo>,
@@ -650,24 +719,22 @@ impl PeerS3Client for LocalPeerS3Client {
             return Err(Error::ErasureWriteQuorum);
         }
 
-        let force = if opts.force_if_empty && !opts.force {
+        if opts.force_if_empty && !opts.force {
             for disk in local_disks.iter() {
                 if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
                     return Err(Error::VolumeNotEmpty);
                 }
             }
-            true
-        } else {
-            opts.force
-        };
+            #[cfg(test)]
+            pause_after_delete_bucket_empty_scan().await;
+        }
 
         let mut futures = Vec::with_capacity(local_disks.len());
 
         for disk in local_disks.iter() {
-            // Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
-            // the recreate loop below turns into BucketNotEmpty; only an explicit
-            // force delete removes recursively (backlog#799 B1).
-            futures.push(disk.delete_volume(bucket, force));
+            // `force_if_empty` is validation-only. Passing it as force would let
+            // a PutObject committed after the scan be removed recursively.
+            futures.push(disk.delete_volume(bucket, opts.force));
         }
 
         let results = join_all(futures).await;
@@ -730,6 +797,15 @@ pub struct RemotePeerS3Client {
 }
 
 impl RemotePeerS3Client {
+    fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
+        let mut remote_opts = opts.clone();
+        // Older peers promote `force_if_empty` to recursive force after their
+        // metadata scan. Keep this coordinator-only hint off the wire so a
+        // mixed-version delete fails closed on non-empty directory remnants.
+        remote_opts.force_if_empty = false;
+        serde_json::to_string(&remote_opts).map_err(Into::into)
+    }
+
     fn recovery_monitor_span(addr: &str) -> tracing::Span {
         tracing::info_span!(
             "recovery-monitor",
@@ -756,7 +832,7 @@ impl RemotePeerS3Client {
         client
     }
 
-    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
             .await
             .map_err(|err| Error::other(format!("can not get client, err: {err}")))
@@ -1040,7 +1116,7 @@ impl PeerS3Client for RemotePeerS3Client {
     async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
         self.execute_with_timeout(
             || async {
-                let options = serde_json::to_string(opts)?;
+                let options = Self::encode_delete_bucket_options(opts)?;
                 let mut client = self.get_client().await?;
 
                 let mut request = Request::new(DeleteBucketRequest {
@@ -1414,6 +1490,49 @@ mod tests {
         }
     }
 
+    #[test]
+    fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
+        let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
+            no_lock: true,
+            no_recreate: true,
+            force_if_empty: true,
+            ..Default::default()
+        })
+        .expect("remote delete options should serialize");
+        let legacy_opts: DeleteBucketOptions =
+            serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
+
+        assert!(legacy_opts.no_lock);
+        assert!(legacy_opts.no_recreate);
+        assert!(!legacy_opts.force);
+        assert!(!legacy_opts.force_if_empty);
+
+        let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
+            true
+        } else {
+            legacy_opts.force
+        };
+        assert!(
+            !legacy_recursive_force,
+            "legacy peer must not upgrade empty-only delete to recursive force"
+        );
+    }
+
+    #[test]
+    fn remote_delete_bucket_options_preserve_explicit_force() {
+        let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
+            force: true,
+            force_if_empty: true,
+            ..Default::default()
+        })
+        .expect("remote force-delete options should serialize");
+        let remote_opts: DeleteBucketOptions =
+            serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
+
+        assert!(remote_opts.force);
+        assert!(!remote_opts.force_if_empty);
+    }
+
     #[tokio::test]
     async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
         let client = test_remote_peer("http://peer-network-error:9000");
@@ -1571,6 +1690,54 @@ mod tests {
         reset_local_disk_test_state().await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
+        let disks = init_test_local_disks_for_pools(
+            &temp_dir,
+            &[(0, 1), (1, 1)],
+            "local-peer-force-if-empty-preserves-unclassified-file",
+        )
+        .await;
+        let bucket = "empty-only-delete-bucket";
+        let marker = "object/commit-marker";
+        let data = bytes::Bytes::from_static(b"committed object data");
+
+        disks[1]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should be created in the selected pool");
+        disks[1]
+            .write_all(bucket, marker, data.clone())
+            .await
+            .expect("unclassified committed file should be written");
+
+        let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
+            .delete_bucket(
+                bucket,
+                &DeleteBucketOptions {
+                    force_if_empty: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("empty-only delete must not recursively remove an unclassified file");
+
+        assert_eq!(err, Error::VolumeNotEmpty);
+        assert_eq!(
+            disks[1]
+                .read_all(bucket, marker)
+                .await
+                .expect("unclassified committed file should be preserved"),
+            data
+        );
+
+        reset_local_disk_test_state().await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn heal_bucket_local_recreates_missing_bucket_volumes() {

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use crate::cluster::rpc::client::{
-    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
-    node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
+    node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
 };
 use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
 use crate::cluster::rpc::internode_data_transport::{
@@ -71,7 +71,7 @@ use tokio::{
     time::timeout,
 };
 use tokio_util::sync::CancellationToken;
-use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
+use tonic::{Code, Request, service::interceptor::InterceptedService};
 use tracing::{debug, trace, warn};
 use uuid::Uuid;
 
@@ -1083,7 +1083,7 @@ impl RemoteDisk {
         internode_offline_bypass_reason(&self.addr).map(Error::other)
     }
 
-    async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if let Some(err) = self.offline_bypass_error() {
             return Err(err);
         }
@@ -1096,7 +1096,7 @@ impl RemoteDisk {
     /// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
     /// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
     /// is disabled.
-    async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if let Some(err) = self.offline_bypass_error() {
             return Err(err);
         }
@@ -3005,6 +3005,7 @@ mod tests {
     use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
     use crate::runtime::sources as runtime_sources;
     use serde_json::Value;
+    use serial_test::serial;
     use std::io::{self as std_io, Write};
     use std::pin::Pin;
     use std::sync::{Arc, Mutex, Mutex as StdMutex, Once};
@@ -3018,6 +3019,20 @@ mod tests {
 
     static INIT: Once = Once::new();
 
+    // `#[serial(internode_metrics)]` marks every test that observes
+    // `global_internode_metrics()`. Those counters are a process-wide singleton:
+    // some of these tests snapshot a counter, run one decode, and assert on the
+    // delta, while others deliberately record decode errors or call
+    // `reset_internode_metrics_for_test()`. Run concurrently in one process they
+    // corrupt each other's deltas — a sibling's error bumps the "no decode error"
+    // assertion off zero, and a sibling's reset can drive an `after > before`
+    // assertion backwards.
+    //
+    // The marker only takes effect under the `cargo test` fallback; nextest
+    // already isolates each test in its own process, so every test there gets its
+    // own copy of the counters (see `docs/testing/README.md`). Any new test that
+    // reads or mutates the global internode metrics belongs in this group.
+
     #[test]
     fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
         let token = SnapshotLeaseToken::new();
@@ -3306,6 +3321,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn read_multiple_response_decode_prefers_msgpack_payloads() {
         let endpoint = sample_remote_endpoint();
         let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
@@ -3328,6 +3344,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn read_multiple_response_decode_falls_back_to_json_payloads() {
         let endpoint = sample_remote_endpoint();
         let json_resp = sample_read_multiple_resp("json", b"fallback");
@@ -3349,6 +3366,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn rename_data_response_accepts_legacy_json_without_decode_error() {
         crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
         let response = RenameDataResp {
@@ -3500,6 +3518,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
         let endpoint = sample_remote_endpoint();
         let response = ReadMultipleResponse {
@@ -3525,6 +3544,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn read_multiple_response_decode_reports_corrupt_json_item() {
         let endpoint = sample_remote_endpoint();
         let response = ReadMultipleResponse {
@@ -3565,6 +3585,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn batch_read_version_response_decode_prefers_msgpack_payloads() {
         let endpoint = sample_remote_endpoint();
         let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
@@ -3585,6 +3606,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn batch_read_version_response_rejects_invalid_success_metadata() {
         let endpoint = sample_remote_endpoint();
         let mut response_item = sample_batch_read_version_resp(0, "invalid-object", true);
@@ -3604,6 +3626,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
         let endpoint = sample_remote_endpoint();
         let response = BatchReadVersionResponse {
@@ -3630,6 +3653,7 @@ mod tests {
     }
 
     #[test]
+    #[serial(internode_metrics)]
     fn batch_read_version_response_decode_reports_corrupt_json_item() {
         let endpoint = sample_remote_endpoint();
         let response = BatchReadVersionResponse {
@@ -4419,6 +4443,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial(internode_metrics)]
     async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
         let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
             OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
@@ -4457,6 +4482,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[serial(internode_metrics)]
     async fn test_remote_disk_read_file_stream_retries_once_on_retryable_open_read_error() {
         // A transient reset-by-peer on a shard read during the read-after-write window must be
         // absorbed by one re-dial rather than eroding read quorum (issue #2761).
@@ -5306,6 +5332,11 @@ mod tests {
                 .with_span_list(true),
         );
         let _guard = tracing::subscriber::set_default(subscriber);
+        // The `recovery-monitor` span and the monitor's own log events are
+        // production callsites that sibling tests exercise from subscriber-less
+        // threads; without this they can be cached as `Interest::never()` and go
+        // silently missing here.
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
 
         let endpoint = Endpoint {
             url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
@@ -5360,6 +5391,11 @@ mod tests {
                 .with_span_list(true),
         );
         let _guard = tracing::subscriber::set_default(subscriber);
+        // The `recovery-monitor` span and the monitor's own log events are
+        // production callsites that sibling tests exercise from subscriber-less
+        // threads; without this they can be cached as `Interest::never()` and go
+        // silently missing here.
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
 
         let addr = "http://127.0.0.1:59997".to_string();
         let endpoint = Endpoint {

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::cluster::rpc::client::{
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
+};
 use crate::cluster::rpc::set_tonic_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
@@ -29,7 +31,6 @@ use std::time::Duration;
 use tokio::time::timeout;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
-use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 
 /// Remote lock client implementation
@@ -78,7 +79,7 @@ impl RemoteClient {
         }
     }
 
-    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         // P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
         // offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
         // change quorum; the self-healing re-probe keeps the peer recoverable.

crates/ecstore/src/disk/os.rs

@@ -25,7 +25,7 @@ use std::{
     sync::{Arc, LazyLock, Weak},
 };
 use tokio::fs;
-use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
+use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
 use tracing::warn;
 
 /// Check path length according to OS limits.
@@ -123,6 +123,8 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
 
 fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
     let cpu_scaled = cpu_count
@@ -157,6 +159,24 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
     limiter
 }
 
+/// Serialize a bucket's local metadata commits with physical bucket removal.
+///
+/// The key includes the canonical disk root, so independently reconnected
+/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
+/// disconnected disks do not keep the registry alive.
+pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
+    let key = root.join(volume);
+    let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
+    locks.retain(|_, lock| lock.strong_count() > 0);
+    if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
+        return lock;
+    }
+
+    let lock = Arc::new(RwLock::new(()));
+    locks.insert(key, Arc::downgrade(&lock));
+    lock
+}
+
 /// Always acquire the per-disk permit before the process-wide permit. Keeping
 /// this order uniform prevents one slow disk from reserving global capacity
 /// while it waits for its own concurrency slot.
@@ -572,15 +592,14 @@ async fn reliable_rename_inner(
     base_dir: impl AsRef<Path>,
     warn_on_missing_source: bool,
 ) -> io::Result<()> {
-    if let Some(parent) = dst_file_path.as_ref().parent()
-        && !file_exists(parent)
-    {
-        reliable_mkdir_all(parent, base_dir.as_ref()).await?;
-    }
+    let parent_guard = match dst_file_path.as_ref().parent() {
+        Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
+        None => None,
+    };
 
     let mut i = 0;
     loop {
-        if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
+        if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
             if should_retry_rename(&e, i) {
                 i += 1;
                 continue;
@@ -597,6 +616,159 @@ async fn reliable_rename_inner(
     Ok(())
 }
 
+#[cfg(unix)]
+fn rename_into_existing_parent(
+    src_file_path: &Path,
+    dst_file_path: &Path,
+    parent_guard: Option<&ExistingBaseDirectoryGuard>,
+) -> io::Result<()> {
+    use rustix::fs::{Mode, OFlags, open, renameat};
+
+    let Some(parent_guard) = parent_guard else {
+        return super::fs::rename_std(src_file_path, dst_file_path);
+    };
+    let src_parent = src_file_path
+        .parent()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
+    let src_name = src_file_path
+        .file_name()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
+    let dst_name = dst_file_path
+        .file_name()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
+    let src_parent = open(
+        src_parent,
+        OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+        Mode::empty(),
+    )
+    .map_err(io::Error::from)?;
+    let dst_parent = parent_guard
+        .last()
+        .ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
+
+    renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
+}
+
+#[cfg(not(unix))]
+fn rename_into_existing_parent(
+    src_file_path: &Path,
+    dst_file_path: &Path,
+    _parent_guard: Option<&ExistingBaseDirectoryGuard>,
+) -> io::Result<()> {
+    super::fs::rename_std(src_file_path, dst_file_path)
+}
+
+async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
+    let dir_path = dir_path.to_path_buf();
+    let base_dir = base_dir.to_path_buf();
+
+    tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
+}
+
+#[cfg(windows)]
+pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
+
+#[cfg(unix)]
+pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
+
+#[cfg(all(not(unix), not(windows)))]
+pub(crate) type ExistingBaseDirectoryGuard = ();
+
+#[cfg(windows)]
+fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
+    use std::os::windows::fs::OpenOptionsExt;
+
+    const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
+    const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
+    const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
+    const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
+    const FILE_SHARE_READ: u32 = 0x1;
+
+    let file = std::fs::OpenOptions::new()
+        .read(true)
+        .share_mode(FILE_SHARE_READ)
+        .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
+        .open(path)?;
+    let handle = winapi_util::Handle::from_file(file);
+    let info = winapi_util::file::information(&handle)?;
+    if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
+        || info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
+    {
+        return Err(io::Error::from(io::ErrorKind::NotADirectory));
+    }
+    Ok(handle)
+}
+
+pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
+    let relative = dir_path
+        .strip_prefix(base_dir)
+        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
+    for component in relative.components() {
+        if !matches!(component, Component::Normal(_) | Component::CurDir) {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "rename destination contains an invalid path component",
+            ));
+        }
+    }
+
+    #[cfg(unix)]
+    {
+        use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
+        use rustix::io::Errno;
+
+        let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
+        let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
+        let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
+
+        for component in relative.components() {
+            let Component::Normal(component) = component else {
+                continue;
+            };
+            let parent = parents
+                .last()
+                .expect("base directory guard should contain the base directory");
+            match mkdirat(parent, component, mode) {
+                Ok(()) => {}
+                Err(Errno::EXIST) => {}
+                Err(err) => return Err(err.into()),
+            }
+            parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
+        }
+
+        Ok(parents)
+    }
+
+    #[cfg(windows)]
+    {
+        let mut handles = vec![lock_windows_directory(base_dir)?];
+        let mut current = base_dir.to_path_buf();
+        for component in relative.components() {
+            let Component::Normal(component) = component else {
+                continue;
+            };
+            current.push(component);
+            match std::fs::create_dir(&current) {
+                Ok(()) => {}
+                Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
+                Err(err) => return Err(err),
+            }
+            handles.push(lock_windows_directory(&current)?);
+        }
+
+        Ok(handles)
+    }
+
+    #[cfg(all(not(unix), not(windows)))]
+    {
+        let _ = relative;
+        Err(io::Error::new(
+            io::ErrorKind::Unsupported,
+            "safe recursive directory creation is unavailable on this platform",
+        ))
+    }
+}
+
 fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
     warn!(
         "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
@@ -727,6 +899,20 @@ mod tests {
         Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
     }
 
+    #[tokio::test]
+    async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
+        let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
+        let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
+
+        assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
+        assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
+
+        let _write_guard = first.write().await;
+        assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
+    }
+
     #[derive(Clone, Default)]
     struct CapturedLogs {
         buffer: Arc<Mutex<Vec<u8>>>,
@@ -771,9 +957,26 @@ mod tests {
         }
     }
 
+    /// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
+    /// the pin that keeps tracing's process-global callsite-interest cache from
+    /// being decided by some other test's thread.
+    struct WarnCaptureGuard {
+        _subscriber: tracing::subscriber::DefaultGuard,
+        _callsite_pin: tracing::Dispatch,
+    }
+
     /// Capture WARN-level output on the current thread; tokio tests here run on
     /// the current-thread runtime, so the guard covers the whole test body.
-    fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
+    ///
+    /// The callsite pin matters because `warn_reliable_rename_failure` is a
+    /// single production callsite shared with tests that call `rename_all`
+    /// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
+    /// is one. Whichever thread reaches it first fixes its `Interest`
+    /// process-wide, so without the pin that sibling can cache
+    /// `Interest::never()` and the WARN never fires here at all, leaving the
+    /// "must keep the WARN" assertions staring at empty output. See
+    /// [`crate::test_tracing::pin_callsite_interest_for_test`].
+    fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
         let logs = CapturedLogs::default();
         let subscriber = tracing_subscriber::fmt()
             .with_max_level(tracing::Level::WARN)
@@ -781,7 +984,10 @@ mod tests {
             .with_ansi(false)
             .without_time()
             .finish();
-        let guard = tracing::subscriber::set_default(subscriber);
+        let guard = WarnCaptureGuard {
+            _subscriber: tracing::subscriber::set_default(subscriber),
+            _callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
+        };
         (logs, guard)
     }
 
@@ -968,6 +1174,112 @@ mod tests {
         assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
     }
 
+    #[tokio::test]
+    async fn rename_all_does_not_recreate_missing_base_directory() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        std::fs::create_dir(&base).expect("create destination base");
+        let src = temp_dir.path().join("staged-object");
+        std::fs::write(&src, b"payload").expect("write staged object");
+        let dst = base.join("object").join("xl.meta");
+        std::fs::remove_dir(&base).expect("delete destination base before commit");
+
+        let err = rename_all(&src, &dst, &base)
+            .await
+            .expect_err("rename must not recreate a deleted destination base");
+
+        assert!(matches!(err, DiskError::FileNotFound));
+        assert!(src.exists(), "failed commit must preserve the staged source");
+        assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir(&base).expect("create destination base");
+        std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
+        let src = temp_dir.path().join("staged-object");
+        std::fs::write(&src, b"payload").expect("write staged object");
+        let dst = base.join("object").join("xl.meta");
+
+        std::fs::remove_dir(&base).expect("remove destination base before replacement");
+        symlink(&outside, &base).expect("replace destination base with a symlink");
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect_err("rename must reject an existing destination parent below a replaced base");
+
+        assert!(src.exists(), "rejected rename must preserve the staged source");
+        assert!(
+            !outside.join("object/xl.meta").exists(),
+            "rename must not publish through the replacement symlink"
+        );
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        std::fs::create_dir(&base).expect("create destination base");
+        let parent = base.join("object").join("nested");
+        let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
+
+        std::fs::rename(&base, temp_dir.path().join("replacement-base"))
+            .expect_err("the locked base must not be replaceable before commit");
+        std::fs::rename(base.join("object"), base.join("replacement-object"))
+            .expect_err("a locked intermediate directory must not be replaceable before commit");
+
+        drop(guard);
+        std::fs::rename(base.join("object"), base.join("replacement-object"))
+            .expect("replacement should succeed after the commit guard is released");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_parent_creation_rejects_symlinked_base() {
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir(&outside).expect("create outside directory");
+        let base = temp_dir.path().join("bucket");
+        symlink(&outside, &base).expect("create symlinked base");
+
+        mkdir_all_below_existing_base(&base.join("object"), &base)
+            .await
+            .expect_err("symlinked base must be rejected");
+
+        assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_parent_creation_rejects_symlink_below_base() {
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir(&base).expect("create destination base");
+        std::fs::create_dir(&outside).expect("create outside directory");
+        symlink(&outside, base.join("linked")).expect("create symlink below base");
+
+        mkdir_all_below_existing_base(&base.join("linked/object"), &base)
+            .await
+            .expect_err("symlink below base must be rejected");
+
+        assert!(
+            !outside.join("object").exists(),
+            "parent creation must not follow a symlink outside the base"
+        );
+    }
+
     #[tokio::test]
     async fn fsync_dir_succeeds_on_directory() {
         let temp_dir = tempdir().expect("create temp dir");

crates/ecstore/src/lib.rs

@@ -93,3 +93,6 @@ pub(crate) mod ecstore_validation_blackbox;
 
 #[cfg(test)]
 pub(crate) mod test_metrics;
+
+#[cfg(test)]
+pub(crate) mod test_tracing;

crates/ecstore/src/runtime/sources.rs

@@ -202,10 +202,6 @@ pub(crate) async fn ensure_boot_time() {
     GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
 }
 
-pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
-    rustfs_common::get_global_init_time().await
-}
-
 pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
     if is_erasure_sd().await {
         None

crates/ecstore/src/services/metrics_realtime.rs

@@ -71,6 +71,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
     MadminScannerMetrics {
         collected_at: metrics.collected_at,
         current_cycle: metrics.current_cycle,
+        current_cycle_active: Some(metrics.current_cycle_active),
         current_started: metrics.current_started,
         cycles_completed_at: metrics.cycles_completed_at,
         ongoing_buckets: metrics.ongoing_buckets,
@@ -398,10 +399,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
 
     if types.contains(&MetricType::SCANNER) {
         debug!("start get scanner metrics");
-        let mut metrics = global_metrics().report().await;
-        if let Some(init_time) = runtime_sources::scanner_init_time().await {
-            metrics.current_started = init_time;
-        }
+        let metrics = global_metrics().report().await;
         real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
     }
 
@@ -540,7 +538,9 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
 #[cfg(test)]
 mod test {
     use super::*;
+    use rustfs_common::metrics::CurrentCycle;
     use rustfs_io_metrics::internode_metrics::global_internode_metrics;
+    use serial_test::serial;
     use std::time::Duration;
 
     #[test]
@@ -588,7 +588,10 @@ mod test {
 
     #[test]
     fn scanner_metrics_mapping_preserves_partial_source_status() {
+        let current_started = Utc::now() - chrono::Duration::seconds(5);
         let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
+            current_cycle_active: true,
+            current_started,
             last_cycle_partial_source: "usage".to_string(),
             last_cycle_partial_source_code: 1,
             partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
@@ -598,6 +601,8 @@ mod test {
             ..Default::default()
         });
 
+        assert_eq!(scanner.current_cycle_active, Some(true));
+        assert_eq!(scanner.current_started, current_started);
         assert_eq!(scanner.last_cycle_partial_source, "usage");
         assert_eq!(scanner.last_cycle_partial_source_code, 1);
         let usage = scanner
@@ -608,6 +613,39 @@ mod test {
         assert_eq!(usage.cycles, 2);
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
+        let previous_init_time = *rustfs_common::globals::GLOBAL_INIT_TIME.read().await;
+        let previous_cycle = global_metrics().get_cycle().await;
+        let init_time = Utc::now() - chrono::Duration::hours(1);
+        let cycle_started = Utc::now() - chrono::Duration::seconds(5);
+        *rustfs_common::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
+        let cycle = CurrentCycle {
+            current: 0,
+            next: 1,
+            started: cycle_started,
+            ..Default::default()
+        };
+        let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;
+
+        let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;
+
+        global_metrics()
+            .finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
+            .await;
+        global_metrics().set_cycle(previous_cycle).await;
+        *rustfs_common::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
+
+        let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
+        let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
+        let mut aggregated = RealtimeMetrics::default();
+        aggregated.merge(decoded);
+        let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
+        assert_eq!(scanner.current_cycle_active, Some(true));
+        assert_eq!(scanner.current_started, cycle_started);
+    }
+
     #[test]
     fn scanner_metrics_mapping_preserves_pacing_pressure() {
         let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {

crates/ecstore/src/services/notification_sys.rs

@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use rustfs_utils::XHost;
-use std::collections::{HashMap, hash_map::DefaultHasher};
+use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
 use std::future::Future;
 use std::hash::{Hash, Hasher};
 use std::sync::{Arc, Mutex, OnceLock};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
@@ -47,6 +47,9 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
 const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
 const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
+const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
+const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
+const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 
 /// Cached result from the last successful admin call to a peer.
 struct PeerAdminCache {
@@ -91,6 +94,180 @@ lazy_static! {
     pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
 }
 
+#[derive(Clone)]
+struct RemoteVersionStateFleetProof {
+    topology_fingerprint: String,
+    peer_epochs: Arc<BTreeMap<String, Uuid>>,
+    expires_at: Instant,
+}
+
+impl RemoteVersionStateFleetProof {
+    fn token(&self) -> RemoteVersionStateFleetProofToken {
+        RemoteVersionStateFleetProofToken {
+            topology_fingerprint: self.topology_fingerprint.clone(),
+            peer_epochs: self.peer_epochs.clone(),
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub(crate) struct RemoteVersionStateFleetProofToken {
+    topology_fingerprint: String,
+    peer_epochs: Arc<BTreeMap<String, Uuid>>,
+}
+
+#[derive(Default)]
+struct RemoteVersionStateFleetProofState {
+    proof: Option<RemoteVersionStateFleetProof>,
+    topology_conflict: bool,
+}
+
+static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
+static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
+
+fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
+    REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
+}
+
+fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
+    replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
+}
+
+fn replace_remote_version_state_fleet_proof_in(
+    slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
+    proof: Option<RemoteVersionStateFleetProof>,
+) {
+    slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
+}
+
+fn publish_remote_version_state_probe_result(
+    slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
+    topology_fingerprint: &str,
+    result: Result<BTreeMap<String, Uuid>>,
+    observed_at: Instant,
+) -> Option<Error> {
+    match result {
+        Ok(peer_epochs) => {
+            let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+            let peer_epochs = state
+                .proof
+                .as_ref()
+                .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
+                .map(|proof| Arc::clone(&proof.peer_epochs))
+                .unwrap_or_else(|| Arc::new(peer_epochs));
+            state.proof = Some(RemoteVersionStateFleetProof {
+                topology_fingerprint: topology_fingerprint.to_string(),
+                peer_epochs,
+                expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
+            });
+            None
+        }
+        Err(err) => {
+            replace_remote_version_state_fleet_proof_in(slot, None);
+            Some(err)
+        }
+    }
+}
+
+pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = remote_version_state_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
+}
+
+fn acquire_remote_version_state_fleet_proof_from(
+    state: &RemoteVersionStateFleetProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<RemoteVersionStateFleetProofToken> {
+    if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
+        return None;
+    }
+    state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
+}
+
+pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    let state = remote_version_state_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    if state.topology_conflict {
+        return false;
+    }
+    state.proof.as_ref().is_some_and(|current| {
+        current.topology_fingerprint == *expected_topology
+            && current.topology_fingerprint == proof.topology_fingerprint
+            && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
+            && Instant::now() < current.expires_at
+    })
+}
+
+fn remote_version_state_fleet_proof_valid_at(
+    proof: Option<&RemoteVersionStateFleetProof>,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
+}
+
+fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
+    if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
+        return Err(Error::other("remote version state capability peer identity is invalid"));
+    }
+    Ok(())
+}
+
+pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
+    if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
+        if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
+            let mut state = remote_version_state_fleet_proof_slot()
+                .write()
+                .unwrap_or_else(std::sync::PoisonError::into_inner);
+            state.topology_conflict = true;
+            state.proof = None;
+        }
+        return;
+    }
+
+    tokio::spawn(async move {
+        loop {
+            let result = match get_global_notification_sys() {
+                Some(notification_sys) => {
+                    match timeout(
+                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                        notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
+                    )
+                    .await
+                    {
+                        Ok(result) => result,
+                        Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
+                    }
+                }
+                None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
+            };
+            let topology_conflict = remote_version_state_fleet_proof_slot()
+                .read()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .topology_conflict;
+            if topology_conflict {
+                replace_remote_version_state_fleet_proof(None);
+            } else if let Some(err) = publish_remote_version_state_probe_result(
+                remote_version_state_fleet_proof_slot(),
+                &topology_fingerprint,
+                result,
+                Instant::now(),
+            ) {
+                debug!(error = %err, "remote version state fleet capability probe failed closed");
+            }
+            sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
+        }
+    });
+}
+
 pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
     let _ = GLOBAL_NOTIFICATION_SYS
         .set(Arc::new(NotificationSys::new(eps).await))
@@ -115,7 +292,17 @@ pub struct NotificationSys {
 
 impl NotificationSys {
     pub async fn new(eps: EndpointServerPools) -> Self {
+        let expected_remote_hosts = eps
+            .peer_grid_host_slots_sorted()
+            .into_iter()
+            .filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
+            .collect::<Vec<_>>();
         let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
+        let peer_topology_hosts = if peer_topology_hosts.is_empty() {
+            expected_remote_hosts
+        } else {
+            peer_topology_hosts
+        };
         let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
         Self {
             peer_clients,
@@ -125,6 +312,24 @@ impl NotificationSys {
             tier_config_reload_workers: Default::default(),
         }
     }
+
+    async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
+        if self.peer_clients.len() != self.peer_topology_hosts.len() {
+            return Err(Error::other("remote version state capability fleet membership is incomplete"));
+        }
+        let probes = self.peer_clients.iter().map(|client| async {
+            let client = client
+                .as_ref()
+                .ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
+            client.probe_remote_version_state(topology_fingerprint.to_string()).await
+        });
+        let mut peer_epochs = BTreeMap::new();
+        for result in join_all(probes).await {
+            let (peer, epoch) = result?;
+            insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
+        }
+        Ok(peer_epochs)
+    }
 }
 
 pub struct NotificationPeerErr {
@@ -1890,6 +2095,183 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 mod tests {
     use super::*;
 
+    #[test]
+    fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
+        let now = Instant::now();
+        let mut peer_epochs = BTreeMap::new();
+        peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(peer_epochs),
+            expires_at: now + Duration::from_secs(1),
+        };
+
+        assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
+        assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
+        assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
+        assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
+        let mut peer_epochs = BTreeMap::new();
+
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
+        assert!(peer_epochs.is_empty());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_accepts_single_node_membership() {
+        let now = Instant::now();
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(BTreeMap::new()),
+            expires_at: now + Duration::from_secs(1),
+        };
+
+        assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
+        let now = Instant::now();
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            expires_at: now + Duration::from_secs(1),
+        };
+        let captured = proof.token();
+        let restarted = RemoteVersionStateFleetProof {
+            topology_fingerprint: proof.topology_fingerprint.clone(),
+            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            expires_at: proof.expires_at,
+        };
+
+        assert!(captured != restarted.token());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
+        let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
+        let now = Instant::now();
+        let epoch = Uuid::new_v4();
+        let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
+        let original = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("successful probe should publish proof")
+            .token();
+
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
+        );
+        let renewed = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("renewal should retain proof")
+            .token();
+        assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
+
+        let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
+                .is_none()
+        );
+        let replaced = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("restarted peer should publish a new proof")
+            .token();
+        assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
+        let now = Instant::now();
+        let mut state = RemoteVersionStateFleetProofState {
+            proof: Some(RemoteVersionStateFleetProof {
+                topology_fingerprint: "topology-a".to_string(),
+                peer_epochs: Arc::new(BTreeMap::new()),
+                expires_at: now + Duration::from_secs(1),
+            }),
+            topology_conflict: false,
+        };
+        assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
+
+        state.topology_conflict = true;
+        assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
+        let epoch = Uuid::new_v4();
+        let mut peer_epochs = BTreeMap::new();
+        insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
+            .expect("first member should be admitted");
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
+        let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
+        let now = Instant::now();
+        let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
+        assert!(slot.read().expect("proof slot should not poison").proof.is_some());
+
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
+        );
+        assert!(slot.read().expect("proof slot should not poison").proof.is_none());
+
+        let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
+        assert!(slot.read().expect("proof slot should not poison").proof.is_some());
+    }
+
+    #[tokio::test]
+    async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
+        let notification_sys = NotificationSys {
+            peer_clients: vec![None],
+            all_peer_clients: vec![None, None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+            tier_config_reload_workers: Default::default(),
+        };
+
+        let err = notification_sys
+            .probe_remote_version_state_fleet("topology-a")
+            .await
+            .expect_err("an unreachable configured member must fail the fleet proof");
+        assert!(err.to_string().contains("unreachable"));
+    }
+
+    #[tokio::test]
+    async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
+        let notification_sys = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: Vec::new(),
+            tier_config_reload_workers: Default::default(),
+        };
+
+        let err = notification_sys
+            .probe_remote_version_state_fleet("topology-a")
+            .await
+            .expect_err("a missing configured member slot must fail the fleet proof");
+        assert!(err.to_string().contains("incomplete"));
+    }
+
     fn build_props(endpoint: &str) -> ServerProperties {
         ServerProperties {
             endpoint: endpoint.to_string(),

crates/ecstore/src/services/tier/tier.rs

@@ -236,6 +236,7 @@ struct TierPublishTransition {
 
 struct PreparedTierDriver {
     tier_name: String,
+    tier_config: TierConfig,
     config_fingerprint: TierDriverFingerprint,
     backend_identity: TierDestinationId,
     exact_get_delete: bool,
@@ -1342,6 +1343,7 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
 
 struct TierDriverGeneration {
     tier_name: Arc<str>,
+    tier_config: TierConfig,
     generation: DriverRevision,
     // Process-local only: this may reflect credential changes and must never be persisted or logged.
     config_fingerprint: TierDriverFingerprint,
@@ -1349,6 +1351,9 @@ struct TierDriverGeneration {
     backend_identity: TierDestinationId,
     exact_get_delete: bool,
     driver: SharedWarmBackend,
+    reconciler: tokio::sync::OnceCell<
+        Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
+    >,
     accepting: AtomicBool,
     active_leases: AtomicUsize,
     drained: tokio::sync::Notify,
@@ -1473,6 +1478,35 @@ impl TierOperationLease {
         self.inner.backend_identity
     }
 
+    pub(crate) async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        transaction_id: uuid::Uuid,
+    ) -> io::Result<TransitionCandidateProbe> {
+        let Some(reconciler) = self
+            .inner
+            .reconciler
+            .get_or_try_init(|| async {
+                crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
+                    .await
+                    .map(|reconciler| reconciler.map(Arc::from))
+            })
+            .await
+            .map_err(|err| io::Error::other(err.message))?
+        else {
+            return self.inner.driver.probe_transition_candidate(object).await;
+        };
+        reconciler
+            .probe_transition_candidate_for(
+                object,
+                crate::services::tier::warm_backend::TransitionCandidateIdentity {
+                    transaction_id,
+                    destination_id: self.backend_identity(),
+                },
+            )
+            .await
+    }
+
     pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
         self.inner.driver.validate_remote_version_id(remote_version_id)?;
         if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
@@ -2833,6 +2867,7 @@ impl TierConfigMgr {
                     let exact_get_delete = tier_exact_get_delete(config);
                     Some(PreparedTierDriver {
                         tier_name: tier_name.to_string(),
+                        tier_config: config.clone(),
                         config_fingerprint,
                         backend_identity,
                         exact_get_delete,
@@ -2861,11 +2896,13 @@ impl TierConfigMgr {
             })?;
             let entry = Arc::new(TierDriverGeneration {
                 tier_name: Arc::from(prepared.tier_name.as_str()),
+                tier_config: prepared.tier_config.clone(),
                 generation,
                 config_fingerprint: prepared.config_fingerprint,
                 backend_identity: prepared.backend_identity,
                 exact_get_delete: prepared.exact_get_delete,
                 driver: prepared.driver.clone(),
+                reconciler: tokio::sync::OnceCell::new(),
                 accepting: AtomicBool::new(true),
                 active_leases: AtomicUsize::new(0),
                 drained: tokio::sync::Notify::new(),
@@ -3609,11 +3646,13 @@ impl TierConfigMgr {
         let driver: SharedWarmBackend = Arc::from(driver);
         let entry = Arc::new(TierDriverGeneration {
             tier_name: Arc::from(tier_name),
+            tier_config: config.clone(),
             generation,
             config_fingerprint,
             backend_identity,
             exact_get_delete,
             driver: driver.clone(),
+            reconciler: tokio::sync::OnceCell::new(),
             accepting: AtomicBool::new(true),
             active_leases: AtomicUsize::new(0),
             drained: tokio::sync::Notify::new(),

crates/ecstore/src/services/tier/warm_backend.rs

@@ -73,6 +73,21 @@ pub enum TransitionCandidateProbe {
     Unsupported,
 }
 
+#[derive(Clone, Copy)]
+pub(crate) struct TransitionCandidateIdentity {
+    pub transaction_id: uuid::Uuid,
+    pub destination_id: [u8; 32],
+}
+
+#[async_trait::async_trait]
+pub(crate) trait TransitionCandidateReconciler {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error>;
+}
+
 #[async_trait::async_trait]
 pub trait WarmBackend {
     async fn validate(&self) -> Result<(), std::io::Error> {
@@ -189,6 +204,20 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
         metadata.remove(key);
     }
 
+    for suffix in [
+        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+    ] {
+        for key in [
+            rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
+            format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
+        ] {
+            if let Some(value) = metadata.remove(&key) {
+                metadata.insert(format!("x-amz-meta-{key}"), value);
+            }
+        }
+    }
+
     opts.user_metadata = metadata;
     opts
 }
@@ -446,6 +475,51 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
     Ok(d)
 }
 
+pub(crate) async fn new_transition_candidate_reconciler(
+    tier: &TierConfig,
+) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
+    let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
+        TierType::S3 => Box::new(
+            WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::MinIO => Box::new(
+            WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::RustFS => Box::new(
+            WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::R2 => Box::new(
+            WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        _ => return Ok(None),
+    };
+    Ok(Some(reconciler))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -775,6 +849,37 @@ mod tests {
         assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
     }
 
+    #[test]
+    fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            "5a".repeat(32),
+        );
+
+        let opts = build_transition_put_options("COLD".to_string(), metadata);
+
+        for suffix in [
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+        ] {
+            assert!(opts.user_metadata.contains_key(&format!(
+                "x-amz-meta-{}",
+                rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
+            )));
+            assert!(opts.user_metadata.contains_key(&format!(
+                "x-amz-meta-{}{suffix}",
+                rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
+            )));
+        }
+    }
+
     #[test]
     fn build_transition_put_options_requests_no_checksum_and_content_md5() {
         // Regression for rustfs/rustfs#4811: transition uploads must leave the

crates/ecstore/src/services/tier/warm_backend_gcs.rs

@@ -19,6 +19,7 @@
 #![allow(clippy::all)]
 
 use std::collections::HashMap;
+use std::io::{Error, ErrorKind};
 use std::sync::Arc;
 
 use bytes::Bytes;
@@ -45,6 +46,19 @@ const MAX_PARTS_COUNT: i64 = 10000;
 const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
 const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
 
+fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
+    if remote_version.is_empty() {
+        return Ok(None);
+    }
+    let generation = remote_version
+        .parse::<i64>()
+        .map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
+    if generation <= 0 {
+        return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
+    }
+    Ok(Some(generation))
+}
+
 pub struct WarmBackendGCS {
     pub client: Arc<Storage>,
     pub control: Arc<StorageControl>,
@@ -105,6 +119,10 @@ impl WarmBackendGCS {
 
 #[async_trait::async_trait]
 impl WarmBackend for WarmBackendGCS {
+    fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
+        parse_generation(remote_version_id).map(|_| ())
+    }
+
     async fn put_with_meta(
         &self,
         object: &str,
@@ -135,6 +153,9 @@ impl WarmBackend for WarmBackendGCS {
 
     async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
         let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
+        if let Some(generation) = parse_generation(rv)? {
+            req = req.set_generation(generation);
+        }
 
         // Honor the requested byte range so Range GETs on tiered objects return the exact
         // interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
@@ -164,13 +185,15 @@ impl WarmBackend for WarmBackendGCS {
         // gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
         // deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
         // remote data forever.
-        self.control
+        let mut req = self
+            .control
             .delete_object()
             .set_bucket(format!("projects/_/buckets/{}", self.bucket))
-            .set_object(self.get_dest(object))
-            .send()
-            .await
-            .map_err(|e| std::io::Error::other(e.to_string()))?;
+            .set_object(self.get_dest(object));
+        if let Some(generation) = parse_generation(rv)? {
+            req = req.set_generation(generation);
+        }
+        req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
         Ok(())
     }
 
@@ -191,6 +214,30 @@ impl WarmBackend for WarmBackendGCS {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::parse_generation;
+    use std::io::ErrorKind;
+
+    #[test]
+    fn generation_parser_preserves_exact_numeric_versions() {
+        assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
+        assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
+        assert_eq!(
+            parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
+            Some(i64::MAX)
+        );
+    }
+
+    #[test]
+    fn generation_parser_rejects_unknown_or_non_positive_versions() {
+        for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
+            let err = parse_generation(value).expect_err("unknown generation must fail closed");
+            assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
+        }
+    }
+}
+
 /*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
   if err == nil {
     return nil

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendMinIO {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_r2.rs

@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendR2 {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -132,6 +132,20 @@ impl WarmBackend for WarmBackendRustFS {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -29,6 +29,7 @@ use crate::client::{
     api_remove::{RemoveObjectOptions, RemoveObjectResult},
     api_s3_datatypes::ListVersionsResult,
     credentials::{Credentials, SignatureType, Static, Value},
+    provider_versions::validate_remote_version_id,
     transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
     transition_api::{ReadCloser, ReaderImpl},
 };
@@ -36,7 +37,10 @@ use crate::error::ErrorResponse;
 use crate::error::error_resp_to_object_err;
 use crate::services::tier::{
     tier_config::TierS3,
-    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{
+        TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
+        build_transition_put_options,
+    },
 };
 use http::HeaderMap;
 use rustfs_utils::egress::validate_outbound_url;
@@ -189,44 +193,188 @@ impl WarmBackendS3 {
         remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
     }
 
-    async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
+    async fn probe_transition_candidate_versions(
+        &self,
+        object: &str,
+        bucket_versioning: RemoteBucketVersioning,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
         let mut opts = ListObjectsOptions::default();
-        opts.set("prefix", &self.get_dest(object));
-        opts.set("max-keys", "2");
-        self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
+        opts.set("prefix", &remote_object);
+        opts.set("max-keys", "1000");
+
+        let mut key_marker = String::new();
+        let mut version_id_marker = String::new();
+        let mut candidates = TransitionCandidateVersions::default();
+        loop {
+            let versions = self
+                .client
+                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
+                .await?;
+            candidates.extend(&remote_object, &versions);
+            if candidates.is_ambiguous() {
+                return Ok(TransitionCandidateProbe::Ambiguous);
+            }
+            if !versions.is_truncated {
+                return classify_transition_candidates(candidates, bucket_versioning);
+            }
+
+            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
+        }
+    }
+
+    async fn probe_transition_candidate_identity(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+        bucket_versioning: RemoteBucketVersioning,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
+        let mut opts = ListObjectsOptions::default();
+        opts.set("prefix", &remote_object);
+        opts.set("max-keys", "1000");
+        let mut key_marker = String::new();
+        let mut version_id_marker = String::new();
+        let mut matched_version = None;
+        let mut saw_unproven_candidate = false;
+
+        loop {
+            let versions = self
+                .client
+                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
+                .await?;
+            for version in versions.versions.iter().filter(|version| version.key == remote_object) {
+                let mut stat_opts = GetObjectOptions::default();
+                stat_opts.version_id.clone_from(&version.version_id);
+                let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
+                let mut metadata = info.user_metadata;
+                for (name, value) in &info.metadata {
+                    if (name
+                        .as_str()
+                        .starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
+                        || name
+                            .as_str()
+                            .starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
+                        && let Ok(value) = value.to_str()
+                    {
+                        metadata.insert(name.as_str().to_string(), value.to_string());
+                    }
+                }
+                if transition_candidate_metadata_matches(&metadata, identity)? {
+                    if matched_version.is_some() {
+                        return Ok(TransitionCandidateProbe::Ambiguous);
+                    }
+                    matched_version = Some(version.version_id.clone());
+                } else {
+                    saw_unproven_candidate = true;
+                }
+            }
+            if !versions.is_truncated {
+                if matched_version.is_none() && saw_unproven_candidate {
+                    return Ok(TransitionCandidateProbe::Unsupported);
+                }
+                let candidates = TransitionCandidateVersions {
+                    version_id: matched_version,
+                    ambiguous: false,
+                };
+                return classify_transition_candidates(candidates, bucket_versioning);
+            }
+            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
+        }
     }
 }
 
-fn classify_transition_candidate_versions(
-    remote_object: &str,
-    bucket_versioning: RemoteBucketVersioning,
-    versions: &ListVersionsResult,
-) -> TransitionCandidateProbe {
-    if versions.is_truncated {
-        return TransitionCandidateProbe::Ambiguous;
-    }
-
-    if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
-        return TransitionCandidateProbe::Ambiguous;
-    }
-
-    let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
-    let Some(version) = exact_versions.next() else {
-        return TransitionCandidateProbe::Missing;
+fn transition_candidate_metadata_matches(
+    metadata: &HashMap<String, String>,
+    identity: TransitionCandidateIdentity,
+) -> Result<bool, std::io::Error> {
+    use rustfs_utils::http::metadata_compat::{
+        SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
     };
-    if exact_versions.next().is_some() {
-        return TransitionCandidateProbe::Ambiguous;
+
+    let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
+    let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
+    if transaction_id.is_none() || destination_id.is_none() {
+        if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
+            || contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
+        {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                "transition candidate identity metadata is empty or conflicting",
+            ));
+        }
+        return Ok(false);
+    }
+    let expected_transaction_id = identity.transaction_id.to_string();
+    let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
+    Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
+}
+
+fn classify_transition_candidates(
+    candidates: TransitionCandidateVersions,
+    bucket_versioning: RemoteBucketVersioning,
+) -> Result<TransitionCandidateProbe, std::io::Error> {
+    let probe = candidates.classify(bucket_versioning);
+    if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
+        validate_remote_version_id(version_id)?;
+    }
+    Ok(probe)
+}
+
+fn advance_version_markers(
+    key_marker: &mut String,
+    version_id_marker: &mut String,
+    versions: &ListVersionsResult,
+) -> Result<(), std::io::Error> {
+    let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
+    if next_markers == (&*key_marker, &*version_id_marker) {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            "ListObjectVersions pagination markers did not advance",
+        ));
+    }
+    key_marker.clone_from(&versions.next_key_marker);
+    version_id_marker.clone_from(&versions.next_version_id_marker);
+    Ok(())
+}
+
+#[derive(Default)]
+struct TransitionCandidateVersions {
+    version_id: Option<String>,
+    ambiguous: bool,
+}
+
+impl TransitionCandidateVersions {
+    fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
+        for version in versions.versions.iter().filter(|version| version.key == remote_object) {
+            if self.version_id.is_some() {
+                self.ambiguous = true;
+                return;
+            }
+            self.version_id = Some(version.version_id.clone());
+        }
     }
 
-    match bucket_versioning {
-        RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
-        RemoteBucketVersioning::Suspended if version.version_id == "null" => {
-            TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
+    fn is_ambiguous(&self) -> bool {
+        self.ambiguous
+    }
+
+    fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
+        if self.ambiguous {
+            return TransitionCandidateProbe::Ambiguous;
         }
-        RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
-            TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
+        let Some(version_id) = self.version_id else {
+            return TransitionCandidateProbe::Missing;
+        };
+
+        match bucket_versioning {
+            RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
+            RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
+            RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
+                TransitionCandidateProbe::VersionedPresent(version_id)
+            }
+            RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
         }
-        RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
     }
 }
 
@@ -275,74 +423,163 @@ mod tests {
         }
     }
 
+    fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
+        let mut candidates = TransitionCandidateVersions::default();
+        for page in pages {
+            candidates.extend("archive/object", page);
+        }
+        candidates.classify(bucket_versioning)
+    }
+
+    fn candidate_identity() -> TransitionCandidateIdentity {
+        TransitionCandidateIdentity {
+            transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
+            destination_id: [0x5a; 32],
+        }
+    }
+
+    fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            identity.transaction_id.to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(identity.destination_id),
+        );
+        metadata
+    }
+
+    #[test]
+    fn transition_candidate_identity_requires_exact_compatible_metadata() {
+        let identity = candidate_identity();
+        let metadata = candidate_metadata(identity);
+        assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
+
+        let mut adjacent = metadata;
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut adjacent,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
+                .unwrap()
+                .to_string(),
+        );
+        assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
+
+        assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
+    }
+
+    #[test]
+    fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
+        let identity = candidate_identity();
+        let mut metadata = candidate_metadata(identity);
+        metadata.insert(
+            rustfs_utils::http::metadata_compat::internal_key_rustfs(
+                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            ),
+            uuid::Uuid::new_v4().to_string(),
+        );
+        assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
+    }
+
     #[test]
     fn transition_candidate_probe_classifier_is_fail_closed() {
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
-                RemoteBucketVersioning::Disabled,
-                &list_versions(&[], &[], false),
-            ),
+            classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
             TransitionCandidateProbe::Missing
         );
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
-                RemoteBucketVersioning::Disabled,
-                &list_versions(&[("archive/object", "")], &[], false),
-            ),
+            classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
             TransitionCandidateProbe::UnversionedPresent
         );
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
+            classify_pages(
                 RemoteBucketVersioning::Enabled,
-                &list_versions(&[("archive/object", "version-a")], &[], false),
+                &[list_versions(&[("archive/object", "version-a")], &[], false)],
             ),
             TransitionCandidateProbe::VersionedPresent("version-a".to_string())
         );
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
+            classify_pages(
                 RemoteBucketVersioning::Suspended,
-                &list_versions(&[("archive/object", "null")], &[], false),
+                &[list_versions(&[("archive/object", "null")], &[], false)],
             ),
             TransitionCandidateProbe::VersionedPresent("null".to_string())
         );
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
-                RemoteBucketVersioning::Enabled,
-                &list_versions(&[("archive/object", "")], &[], false),
-            ),
+            classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
             TransitionCandidateProbe::Ambiguous
         );
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
+            classify_pages(
                 RemoteBucketVersioning::Enabled,
-                &list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
+                &[list_versions(
+                    &[("archive/object", "version-a"), ("archive/object", "version-b")],
+                    &[],
+                    false,
+                )],
             ),
             TransitionCandidateProbe::Ambiguous
         );
+    }
+
+    #[test]
+    fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
+        assert_eq!(
+            classify_pages(
+                RemoteBucketVersioning::Enabled,
+                &[
+                    list_versions(&[], &[("archive/object", "marker-a")], true),
+                    list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
+                ],
+            ),
+            TransitionCandidateProbe::VersionedPresent("version-a".to_string())
+        );
         assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
+            classify_pages(
                 RemoteBucketVersioning::Enabled,
-                &list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
-            ),
-            TransitionCandidateProbe::Ambiguous
-        );
-        assert_eq!(
-            classify_transition_candidate_versions(
-                "archive/object",
-                RemoteBucketVersioning::Enabled,
-                &list_versions(&[("archive/object", "version-a")], &[], true),
+                &[
+                    list_versions(&[("archive/object", "version-a")], &[], true),
+                    list_versions(&[("archive/object", "version-b")], &[], false),
+                ],
             ),
             TransitionCandidateProbe::Ambiguous
         );
     }
 
+    #[test]
+    fn transition_candidate_pagination_advances_both_markers() {
+        let mut key_marker = "old-key".to_string();
+        let mut version_id_marker = "old-version".to_string();
+        let page = ListVersionsResult {
+            next_key_marker: "next-key".to_string(),
+            next_version_id_marker: "next-version".to_string(),
+            ..Default::default()
+        };
+
+        advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
+            .expect("new ListObjectVersions markers should advance pagination");
+        assert_eq!(key_marker, "next-key");
+        assert_eq!(version_id_marker, "next-version");
+
+        let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
+            .expect_err("repeated ListObjectVersions markers must fail closed");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
+    #[test]
+    fn transition_candidate_probe_rejects_untrusted_version_ids() {
+        let mut candidates = TransitionCandidateVersions::default();
+        candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
+
+        let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
+            .expect_err("control characters in listed version IDs must fail closed");
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
     #[test]
     fn remote_bucket_versioning_status_parser_fails_closed() {
         assert_eq!(
@@ -397,12 +634,7 @@ impl WarmBackend for WarmBackendS3 {
 
     async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
         let bucket_versioning = self.remote_bucket_versioning().await?;
-        let versions = self.list_transition_candidate_versions(object).await?;
-        Ok(classify_transition_candidate_versions(
-            &self.get_dest(object),
-            bucket_versioning,
-            &versions,
-        ))
+        self.probe_transition_candidate_versions(object, bucket_versioning).await
     }
 
     async fn in_use(&self) -> Result<bool, std::io::Error> {
@@ -414,3 +646,16 @@ impl WarmBackend for WarmBackendS3 {
         Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
     }
 }
+
+#[async_trait::async_trait]
+impl TransitionCandidateReconciler for WarmBackendS3 {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let bucket_versioning = self.remote_bucket_versioning().await?;
+        self.probe_transition_candidate_identity(object, identity, bucket_versioning)
+            .await
+    }
+}

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -46,6 +46,7 @@ use crate::diagnostics::get::{
     GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
     record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
 };
+use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
 use crate::disk::{
     DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
     PartTransactionAction, part_transaction_path,
@@ -3952,9 +3953,9 @@ impl SetDisks {
     /// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
     ///   every online disk's `xl.meta`, so a dir named by *any* replica is kept.
     /// * If a disk holds the object directory but its `xl.meta` is missing or
-    ///   unparsable, the object is treated as degraded and NOTHING is removed —
-    ///   the unreadable copy could be the only one naming a live data dir, and a
-    ///   heal must run first.
+    ///   unparsable, the object is treated as degraded and unmarked data dirs are
+    ///   never removed. A data dir carrying a committed delete-transaction marker
+    ///   remains reclaimable after a downgrade/re-upgrade cleanup interruption.
     /// * Only subdirectories whose names parse as a UUID are ever considered;
     ///   removal is non-recursive-safe via a recursive delete of the full stray
     ///   data-dir path only.
@@ -3969,7 +3970,7 @@ impl SetDisks {
         // physical UUID subdirectories present on each disk. Abort on any degraded
         // copy so a healable object is never stripped of a referenced data dir.
         let mut referenced: HashSet<Uuid> = HashSet::new();
-        let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
+        let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
         let mut healthy_metas = 0usize;
 
         for (i, disk) in disks.iter().enumerate() {
@@ -4005,6 +4006,22 @@ impl SetDisks {
                         // to the orphan-dir / dangling-object heal paths.
                         continue;
                     }
+                    let mut committed = Vec::with_capacity(physical.len());
+                    for dir in physical {
+                        let data_dir = format!("{object}/{dir}");
+                        let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
+                            entries.iter().any(|entry| {
+                                entry
+                                    .strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
+                                    .is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
+                            })
+                        });
+                        committed.push((dir, committed_delete));
+                    }
+                    if committed.iter().all(|(_, committed_delete)| *committed_delete) {
+                        per_disk_dirs.push((i, committed));
+                        continue;
+                    }
                     warn!(
                         target: "rustfs_ecstore::set_disk",
                         bucket, object,
@@ -4041,22 +4058,16 @@ impl SetDisks {
 
             healthy_metas += 1;
             if !physical.is_empty() {
-                per_disk_dirs.push((i, physical));
+                per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
             }
         }
 
-        // No healthy metadata anywhere: this is not a live object, so surplus dirs
-        // (if any) belong to the dangling-object heal path, not here.
-        if healthy_metas == 0 {
-            return Ok(0);
-        }
-
         // Phase 2: delete every physical data dir not referenced by the union.
         let mut removed = 0usize;
         for (i, physical) in per_disk_dirs {
             let Some(disk) = disks[i].as_ref() else { continue };
-            for dir in physical {
-                if referenced.contains(&dir) {
+            for (dir, committed_delete) in physical {
+                if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
                     continue;
                 }
                 let stray = format!("{object}/{dir}");

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -3379,85 +3379,98 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn complete_holds_object_then_upload_lock_through_commit() {
-        temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
-            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
-            let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
-            let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
-            let bucket = "multipart-layout-lock-order-bucket";
-            let object = "object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-            let create_opts = ObjectOptions {
-                user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
-                ..Default::default()
-            };
-            let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
-            let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
-            let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
-            let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
-            signaling.set_target(upload_resource.clone());
-            signaling.clear_observed();
-            let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
-            let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
+        temp_env::async_with_vars(
+            [
+                (crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
+                (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
+            ],
+            async {
+                let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
+                let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
+                let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
+                let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+                let bucket = "multipart-layout-lock-order-bucket";
+                let object = "object";
+                make_bucket_on_all(&disk_stores, bucket).await;
+                let create_opts = ObjectOptions {
+                    user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
+                    ..Default::default()
+                };
+                let (upload_id, parts) =
+                    stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
+                let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
+                let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
+                let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
+                signaling.set_target(upload_resource.clone());
+                signaling.clear_observed();
+                let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+                let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
 
-            let complete_store = set_disks.clone();
-            let complete_upload_id = upload_id.clone();
-            let complete = tokio::spawn(async move {
-                complete_store
-                    .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
+                let complete_store = set_disks.clone();
+                let complete_upload_id = upload_id.clone();
+                let complete = tokio::spawn(async move {
+                    complete_store
+                        .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
+                        .await
+                });
+                barrier.wait_until_paused().await;
+
+                let observed = signaling.observed();
+                let object_position = observed
+                    .iter()
+                    .position(|resource| resource == &object_resource)
+                    .expect("completion should acquire the object lock");
+                let upload_position = observed
+                    .iter()
+                    .position(|resource| resource == &upload_resource)
+                    .expect("completion should acquire the upload lock");
+                assert!(object_position < upload_position, "completion must acquire object before upload");
+
+                let abort_store = set_disks.clone();
+                let abort_upload_id = upload_id.clone();
+                let abort = tokio::spawn(async move {
+                    abort_store
+                        .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
+                        .await
+                });
+                signaling.wait_for_attempts(2).await;
+                tokio::task::yield_now().await;
+                assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
+
+                let list_store = set_disks.clone();
+                let list_upload_id = upload_id.clone();
+                let list = tokio::spawn(async move {
+                    list_store
+                        .list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
+                        .await
+                });
+                signaling.wait_for_attempts(3).await;
+                tokio::task::yield_now().await;
+                assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
+
+                barrier.release();
+                complete
                     .await
-            });
-            barrier.wait_until_paused().await;
-
-            let observed = signaling.observed();
-            let object_position = observed
-                .iter()
-                .position(|resource| resource == &object_resource)
-                .expect("completion should acquire the object lock");
-            let upload_position = observed
-                .iter()
-                .position(|resource| resource == &upload_resource)
-                .expect("completion should acquire the upload lock");
-            assert!(object_position < upload_position, "completion must acquire object before upload");
-
-            let abort_store = set_disks.clone();
-            let abort_upload_id = upload_id.clone();
-            let abort = tokio::spawn(async move {
-                abort_store
-                    .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
+                    .expect("completion task should not panic")
+                    .expect("completion should commit after the barrier is released");
+                let abort_err = abort
                     .await
-            });
-            signaling.wait_for_attempts(2).await;
-            tokio::task::yield_now().await;
-            assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
-
-            let list_store = set_disks.clone();
-            let list_upload_id = upload_id.clone();
-            let list = tokio::spawn(async move {
-                list_store
-                    .list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
+                    .expect("abort task should not panic")
+                    .expect_err("the committed upload should no longer exist when abort acquires the lock");
+                assert!(
+                    matches!(abort_err, StorageError::InvalidUploadID(..)),
+                    "abort should return InvalidUploadID after completion, got {abort_err:?}"
+                );
+                let list_err = list
                     .await
-            });
-            signaling.wait_for_attempts(3).await;
-            tokio::task::yield_now().await;
-            assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
-
-            barrier.release();
-            complete
-                .await
-                .expect("completion task should not panic")
-                .expect("completion should commit after the barrier is released");
-            let abort_err = abort
-                .await
-                .expect("abort task should not panic")
-                .expect_err("the committed upload should no longer exist when abort acquires the lock");
-            assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
-            let list_err = list
-                .await
-                .expect("ListParts task should not panic")
-                .expect_err("the committed upload should no longer exist when ListParts acquires the lock");
-            assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
-        })
+                    .expect("ListParts task should not panic")
+                    .expect_err("the committed upload should no longer exist when ListParts acquires the lock");
+                assert!(
+                    matches!(list_err, StorageError::InvalidUploadID(..)),
+                    "ListParts should return InvalidUploadID after completion, got {list_err:?}"
+                );
+            },
+        )
         .await;
     }
 

crates/ecstore/src/set_disk/ops/object.rs

@@ -562,13 +562,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                 &self.ctx.tier_config_mgr(),
             )
             .await?;
-            return Ok(finish_set_disk_read_lock(
-                gr,
-                read_lock_guard.take(),
-                lock_optimization_enabled,
-                bucket,
-                object,
-            ));
+            return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), None, bucket, object));
         }
 
         // App-layer object data cache probe: metadata (etag/size) is resolved
@@ -695,6 +689,18 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
             return Ok(reader);
         }
 
+        let snapshot_lease = if lock_optimization_enabled {
+            match fi.data_dir.filter(|data_dir| !data_dir.is_nil()) {
+                Some(data_dir) => {
+                    let data_dir_path = format!("{object}/{data_dir}");
+                    acquire_snapshot_leases(&disks, bucket, &data_dir_path, fi.erasure.data_blocks).await
+                }
+                None => None,
+            }
+        } else {
+            None
+        };
+
         match codec_streaming_gate.decision {
             GetCodecStreamingDecision::Use => {
                 match Self::get_object_decode_reader_with_fileinfo(
@@ -724,13 +730,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                         // Carry the hook probe result so the app layer skips its
                         // now-redundant lookup on the streaming miss path (ODC-16).
                         reader.body_source = body_source;
-                        return Ok(finish_set_disk_read_lock(
-                            reader,
-                            read_lock_guard.take(),
-                            lock_optimization_enabled,
-                            bucket,
-                            object,
-                        ));
+                        return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), snapshot_lease, bucket, object));
                     }
                     core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
                         record_get_codec_streaming_gate_decision(
@@ -770,15 +770,22 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let set_index = self.set_index;
         let pool_index = self.pool_index;
         let skip_verify = opts.skip_verify_bitrot;
-        if lock_optimization_enabled {
+        let producer_snapshot_lease = snapshot_lease.clone();
+        if let Some(lease) = snapshot_lease {
             release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
-            debug!(bucket, object, "Lock optimization: released read lock before streaming read");
+            reader.stream = Box::new(SnapshotLeaseReader {
+                inner: reader.stream,
+                lease: Some(lease),
+                terminal_error: false,
+            });
+            debug!(bucket, object, "Lock optimization: replaced read lock with snapshot leases");
         }
 
-        // When lock optimization is disabled, keep the read-lock guard in the
-        // task so it lives for the duration of the streaming read.
+        // The producer shares the lease lifetime with the body so cancellation
+        // cannot release the snapshot while the duplex task is still unwinding.
         tokio::spawn(async move {
             let _guard = read_lock_guard;
+            let _snapshot_lease = producer_snapshot_lease;
             let mut writer = GetObjectDownstreamWriter::new(wd);
             // Do not wrap the entire read+write pipeline in `disk_read_timeout`.
             // `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
@@ -2327,13 +2334,25 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
     }
 }
 
+#[cfg(test)]
 fn persisted_transition_version(
     remote_version: &str,
 ) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
     persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
 }
 
+#[cfg(test)]
 fn remote_version_state_writer_enabled() -> bool {
+    remote_version_state_writer_fleet_proof().is_some()
+}
+
+fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
+    remote_version_state_writer_requested()
+        .then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
+        .flatten()
+}
+
+fn remote_version_state_writer_requested() -> bool {
     remote_version_state_writer_enabled_for(
         rustfs_utils::get_env_bool(
             rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
@@ -2343,11 +2362,25 @@ fn remote_version_state_writer_enabled() -> bool {
             rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
             rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
         ),
+        true,
     )
 }
 
-fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
-    requested && fleet_confirmed
+fn remote_version_state_writer_fleet_proof_matches(
+    proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
+) -> bool {
+    remote_version_state_writer_fleet_proof_matches_for(
+        remote_version_state_writer_requested(),
+        crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
+    )
+}
+
+fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
+    requested && fleet_proof_matches
+}
+
+fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
+    requested && fleet_confirmed && fleet_proof_valid
 }
 
 fn persisted_transition_version_with_gate(
@@ -2366,7 +2399,7 @@ fn persisted_transition_version_with_gate(
         Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
         Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
             std::io::ErrorKind::Unsupported,
-            "opaque remote tier versions require the operator-attested fleet gate",
+            "opaque remote tier versions require the operator-attested live fleet capability gate",
         )),
         Err(_) if remote_version == "null" => {
             Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
@@ -2612,7 +2645,7 @@ mod transition_upload_completion_tests {
 mod transition_version_id_tests {
     use super::{
         TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
-        remote_version_state_writer_enabled_for,
+        remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
     };
     use rustfs_filemeta::TransitionVersionState;
     use uuid::Uuid;
@@ -2655,15 +2688,35 @@ mod transition_version_id_tests {
 
     #[test]
     fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
-        for (case, requested, fleet_confirmed, expected) in [
-            ("old defaults", false, false, false),
-            ("missing fleet confirmation", true, false, false),
-            ("missing local opt-in", false, true, false),
-            ("explicitly unconfirmed fleet", true, false, false),
-            ("rolled-back writer", false, true, false),
-            ("fully upgraded fleet", true, true, true),
+        for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
+            ("old defaults", false, false, false, false),
+            ("missing fleet confirmation", true, false, true, false),
+            ("missing local opt-in", false, true, true, false),
+            ("missing fleet proof", true, true, false, false),
+            ("explicitly unconfirmed fleet", true, false, true, false),
+            ("rolled-back writer", false, true, true, false),
+            ("fully upgraded fleet", true, true, true, true),
         ] {
-            assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
+            assert_eq!(
+                remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
+                expected,
+                "{case}"
+            );
+        }
+    }
+
+    #[test]
+    fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
+        for (case, requested, fleet_proof_matches, expected) in [
+            ("operator gate closed", false, true, false),
+            ("fleet proof changed", true, false, false),
+            ("current authorization", true, true, true),
+        ] {
+            assert_eq!(
+                remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
+                expected,
+                "{case}"
+            );
         }
     }
 
@@ -3848,6 +3901,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let dest_obj = transaction.remote_object.clone();
         let mut transition_meta = (*oi.user_defined).clone();
         transition_meta.insert("name".to_string(), object.to_string());
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut transition_meta,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            transaction.transaction_id.to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut transition_meta,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(transaction.backend_fingerprint),
+        );
 
         if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
             transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
@@ -3958,20 +4021,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
             return Err(err.into());
         }
-        let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
-            Ok(version) => version,
-            Err(err) => {
-                let cleanup_api = transition_cleanup_store(&self.ctx).await;
-                if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
-                    return Err(StorageError::Io(std::io::Error::other(format!(
-                        "{err}; rejected remote upload cleanup failed: {cleanup_err}"
-                    ))));
+        let fleet_proof = remote_version_state_writer_fleet_proof();
+        let remote_version_requires_fleet_proof =
+            !candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
+        let (transition_version_id, transition_version_state) =
+            match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
+                Ok(version) => version,
+                Err(err) => {
+                    let cleanup_api = transition_cleanup_store(&self.ctx).await;
+                    if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
+                        return Err(StorageError::Io(std::io::Error::other(format!(
+                            "{err}; rejected remote upload cleanup failed: {cleanup_err}"
+                        ))));
+                    }
+                    delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
+                        .await;
+                    return Err(err.into());
                 }
-                delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
-                    .await;
-                return Err(err.into());
-            }
-        };
+            };
         if let Err(err) = advance_and_save_transition_transaction(
             transaction_api.as_ref(),
             &mut transaction,
@@ -4087,6 +4154,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
         #[cfg(test)]
         pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
+        // This check is the fleet-proof lease linearization point. Revocation
+        // blocks later commits; an already-authorized local quorum commit is
+        // allowed to finish without holding a synchronous lock across I/O.
+        if remote_version_requires_fleet_proof
+            && !fleet_proof
+                .as_ref()
+                .is_some_and(remote_version_state_writer_fleet_proof_matches)
+        {
+            drop(transition_lock_guard);
+            if upload_cleanup.cleanup().await.is_ok() {
+                delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
+                    .await;
+            }
+            return Err(Error::other("remote version state fleet capability changed during transition"));
+        }
         if let Err(err) = advance_and_save_transition_transaction(
             transaction_api.as_ref(),
             &mut transaction,

crates/ecstore/src/store/bucket.rs

@@ -568,6 +568,7 @@ mod tests {
     };
     use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
     use crate::bucket::metadata_sys;
+    use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
     use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
     use crate::error::StorageError;
     use crate::object_api::{ObjectOptions, PutObjReader};
@@ -589,6 +590,7 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use std::time::{Duration, SystemTime};
     use time::OffsetDateTime;
+    use tokio::io::AsyncReadExt;
     use tokio::sync::{Notify, OnceCell};
     use tokio_util::sync::CancellationToken;
     use uuid::Uuid;
@@ -1048,7 +1050,7 @@ mod tests {
             .await
             .expect("bucket should be created");
         for disk_path in &disk_paths {
-            tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
+            tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
                 .await
                 .expect("empty directory remnant should be created");
         }
@@ -1257,6 +1259,55 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn bucket_delete_preserves_put_committed_after_empty_scan() {
+        let (_, ecstore) = setup_bucket_delete_test_env().await;
+        let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
+        let object = "committed-after-empty-scan";
+        let payload = b"object committed after DeleteBucket empty scan".to_vec();
+
+        ecstore
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        let barrier = install_delete_bucket_empty_scan_barrier();
+        let delete_store = ecstore.clone();
+        let delete_bucket = bucket.clone();
+        let delete = tokio::spawn(async move {
+            delete_store
+                .delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
+                .await
+        });
+        barrier.wait_until_paused().await;
+
+        let mut put_reader = PutObjReader::from_vec(payload.clone());
+        ecstore
+            .put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
+            .await
+            .expect("PUT should commit while DeleteBucket is paused after its empty scan");
+
+        barrier.release();
+        let err = delete
+            .await
+            .expect("DeleteBucket task should join")
+            .expect_err("DeleteBucket must reject a PUT committed after its empty scan");
+        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
+
+        let mut reader = ecstore
+            .get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("committed object should remain readable after DeleteBucket fails");
+        let mut restored = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut restored)
+            .await
+            .expect("object body should remain readable");
+        assert_eq!(restored, payload);
+    }
+
     #[tokio::test]
     #[serial]
     async fn bucket_recreation_does_not_publish_unverified_usage() {

crates/ecstore/src/store/init.rs

@@ -562,10 +562,12 @@ mod tests {
             },
             tier_sweeper::Jentry,
             transition_transaction::{
-                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
-                TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
-                TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
-                save_transition_transaction_record,
+                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
+                TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
+                TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator, load_transition_transaction_record,
+                recover_transition_transaction_records, save_transition_transaction_record,
             },
         },
         client::transition_api::ReaderImpl,
@@ -575,6 +577,7 @@ mod tests {
         services::tier::{
             test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
             tier::{TIER_CONFIG_FILE, TierConfigMgr},
+            tier_config::{TierConfig, TierType, TierWasabi},
             tier_mutation_intent::{
                 TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
                 TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -1163,6 +1166,37 @@ mod tests {
             .len()
     }
 
+    #[cfg(feature = "test-util")]
+    async fn register_transition_reconcile_test_tier(
+        handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
+        tier_name: &str,
+    ) -> MockWarmBackend {
+        let backend = MockWarmBackend::new();
+        let mut manager = handle.write().await;
+        manager.tiers.insert(
+            tier_name.to_string(),
+            TierConfig {
+                version: "v1".to_string(),
+                tier_type: TierType::Wasabi,
+                name: tier_name.to_string(),
+                wasabi: Some(TierWasabi {
+                    name: tier_name.to_string(),
+                    endpoint: "https://s3.wasabisys.com".to_string(),
+                    access_key: "test-access-key".to_string(),
+                    secret_key: "test-secret-key".to_string(),
+                    bucket: "mock-tier".to_string(),
+                    prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
+                    region: "us-east-1".to_string(),
+                }),
+                ..Default::default()
+            },
+        );
+        manager
+            .install_test_driver(tier_name, Box::new(backend.clone()))
+            .expect("mock fallback tier driver should install");
+        backend
+    }
+
     #[cfg(feature = "test-util")]
     async fn wait_for_tier_delete_journal_recovery(
         store: Arc<crate::store::ECStore>,
@@ -2738,7 +2772,7 @@ mod tests {
             .await;
             crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
 
-            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+            let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
             let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
                 .await
                 .expect("tier lease should resolve")
@@ -2809,6 +2843,157 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXOPERATOR";
+        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: None,
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Unversioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("transaction should enter unknown upload outcome state");
+
+        let remote_version = uuid::Uuid::new_v4().to_string();
+        backend.set_put_remote_version(Some(remote_version.clone())).await;
+        let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
+        backend
+            .put(
+                &transaction.remote_object,
+                ReaderImpl::Body(candidate.clone()),
+                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
+            )
+            .await
+            .expect("mock backend should accept candidate");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+
+        let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
+            .await
+            .expect("operator inspection should probe the candidate");
+        assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
+
+        let wrong_version = uuid::Uuid::new_v4().to_string();
+        let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
+            .await
+            .expect_err("a mismatched exact version must fail before deleting a candidate");
+        assert!(matches!(
+            err,
+            TransitionOperatorError::CandidateVersionMismatch {
+                expected,
+                actual: TransitionOperatorProbe::VersionedPresent(ref observed),
+            } if expected == wrong_version && observed == &remote_version
+        ));
+        assert!(backend.contains(&transaction.remote_object).await);
+        assert_eq!(backend.exact_remove_count(), 0);
+        load_transition_transaction_record(store.clone(), transaction.transaction_id)
+            .await
+            .expect("an incorrect exact version must retain the transaction journal");
+
+        let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
+            .await
+            .expect("operator-confirmed exact candidate should be deleted");
+        assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
+        assert!(result.journal_observed_after_delete);
+        assert_eq!(backend.exact_remove_count(), 1);
+        assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
+        load_transition_transaction_record(store.clone(), transaction.transaction_id)
+            .await
+            .expect("candidate deletion must retain the transaction journal");
+
+        finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
+            .await
+            .expect("a separately confirmed missing candidate should permit finalization");
+        assert!(matches!(
+            load_transition_transaction_record(store, transaction.transaction_id).await,
+            Err(Error::ConfigNotFound)
+        ));
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn operator_finalize_retains_record_without_missing_proof() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXOPERATORFAIL";
+        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: None,
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Unversioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("transaction should enter unknown upload outcome state");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+        backend
+            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
+            .await;
+
+        assert!(matches!(
+            finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
+            Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
+        ));
+        load_transition_transaction_record(store, transaction.transaction_id)
+            .await
+            .expect("an unsupported probe must retain the transaction journal");
+        assert_eq!(backend.remove_count().await, 0);
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -2819,7 +3004,7 @@ mod tests {
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
 
         let tier_name = "TXRESPONSELOSS";
-        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
         let bucket = "transition-response-loss-bucket";
         let object = "source.bin";
         store

crates/ecstore/src/store/list_objects.rs

@@ -6895,6 +6895,7 @@ mod test {
         )
         .await
         .expect("local disk should be created");
+        disk.make_volume("bucket").await.expect("test bucket should be created");
         let mut fi = FileInfo::new("object", 1, 1);
         fi.volume = "bucket".to_owned();
         fi.name = "object".to_owned();

crates/ecstore/src/test_tracing.rs

@@ -0,0 +1,43 @@
+//! Test-only guard against tracing's process-global callsite-interest cache.
+//!
+//! Any test that installs its own subscriber and then asserts on span or event
+//! context is asserting on process-global state. See
+//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
+//! guard fixes it.
+
+/// Stop a sibling test thread from poisoning tracing's process-global callsite
+/// interest cache while this test asserts on span or event context.
+///
+/// `tracing` caches every callsite's `Interest` in **process-global** state, and
+/// the first thread to reach a callsite fixes that value. While at most one
+/// dispatcher is registered, tracing-core takes a fast path
+/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
+/// newly-registered callsite's interest from whichever subscriber is current
+/// *on the registering thread*, and registration happens exactly once (guarded
+/// by a CAS in `DefaultCallsite::register`).
+///
+/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
+/// reaches a production callsite first, from a thread with no subscriber
+/// installed: the interest is derived from that thread's `NoSubscriber` and
+/// cached as `Interest::never()` for the whole process. From then on the
+/// callsite is dead for *every* later caller, including a test that installed
+/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
+/// a `warn!`/`info!` event never fires at all.
+///
+/// Registering a second, inert dispatcher closes both halves of that race:
+///
+/// * constructing it rebuilds every *already-registered* callsite's interest
+///   against the live dispatcher set — which includes the caller's subscriber —
+///   repairing whatever a sibling may already have poisoned; and
+/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
+///   a callsite registered *later* by any thread is resolved against that live
+///   set instead of the registering thread's `NoSubscriber`.
+///
+/// Call this **after** installing the test's subscriber, and hold the returned
+/// guard for the rest of the test. Only the `cargo test` fallback needs it:
+/// nextest runs each test in its own process, where there is no sibling thread
+/// to lose the race to (see `docs/testing/README.md`).
+#[must_use = "callsite interest is only pinned while the returned guard is held"]
+pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
+    tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
+}

crates/io-metrics/src/internode_metrics.rs

@@ -67,6 +67,7 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
 const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
 const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
 const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
+const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
 const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
 const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
 
@@ -159,6 +160,7 @@ pub struct InternodeMetricsSnapshot {
     pub operation_write_shutdown_errors_total: u64,
     pub signature_v1_fallback_total: u64,
     pub body_digest_fallback_total: u64,
+    pub replay_scope_fallback_total: u64,
     pub replay_cache_overflow_total: u64,
 }
 
@@ -180,6 +182,7 @@ pub struct InternodeMetrics {
     msgpack_json_decode_error_total: AtomicU64,
     signature_v1_fallback_total: AtomicU64,
     body_digest_fallback_total: AtomicU64,
+    replay_scope_fallback_total: AtomicU64,
     replay_cache_overflow_total: AtomicU64,
 }
 
@@ -431,6 +434,13 @@ impl InternodeMetrics {
         counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
     }
 
+    /// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
+    /// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
+    pub fn record_replay_scope_fallback(&self) {
+        self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
+        counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
+    }
+
     /// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
     /// full. Overflow fails closed, so a sustained non-zero rate means
     /// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
@@ -488,6 +498,7 @@ impl InternodeMetrics {
             operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
             signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
             body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
+            replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
             replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
         }
     }
@@ -510,6 +521,7 @@ impl InternodeMetrics {
         self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
         self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
         self.body_digest_fallback_total.store(0, Ordering::Relaxed);
+        self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
         self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
     }
 }
@@ -887,6 +899,19 @@ mod tests {
         assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
     }
 
+    #[test]
+    fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
+        let metrics = InternodeMetrics::default();
+        assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
+
+        metrics.record_replay_scope_fallback();
+        metrics.record_replay_scope_fallback();
+        assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
+
+        metrics.reset_for_test();
+        assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
+    }
+
     #[test]
     fn cluster_peer_flips_offline_after_threshold_and_back_online() {
         // Unique addr keeps this independent of the process-global registry / other tests.

crates/keystone/Cargo.toml

@@ -25,6 +25,9 @@ keywords = ["rustfs", "openstack", "keystone", "authentication", "s3"]
 categories = ["authentication", "web-programming"]
 authors.workspace = true
 
+[lints]
+workspace = true
+
 [dependencies]
 tokio = { workspace = true, features = ["rt", "sync"] }
 reqwest = { workspace = true, features = ["json"] }

crates/madmin/src/metrics.rs

@@ -545,6 +545,8 @@ pub struct ScannerMetrics {
     pub collected_at: DateTime<Utc>,
     #[serde(rename = "current_cycle")]
     pub current_cycle: u64,
+    #[serde(rename = "current_cycle_active", default, skip_serializing_if = "Option::is_none")]
+    pub current_cycle_active: Option<bool>,
     #[serde(rename = "current_started")]
     pub current_started: DateTime<Utc>,
     #[serde(rename = "cycle_complete_times")]
@@ -718,7 +720,31 @@ pub struct ScannerMetrics {
 }
 
 impl ScannerMetrics {
+    pub fn is_current_cycle_active(&self) -> bool {
+        self.current_cycle_active.unwrap_or(self.current_cycle > 0)
+    }
+
     pub fn merge(&mut self, other: &Self) {
+        // Legacy nodes omit the activity field and use a non-zero cycle as
+        // their active signal. New nodes publish explicit first-cycle and idle
+        // states, including cycle zero.
+        let self_cycle_active = self.is_current_cycle_active();
+        let other_cycle_active = other.is_current_cycle_active();
+        let self_cycle_authority = (
+            self_cycle_active,
+            self.current_cycle,
+            self.cycles_completed_at.len(),
+            self.cycles_completed_at.as_slice(),
+            self.current_started,
+        );
+        let other_cycle_authority = (
+            other_cycle_active,
+            other.current_cycle,
+            other.cycles_completed_at.len(),
+            other.cycles_completed_at.as_slice(),
+            other.current_started,
+        );
+        let other_cycle_is_authoritative = self_cycle_authority < other_cycle_authority;
         let other_is_newer = self.collected_at < other.collected_at;
         if other_is_newer {
             self.collected_at = other.collected_at;
@@ -854,15 +880,12 @@ impl ScannerMetrics {
             self.ongoing_buckets = other.ongoing_buckets;
         }
 
-        if self.current_cycle < other.current_cycle {
+        if other_cycle_is_authoritative {
             self.current_cycle = other.current_cycle;
             self.cycles_completed_at = other.cycles_completed_at.clone();
             self.current_started = other.current_started;
         }
-
-        if other.cycles_completed_at.len() > self.cycles_completed_at.len() {
-            self.cycles_completed_at = other.cycles_completed_at.clone();
-        }
+        self.current_cycle_active = Some(self_cycle_active || other_cycle_active);
 
         if !other.life_time_ops.is_empty() && self.life_time_ops.is_empty() {
             self.life_time_ops = other.life_time_ops.clone();
@@ -931,7 +954,13 @@ impl Metrics {
         if let Some(scanner) = other.scanner.as_ref() {
             match self.scanner {
                 Some(ref mut s_scanner) => s_scanner.merge(scanner),
-                None => self.scanner = Some(scanner.clone()),
+                None => {
+                    let mut scanner = scanner.clone();
+                    if scanner.current_cycle_active.is_none() {
+                        scanner.current_cycle_active = Some(scanner.is_current_cycle_active());
+                    }
+                    self.scanner = Some(scanner);
+                }
             }
         }
 
@@ -1389,6 +1418,280 @@ pub struct Operations {
 mod tests {
     use super::*;
 
+    #[test]
+    fn scanner_metrics_serializes_cycle_active_presence() {
+        let missing_value = serde_json::to_value(ScannerMetrics::default()).expect("scanner metrics should serialize");
+        assert!(missing_value.get("current_cycle_active").is_none());
+        let missing: ScannerMetrics =
+            serde_json::from_value(missing_value).expect("older scanner metrics without cycle-active should decode");
+        assert_eq!(missing.current_cycle_active, None);
+
+        let explicit_false_value = serde_json::to_value(ScannerMetrics {
+            current_cycle_active: Some(false),
+            ..Default::default()
+        })
+        .expect("scanner metrics with explicit cycle-active should serialize");
+        assert_eq!(explicit_false_value["current_cycle_active"], serde_json::Value::Bool(false));
+        let explicit_false: ScannerMetrics =
+            serde_json::from_value(explicit_false_value).expect("explicit cycle-active should decode");
+        assert_eq!(explicit_false.current_cycle_active, Some(false));
+    }
+
+    #[test]
+    fn scanner_metrics_merge_prefers_an_active_first_cycle() {
+        let collected_at = Utc::now();
+        let idle_started = collected_at - chrono::Duration::hours(1);
+        let active_started = collected_at - chrono::Duration::seconds(5);
+        let mut scanner = ScannerMetrics {
+            collected_at,
+            current_cycle: 0,
+            current_cycle_active: Some(false),
+            current_started: idle_started,
+            ..Default::default()
+        };
+
+        scanner.merge(&ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::seconds(1),
+            current_cycle: 0,
+            current_cycle_active: Some(true),
+            current_started: active_started,
+            ..Default::default()
+        });
+
+        assert_eq!(scanner.current_cycle_active, Some(true));
+        assert_eq!(scanner.current_cycle, 0);
+        assert_eq!(scanner.current_started, active_started);
+    }
+
+    #[test]
+    fn metrics_merge_preserves_explicit_active_first_cycle() {
+        let mut aggregated = Metrics::default();
+        aggregated.merge(&Metrics {
+            scanner: Some(ScannerMetrics {
+                current_cycle_active: Some(true),
+                ..Default::default()
+            }),
+            ..Default::default()
+        });
+
+        let scanner = aggregated.scanner.expect("aggregated scanner metrics");
+        assert_eq!(scanner.current_cycle_active, Some(true));
+        assert_eq!(scanner.current_cycle, 0);
+    }
+
+    #[test]
+    fn scanner_metrics_merge_preserves_legacy_nonzero_active_signal() {
+        let collected_at = Utc::now();
+        let mut scanner = ScannerMetrics {
+            collected_at,
+            ..Default::default()
+        };
+
+        scanner.merge(&ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::seconds(1),
+            current_cycle: 7,
+            ..Default::default()
+        });
+
+        assert_eq!(scanner.current_cycle_active, Some(true));
+        assert_eq!(scanner.current_cycle, 7);
+    }
+
+    #[test]
+    fn metrics_merge_normalizes_first_legacy_scanner_snapshot() {
+        let legacy = Metrics {
+            scanner: Some(ScannerMetrics {
+                current_cycle: 7,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let mut aggregated = Metrics::default();
+
+        aggregated.merge(&legacy);
+
+        let scanner = aggregated.scanner.expect("aggregated scanner metrics");
+        assert_eq!(scanner.current_cycle_active, Some(true));
+        assert_eq!(scanner.current_cycle, 7);
+    }
+
+    #[test]
+    fn scanner_metrics_merge_preserves_explicit_inactive_nonzero_cycle() {
+        let collected_at = Utc::now();
+        let mut scanner = ScannerMetrics::default();
+
+        scanner.merge(&ScannerMetrics {
+            collected_at,
+            current_cycle: 7,
+            current_cycle_active: Some(false),
+            ..Default::default()
+        });
+
+        assert_eq!(scanner.current_cycle_active, Some(false));
+        assert_eq!(scanner.current_cycle, 7);
+    }
+
+    #[test]
+    fn scanner_metrics_merge_cycle_active_is_order_independent() {
+        let collected_at = Utc::now();
+        let legacy_active = ScannerMetrics {
+            collected_at,
+            current_cycle: 7,
+            current_started: collected_at - chrono::Duration::seconds(10),
+            ..Default::default()
+        };
+        let explicit_idle = ScannerMetrics {
+            collected_at,
+            current_cycle: 0,
+            current_cycle_active: Some(false),
+            current_started: collected_at - chrono::Duration::hours(1),
+            ..Default::default()
+        };
+
+        let mut legacy_first = legacy_active.clone();
+        legacy_first.merge(&explicit_idle);
+        let mut legacy_second = explicit_idle.clone();
+        legacy_second.merge(&legacy_active);
+        assert_eq!(legacy_first.current_cycle_active, Some(true));
+        assert_eq!(legacy_second.current_cycle_active, Some(true));
+        assert_eq!(legacy_first.current_cycle, 7);
+        assert_eq!(legacy_second.current_cycle, 7);
+
+        let explicit_inactive_nonzero = ScannerMetrics {
+            collected_at,
+            current_cycle: 7,
+            current_cycle_active: Some(false),
+            ..Default::default()
+        };
+        let mut inactive_first = explicit_inactive_nonzero.clone();
+        inactive_first.merge(&explicit_idle);
+        let mut inactive_second = explicit_idle;
+        inactive_second.merge(&explicit_inactive_nonzero);
+        assert_eq!(inactive_first.current_cycle_active, Some(false));
+        assert_eq!(inactive_second.current_cycle_active, Some(false));
+        assert_eq!(inactive_first.current_cycle, 7);
+        assert_eq!(inactive_second.current_cycle, 7);
+    }
+
+    #[test]
+    fn scanner_metrics_merge_cycle_authority_is_order_independent() {
+        let collected_at = Utc::now();
+        let completion = collected_at - chrono::Duration::minutes(1);
+        let earlier_active = ScannerMetrics {
+            collected_at,
+            current_cycle: 7,
+            current_cycle_active: Some(true),
+            current_started: collected_at - chrono::Duration::seconds(10),
+            cycles_completed_at: vec![completion],
+            ..Default::default()
+        };
+        let later_active = ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::seconds(1),
+            current_cycle: 7,
+            current_cycle_active: Some(true),
+            current_started: collected_at - chrono::Duration::seconds(5),
+            cycles_completed_at: vec![completion],
+            ..Default::default()
+        };
+
+        let mut earlier_first = earlier_active.clone();
+        earlier_first.merge(&later_active);
+        let mut later_first = later_active.clone();
+        later_first.merge(&earlier_active);
+
+        assert_eq!(earlier_first.current_started, later_active.current_started);
+        assert_eq!(later_first.current_started, later_active.current_started);
+        assert_eq!(earlier_first.cycles_completed_at, later_active.cycles_completed_at);
+        assert_eq!(later_first.cycles_completed_at, later_active.cycles_completed_at);
+
+        let stale_idle = ScannerMetrics {
+            collected_at,
+            current_cycle_active: Some(false),
+            current_started: collected_at - chrono::Duration::hours(1),
+            ..Default::default()
+        };
+        let completed_idle = ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::seconds(1),
+            current_cycle_active: Some(false),
+            current_started: collected_at - chrono::Duration::seconds(5),
+            cycles_completed_at: vec![completion],
+            ..Default::default()
+        };
+
+        let mut stale_first = stale_idle.clone();
+        stale_first.merge(&completed_idle);
+        let mut completed_first = completed_idle.clone();
+        completed_first.merge(&stale_idle);
+
+        assert_eq!(stale_first.current_started, completed_idle.current_started);
+        assert_eq!(completed_first.current_started, completed_idle.current_started);
+        assert_eq!(stale_first.cycles_completed_at, completed_idle.cycles_completed_at);
+        assert_eq!(completed_first.cycles_completed_at, completed_idle.cycles_completed_at);
+    }
+
+    #[test]
+    fn scanner_metrics_merge_cycle_authority_is_associative() {
+        let collected_at = Utc::now();
+        let older_completion = collected_at - chrono::Duration::minutes(3);
+        let last_completion = collected_at - chrono::Duration::minutes(1);
+        let cycle_seven = ScannerMetrics {
+            collected_at,
+            current_cycle: 7,
+            current_cycle_active: Some(true),
+            current_started: collected_at - chrono::Duration::seconds(10),
+            cycles_completed_at: vec![older_completion, last_completion],
+            ..Default::default()
+        };
+        let cycle_eight = ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::seconds(1),
+            current_cycle: 8,
+            current_cycle_active: Some(true),
+            current_started: collected_at - chrono::Duration::seconds(5),
+            cycles_completed_at: vec![last_completion],
+            ..Default::default()
+        };
+        let newer_idle = ScannerMetrics {
+            collected_at: collected_at + chrono::Duration::hours(1),
+            current_cycle_active: Some(false),
+            current_started: collected_at - chrono::Duration::hours(1),
+            ..Default::default()
+        };
+
+        let mut left_associative = cycle_seven.clone();
+        left_associative.merge(&cycle_eight);
+        left_associative.merge(&newer_idle);
+
+        let mut right_group = cycle_eight.clone();
+        right_group.merge(&newer_idle);
+        let mut right_associative = cycle_seven.clone();
+        right_associative.merge(&right_group);
+
+        assert_eq!(left_associative.current_cycle, 8);
+        assert_eq!(right_associative.current_cycle, 8);
+        assert_eq!(left_associative.current_started, cycle_eight.current_started);
+        assert_eq!(right_associative.current_started, cycle_eight.current_started);
+        assert_eq!(left_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
+        assert_eq!(right_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
+
+        for order in [
+            [&cycle_seven, &cycle_eight, &newer_idle],
+            [&cycle_seven, &newer_idle, &cycle_eight],
+            [&cycle_eight, &cycle_seven, &newer_idle],
+            [&cycle_eight, &newer_idle, &cycle_seven],
+            [&newer_idle, &cycle_seven, &cycle_eight],
+            [&newer_idle, &cycle_eight, &cycle_seven],
+        ] {
+            let mut merged = ScannerMetrics::default();
+            for scanner in order {
+                merged.merge(scanner);
+            }
+            assert_eq!(merged.current_cycle_active, Some(true));
+            assert_eq!(merged.current_cycle, 8);
+            assert_eq!(merged.current_started, cycle_eight.current_started);
+            assert_eq!(merged.cycles_completed_at, cycle_eight.cycles_completed_at);
+        }
+    }
+
     #[test]
     fn scanner_metrics_merge_aggregates_partial_cycles_by_source() {
         let collected_at = Utc::now();

crates/obs/src/metrics/stats_collector.rs

@@ -262,11 +262,11 @@ async fn obs_site_replication_stats() -> ReplicationStats {
 }
 
 fn current_scanner_cycle_age_seconds(
-    current_cycle: u64,
+    current_cycle_active: bool,
     current_started: chrono::DateTime<Utc>,
     now: chrono::DateTime<Utc>,
 ) -> u64 {
-    if current_cycle == 0 {
+    if !current_cycle_active {
         0
     } else {
         now.signed_duration_since(current_started).num_seconds().max(0) as u64
@@ -1090,7 +1090,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
     let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
     let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
     let active_paths = metrics.active_scan_paths as u64;
-    let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
+    let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle_active, metrics.current_started, now);
     let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
     let current_cycle_age = current_cycle_age_seconds as f64;
     let last_cycle_duration = metrics.last_cycle_duration_seconds;
@@ -1464,21 +1464,21 @@ mod tests {
     fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
         let now = Utc::now();
 
-        assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
+        assert_eq!(current_scanner_cycle_age_seconds(false, now - chrono::Duration::seconds(30), now), 0);
     }
 
     #[test]
     fn current_scanner_cycle_age_seconds_clamps_future_start() {
         let now = Utc::now();
 
-        assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
+        assert_eq!(current_scanner_cycle_age_seconds(true, now + chrono::Duration::seconds(30), now), 0);
     }
 
     #[test]
-    fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
+    fn current_scanner_cycle_age_seconds_reports_active_first_cycle_elapsed_time() {
         let now = Utc::now();
 
-        assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
+        assert_eq!(current_scanner_cycle_age_seconds(true, now - chrono::Duration::seconds(45), now), 45);
     }
 
     #[test]

crates/protocols/Cargo.toml

@@ -25,6 +25,9 @@ description = "Protocol implementations for RustFS (FTPS, SFTP, etc.)"
 keywords = ["ftp", "sftp", "protocol", "storage", "rustfs"]
 categories = ["network-programming", "filesystem"]
 
+[lints]
+workspace = true
+
 [features]
 default = []
 ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"]

crates/protocols/src/common/dummy_storage.rs

@@ -608,7 +608,7 @@ impl StorageBackend for DummyBackend {
             inner.put_object_calls.push(PutObjectCall {
                 bucket: input.bucket.to_string(),
                 key: input.key.to_string(),
-                metadata: input.metadata.clone(),
+                metadata: input.metadata,
             });
             let stall = inner.stall_put_object;
             let entered = inner.put_object_entered.clone();
@@ -731,7 +731,7 @@ impl StorageBackend for DummyBackend {
             inner.create_multipart_calls.push(CreateMultipartCall {
                 bucket: input.bucket.to_string(),
                 key: input.key.to_string(),
-                metadata: input.metadata.clone(),
+                metadata: input.metadata,
             });
         }
         match self.inner.lock().expect("lock").create_multipart_upload.pop_front() {
@@ -749,7 +749,7 @@ impl StorageBackend for DummyBackend {
             inner.upload_part_calls.push(UploadPartCall {
                 bucket: input.bucket.to_string(),
                 key: input.key.to_string(),
-                upload_id: input.upload_id.to_string(),
+                upload_id: input.upload_id,
                 part_number: input.part_number,
                 content_length: input.content_length,
             });
@@ -787,7 +787,7 @@ impl StorageBackend for DummyBackend {
             inner.complete_multipart_calls.push(CompleteCall {
                 bucket: input.bucket.to_string(),
                 key: input.key.to_string(),
-                upload_id: input.upload_id.to_string(),
+                upload_id: input.upload_id,
                 part_count,
             });
         }
@@ -808,7 +808,7 @@ impl StorageBackend for DummyBackend {
             inner.abort_multipart_calls.push(AbortCall {
                 bucket: input.bucket.to_string(),
                 key: input.key.to_string(),
-                upload_id: input.upload_id.to_string(),
+                upload_id: input.upload_id,
             });
         }
         match self.inner.lock().expect("lock").abort_multipart_upload.pop_front() {

crates/protocols/src/ftps/driver.rs

@@ -379,9 +379,7 @@ where
             .await
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
-        let prefix_with_slash = prefix
-            .clone()
-            .map(|p| if p.ends_with('/') { p.to_string() } else { format!("{}/", p) });
+        let prefix_with_slash = prefix.clone().map(|p| if p.ends_with('/') { p } else { format!("{}/", p) });
 
         let list_input = ListObjectsV2Input::builder()
             .bucket(bucket)

crates/protocols/src/ftps/server.rs

@@ -371,7 +371,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
 
         let ftps_user = FtpsUser {
             username: principal.username.clone(),
-            name: identity.credentials.name.clone(),
+            name: identity.credentials.name,
             session_context,
         };
 

crates/protocols/src/sftp/write.rs

@@ -1495,12 +1495,12 @@ mod tests {
             let buffer_len_u64 = part_buffer_len as u64;
             let phase = match phase_variant {
                 0 => WritePhase::Buffering {
-                    part_buffer: part_buffer.clone(),
+                    part_buffer,
                 },
                 1 => WritePhase::Streaming {
                     upload_id: "UP-proptest".to_string(),
                     abort_authorized: true,
-                    part_buffer: part_buffer.clone(),
+                    part_buffer,
                     uploaded_parts: Vec::new(),
                     next_part_number,
                 },

crates/protocols/src/swift/bulk.rs

@@ -627,9 +627,7 @@ mod tests {
     #[test]
     fn test_parse_paths_with_empty_lines() {
         let body = "/container1/file1.txt\n\n/container2/file2.txt\n   \n/container1/file3.txt";
-        let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect();
-
-        assert_eq!(paths.len(), 3);
+        assert_eq!(body.lines().filter(|line| !line.trim().is_empty()).count(), 3);
     }
 
     /// Tests for the `extract_tar_entries` async function.

crates/protocols/src/swift/object.rs

@@ -206,10 +206,9 @@ impl ObjectKeyMapper {
     #[allow(dead_code)] // Used in: object operations
     pub fn normalize_path(object: &str) -> String {
         // Split by '/', filter out empty segments (except if it's the end)
-        let segments: Vec<&str> = object.split('/').collect();
         let has_trailing_slash = object.ends_with('/');
 
-        let normalized_segments: Vec<&str> = segments.into_iter().filter(|s| !s.is_empty()).collect();
+        let normalized_segments: Vec<&str> = object.split('/').filter(|s| !s.is_empty()).collect();
 
         let mut result = normalized_segments.join("/");
 

crates/protocols/src/swift/router.rs

@@ -84,20 +84,19 @@ impl SwiftRouter {
         Self { enabled, url_prefix }
     }
 
+    /// Return whether a URI matches the Swift route shape without allocating
+    /// decoded route components.
+    pub fn matches(&self, uri: &Uri) -> bool {
+        let Some(path) = self.route_path(uri) else {
+            return false;
+        };
+        let mut segments = path.trim_start_matches('/').split('/');
+        segments.next() == Some("v1") && segments.next().is_some_and(Self::is_valid_account)
+    }
+
     /// Parse a URI and return a SwiftRoute if it matches Swift URL pattern
     pub fn route(&self, uri: &Uri, method: Method) -> Option<SwiftRoute> {
-        if !self.enabled {
-            return None;
-        }
-
-        let path = uri.path();
-
-        // Strip optional prefix
-        let path = if let Some(prefix) = &self.url_prefix {
-            path.strip_prefix(&format!("/{}/", prefix))?
-        } else {
-            path
-        };
+        let path = self.route_path(uri)?;
 
         // Split path into segments - preserve empty segments to maintain object key fidelity
         // Swift allows trailing slashes and consecutive slashes in object names (e.g., "dir/" or "a//b")
@@ -175,6 +174,17 @@ impl SwiftRouter {
     fn is_valid_account(account: &str) -> bool {
         ACCOUNT_PATTERN.is_match(account)
     }
+
+    fn route_path<'a>(&self, uri: &'a Uri) -> Option<&'a str> {
+        if !self.enabled {
+            return None;
+        }
+        let path = uri.path();
+        let Some(prefix) = &self.url_prefix else {
+            return Some(path);
+        };
+        path.strip_prefix('/')?.strip_prefix(prefix)?.strip_prefix('/')
+    }
 }
 
 #[cfg(test)]
@@ -281,6 +291,42 @@ mod tests {
         assert_eq!(route, None);
     }
 
+    #[test]
+    fn test_matches_agrees_with_route_without_decoding_components() {
+        let router = SwiftRouter::new(true, None);
+        for path in [
+            "/v1/AUTH_project",
+            "/v1/AUTH_project/",
+            "/v1/AUTH_project/container",
+            "/v1/AUTH_project/container/a%20long/object",
+            "//v1/AUTH_project/container/object",
+            "/v1/not-a-swift-account/object",
+            "/v1/AUTH_/object",
+            "/bucket/object",
+        ] {
+            let uri = path.parse().expect("Swift route test URI");
+            assert_eq!(
+                router.matches(&uri),
+                router.route(&uri, Method::GET).is_some(),
+                "classification must match full routing for {path}"
+            );
+        }
+
+        let prefixed_router = SwiftRouter::new(true, Some("swift".to_string()));
+        for path in [
+            "/swift/v1/AUTH_project/container",
+            "/swiftish/v1/AUTH_project/container",
+            "/v1/AUTH_project/container",
+        ] {
+            let uri = path.parse().expect("prefixed Swift route test URI");
+            assert_eq!(
+                prefixed_router.matches(&uri),
+                prefixed_router.route(&uri, Method::GET).is_some(),
+                "prefixed classification must match full routing for {path}"
+            );
+        }
+    }
+
     #[test]
     fn test_project_id_extraction() {
         let route = SwiftRoute::Account {

crates/protocols/src/webdav/driver.rs

@@ -222,7 +222,7 @@ where
                         created: modified,
                         is_dir: false,
                         etag: output.e_tag.as_ref().map(etag_to_string),
-                        content_type: output.content_type.map(|c| c.to_string()),
+                        content_type: output.content_type,
                     }) as Box<dyn DavMetaData>)
                 }
                 Err(e) => {
@@ -1185,7 +1185,7 @@ where
                             created: modified,
                             is_dir: false,
                             etag: output.e_tag.as_ref().map(etag_to_string),
-                            content_type: output.content_type.map(|c| c.to_string()),
+                            content_type: output.content_type,
                         }) as Box<dyn DavMetaData>)
                     }
                     ResolvedPath::Directory { metadata, .. } => {
@@ -1204,7 +1204,7 @@ where
                             created: modified,
                             is_dir: true,
                             etag: metadata.as_ref().and_then(|output| output.e_tag.as_ref().map(etag_to_string)),
-                            content_type: metadata.and_then(|output| output.content_type.map(|c| c.to_string())),
+                            content_type: metadata.and_then(|output| output.content_type),
                         }) as Box<dyn DavMetaData>)
                     }
                 };
@@ -2035,7 +2035,7 @@ mod tests {
             _access_key: &str,
             _secret_key: &str,
         ) -> Result<ListObjectsV2Output, Self::Error> {
-            let prefix = input.prefix.map(|p| p.to_string()).unwrap_or_default();
+            let prefix = input.prefix.unwrap_or_default();
             let mut keys: Vec<String> = self
                 .state
                 .lock()

crates/protocols/tests/swift_listing_symlink_tests.rs

@@ -436,12 +436,10 @@ fn test_symlink_to_nested_object() {
 fn test_listing_empty_container() {
     let objects: Vec<&str> = vec![];
 
-    let filtered: Vec<_> = objects.iter().collect();
-    assert_eq!(filtered.len(), 0);
+    assert!(objects.is_empty());
 
     // With prefix
-    let with_prefix: Vec<_> = objects.iter().filter(|o| o.starts_with("prefix/")).collect();
-    assert_eq!(with_prefix.len(), 0);
+    assert!(!objects.iter().any(|o| o.starts_with("prefix/")));
 }
 
 /// Test listing lexicographic ordering

crates/protos/src/lib.rs

@@ -171,6 +171,7 @@ pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SI
 pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
 pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
 pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
+pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
 pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
 pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
 pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
@@ -197,6 +198,49 @@ pub fn is_heal_control_capability_probe(command: &[u8]) -> bool {
     command.len() == HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + 16 && command.starts_with(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX)
 }
 
+pub fn remote_version_state_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
+    let mut probe = Vec::with_capacity(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
+    probe.extend_from_slice(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX);
+    probe.extend_from_slice(nonce);
+    probe
+}
+
+pub fn is_remote_version_state_capability_probe(command: &[u8]) -> bool {
+    command.len() == REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + 16
+        && command.starts_with(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)
+}
+
+pub fn encode_remote_version_state_capability(
+    topology_member: &str,
+    process_epoch: &[u8; 16],
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let topology_member = topology_member.as_bytes();
+    let mut result = Vec::with_capacity(8 + topology_member.len() + process_epoch.len());
+    result.extend_from_slice(&u64::try_from(topology_member.len())?.to_be_bytes());
+    result.extend_from_slice(topology_member);
+    result.extend_from_slice(process_epoch);
+    Ok(result)
+}
+
+pub fn decode_remote_version_state_capability(result: &[u8]) -> Result<(&str, &[u8; 16]), &'static str> {
+    let member_len = result
+        .get(..8)
+        .and_then(|value| value.try_into().ok())
+        .map(u64::from_be_bytes)
+        .ok_or("remote version state capability is truncated")?;
+    let member_len = usize::try_from(member_len).map_err(|_| "remote version state member length cannot be represented")?;
+    let member_end = 8_usize
+        .checked_add(member_len)
+        .ok_or("remote version state member length overflow")?;
+    let topology_member = std::str::from_utf8(result.get(8..member_end).ok_or("remote version state member is truncated")?)
+        .map_err(|_| "remote version state member is not UTF-8")?;
+    let process_epoch = result
+        .get(member_end..)
+        .and_then(|value| value.try_into().ok())
+        .ok_or("remote version state process epoch has an invalid length")?;
+    Ok((topology_member, process_epoch))
+}
+
 /// Builds the stable byte representation authenticated for a heal-control request.
 ///
 /// This deliberately does not reuse protobuf encoding: mixed-version peers may
@@ -1656,10 +1700,12 @@ mod scanner_activity_tests {
 #[cfg(test)]
 mod heal_control_tests {
     use super::{
-        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
-        canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
+        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
+        canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body,
+        decode_remote_version_state_capability, encode_remote_version_state_capability, heal_control_capability_probe,
         heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
-        internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
+        internode_rpc_timeout, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
+        normalize_internode_rpc_timeout, remote_version_state_capability_probe,
     };
     use crate::heal_control;
     use std::time::Duration;
@@ -1716,6 +1762,29 @@ mod heal_control_tests {
         assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
     }
 
+    #[test]
+    fn remote_version_state_capability_probe_requires_exact_nonce() {
+        let probe = remote_version_state_capability_probe(&[7; 16]);
+        assert!(is_remote_version_state_capability_probe(&probe));
+        assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
+    }
+
+    #[test]
+    fn remote_version_state_capability_binds_member_and_process_epoch() {
+        let encoded =
+            encode_remote_version_state_capability("node-a:9000", &[7; 16]).expect("small capability response should encode");
+        assert_eq!(
+            decode_remote_version_state_capability(&encoded).expect("capability response should decode"),
+            ("node-a:9000", &[7; 16])
+        );
+        assert!(decode_remote_version_state_capability(&encoded[..encoded.len() - 1]).is_err());
+
+        let mut invalid_utf8 =
+            encode_remote_version_state_capability("node-a", &[7; 16]).expect("small capability response should encode");
+        invalid_utf8[8] = 0xff;
+        assert!(decode_remote_version_state_capability(&invalid_utf8).is_err());
+    }
+
     #[test]
     fn canonical_response_binds_request_and_result() {
         let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();

crates/s3select-api/Cargo.toml

@@ -25,6 +25,9 @@ keywords = ["s3-select", "api", "rustfs", "Minio", "object-store"]
 categories = ["web-programming", "development-tools", "asynchronous"]
 documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/"
 
+[lints]
+workspace = true
+
 [dependencies]
 metrics = { workspace = true }
 async-trait.workspace = true

crates/s3select-query/Cargo.toml

@@ -25,6 +25,9 @@ keywords = ["s3-select", "query-engine", "rustfs", "Minio", "data-retrieval"]
 categories = ["web-programming", "development-tools", "data-structures"]
 documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_query/"
 
+[lints]
+workspace = true
+
 [dependencies]
 rustfs-s3select-api = { workspace = true }
 async-recursion = { workspace = true }

crates/s3select-query/src/metadata/mod.rs

@@ -76,7 +76,7 @@ impl ContextProviderExtension for MetadataProvider {
 
         let table_handle = self.build_table_handle()?;
 
-        Ok(Arc::new(TableSourceAdapter::try_new(table_ref.clone(), table_name, table_handle)?))
+        Ok(Arc::new(TableSourceAdapter::try_new(table_ref, table_name, table_handle)?))
     }
 }
 

crates/scanner/Cargo.toml

@@ -47,7 +47,7 @@ rmp-serde = { workspace = true }
 hmac = { workspace = true }
 sha2 = { workspace = true }
 rustfs-filemeta = { workspace = true }
-tokio-util = { workspace = true, features = ["io", "compat"] }
+tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
 rustfs-ecstore = { workspace = true }
 rustfs-storage-api = { workspace = true }
 http = { workspace = true }

crates/scanner/src/data_usage_define.rs

@@ -48,6 +48,7 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
 const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
 
 pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
+pub(crate) const DATA_USAGE_CACHE_KEY_FORMAT: u16 = 1;
 const DATA_USAGE_CACHE_SAVE_RETRIES: u32 = 2;
 const DATA_USAGE_CACHE_BACKUP_SAVE_TIMEOUT_SECS_MAX: u64 = 5;
 const DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES: u32 = 0;
@@ -437,6 +438,8 @@ pub struct DataUsageCacheInfo {
     pub snapshot_complete: bool,
     #[serde(default)]
     pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
+    #[serde(default)]
+    pub cache_key_format: u16,
 }
 
 impl Serialize for DataUsageCacheInfo {
@@ -446,7 +449,7 @@ impl Serialize for DataUsageCacheInfo {
     {
         // Keep this metadata map-encoded so older readers can ignore fields
         // appended by newer scanner versions during rolling upgrades.
-        let mut state = serializer.serialize_map(Some(15))?;
+        let mut state = serializer.serialize_map(Some(16))?;
         state.serialize_entry("name", &self.name)?;
         state.serialize_entry("next_cycle", &self.next_cycle)?;
         state.serialize_entry("leader_epoch", &self.leader_epoch)?;
@@ -462,6 +465,7 @@ impl Serialize for DataUsageCacheInfo {
         state.serialize_entry("source", &self.source)?;
         state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
         state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
+        state.serialize_entry("cache_key_format", &self.cache_key_format)?;
         state.end()
     }
 }
@@ -500,19 +504,34 @@ impl DataUsageCache {
 
         let source_matches = self.info.source == Some(source);
         let plan_matches = self.info.scan_plan_digest == Some(scan_plan_digest);
-        let reusable = self.info.name == name
+        let metadata_is_reusable = self.info.name == name
             && self.info.leader_epoch == leader_epoch
             && plan_matches
-            && (source_matches || (!require_source && self.info.source.is_none()));
+            && (source_matches || (!require_source && self.info.source.is_none()))
+            && self.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
+        let reusable = metadata_is_reusable
+            && (self.cache.is_empty()
+                || if name == DATA_USAGE_ROOT || self.info.snapshot_complete {
+                    self.checked_flatten_complete_scope(name).is_some()
+                } else {
+                    self.checked_flatten(name).is_some()
+                });
         if !reusable {
+            let pending_heals = if self.info.name == name {
+                std::mem::take(&mut self.info.pending_heals)
+            } else {
+                Vec::new()
+            };
             *self = Self::default();
             self.info.name = name.to_string();
+            self.info.pending_heals = pending_heals;
         }
 
         self.info.next_cycle = next_cycle;
         self.info.leader_epoch = leader_epoch;
         self.info.source = Some(source);
         self.info.scan_plan_digest = Some(scan_plan_digest);
+        self.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
         self.info.snapshot_complete = false;
         if reusable {
             DataUsageCachePrepareOutcome::Reused
@@ -576,36 +595,60 @@ impl DataUsageCache {
     }
 
     pub(crate) fn checked_flatten(&self, path: &str) -> Option<DataUsageEntry> {
+        self.checked_flatten_inner(path).map(|(entry, _)| entry)
+    }
+
+    pub(crate) fn checked_flatten_complete(&self, path: &str) -> Option<DataUsageEntry> {
+        self.checked_flatten_inner(path)
+            .filter(|(_, visited)| *visited == self.cache.len())
+            .map(|(entry, _)| entry)
+    }
+
+    pub(crate) fn checked_flatten_complete_scope(&self, path: &str) -> Option<DataUsageEntry> {
+        if path == DATA_USAGE_ROOT {
+            return self.checked_flatten_complete(path);
+        }
+        let (entry, visited) = self.checked_flatten_inner(path)?;
+        let root_parent_only = {
+            let path_key = hash_path(path).key();
+            self.cache
+                .get(DATA_USAGE_ROOT)
+                .is_some_and(|root| root_is_parent_only(root, &path_key))
+        };
+        let expected_entries = self.cache.len().saturating_sub(usize::from(root_parent_only));
+        (visited == expected_entries).then_some(entry)
+    }
+
+    fn checked_flatten_inner(&self, path: &str) -> Option<(DataUsageEntry, usize)> {
         let root_key = hash_path(path).key();
-        let root = self.cache.get(&root_key)?;
-        let mut visited = HashSet::from([root_key]);
-        let mut pending = root.children.iter().map(|child| (child.clone(), 1usize)).collect::<Vec<_>>();
+        let (root_key, root) = self.cache.get_key_value(&root_key)?;
+        if root.compacted && !root.children.is_empty() {
+            return None;
+        }
+        let mut visited = HashSet::from([root_key.as_str()]);
+        let mut pending = root.children.iter().map(|child| (child.as_str(), 1usize)).collect::<Vec<_>>();
         let mut flattened = DataUsageEntry::default();
-        let mut root_entry = root.clone();
-        root_entry.children.clear();
-        if !flattened.checked_merge(&root_entry) {
+        if !flattened.checked_merge(root) {
             return None;
         }
         flattened.compacted = root.compacted;
 
         while let Some((key, depth)) = pending.pop() {
-            if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key.clone()) {
+            if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key) {
                 return None;
             }
-            let entry = self.cache.get(&key)?;
-            if depth == MAX_DATA_USAGE_CACHE_DEPTH && !entry.children.is_empty() {
+            let entry = self.cache.get(key)?;
+            if (entry.compacted || depth == MAX_DATA_USAGE_CACHE_DEPTH) && !entry.children.is_empty() {
                 return None;
             }
-            pending.extend(entry.children.iter().map(|child| (child.clone(), depth + 1)));
+            pending.extend(entry.children.iter().map(|child| (child.as_str(), depth + 1)));
 
-            let mut child_entry = entry.clone();
-            child_entry.children.clear();
-            if !flattened.checked_merge(&child_entry) {
+            if !flattened.checked_merge(entry) {
                 return None;
             }
         }
 
-        Some(flattened)
+        Some((flattened, visited.len()))
     }
 
     fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
@@ -1524,6 +1567,18 @@ fn mark_with_depth(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut Has
     }
 }
 
+fn root_is_parent_only(root: &DataUsageEntry, child: &str) -> bool {
+    root.children.len() == 1
+        && root.children.contains(child)
+        && root.size == 0
+        && root.objects == 0
+        && root.versions == 0
+        && root.delete_markers == 0
+        && root.replication_stats.is_none()
+        && !root.compacted
+        && root.failed_objects == 0
+}
+
 /// Trait for storage-specific operations on DataUsageCache
 #[async_trait::async_trait]
 pub trait DataUsageCacheStorage {
@@ -2287,6 +2342,7 @@ mod tests {
         assert!(decoded.source.is_none());
         assert!(!decoded.snapshot_complete);
         assert!(decoded.scan_plan_digest.is_none());
+        assert_eq!(decoded.cache_key_format, 0);
     }
 
     #[test]
@@ -2328,6 +2384,7 @@ mod tests {
         assert!(decoded.source.is_none());
         assert!(!decoded.snapshot_complete);
         assert!(decoded.scan_plan_digest.is_none());
+        assert_eq!(decoded.cache_key_format, 0);
     }
 
     #[test]
@@ -2363,6 +2420,7 @@ mod tests {
                 source: Some(DataUsageCacheSource::new(1, 2)),
                 snapshot_complete: true,
                 scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                 ..Default::default()
             },
             ..Default::default()
@@ -2381,6 +2439,7 @@ mod tests {
         assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
         assert!(current.info.snapshot_complete);
         assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
+        assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
         assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3));
 
         let decoded: OldDataUsageCache = rmp_serde::from_slice(&buf).expect("Old reader failed to deserialize new cache");
@@ -2442,6 +2501,7 @@ mod tests {
                 source: Some(source),
                 snapshot_complete: false,
                 scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                 ..Default::default()
             },
             ..Default::default()
@@ -2466,6 +2526,254 @@ mod tests {
         assert!(!cache.info.snapshot_complete);
     }
 
+    #[test]
+    fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let pending_heal = PendingScannerHeal {
+            kind: PendingScannerHealKind::Object,
+            bucket: "bucket".to_string(),
+            object: Some("prefix/object".to_string()),
+            version_id: Some("version-a".to_string()),
+            scan_mode: HealScanMode::Deep,
+            first_seen: 1,
+            last_attempt: 2,
+            attempts: 3,
+            last_admission_result: "full".to_string(),
+            last_admission_reason: "queue_full".to_string(),
+        };
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                pending_heals: vec![pending_heal.clone()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reused);
+        assert_eq!(cache.info.pending_heals, vec![pending_heal]);
+        assert!(cache.cache.is_empty());
+        assert!(!cache.info.snapshot_complete);
+    }
+
+    #[test]
+    fn data_usage_cache_prepare_for_scan_preserves_namespace_pending_heals_during_key_format_rebuild() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let pending_heal = PendingScannerHeal {
+            kind: PendingScannerHealKind::Object,
+            bucket: "bucket".to_string(),
+            object: Some("prefix/object".to_string()),
+            version_id: Some("version-a".to_string()),
+            scan_mode: HealScanMode::Deep,
+            first_seen: 1,
+            last_attempt: 2,
+            attempts: 3,
+            last_admission_result: "full".to_string(),
+            last_admission_reason: "queue_full".to_string(),
+        };
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: DATA_USAGE_ROOT.to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                pending_heals: vec![pending_heal.clone()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
+
+        let outcome = cache.prepare_for_scan(DATA_USAGE_ROOT, 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
+        assert_eq!(cache.info.pending_heals, vec![pending_heal]);
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
+    #[test]
+    fn data_usage_cache_prepare_for_scan_drops_pending_heals_from_a_different_scope() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "old-bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                pending_heals: vec![PendingScannerHeal {
+                    kind: PendingScannerHealKind::Object,
+                    bucket: "old-bucket".to_string(),
+                    object: Some("prefix/object".to_string()),
+                    version_id: None,
+                    scan_mode: HealScanMode::Normal,
+                    first_seen: 1,
+                    last_attempt: 2,
+                    attempts: 3,
+                    last_admission_result: "full".to_string(),
+                    last_admission_reason: "queue_full".to_string(),
+                }],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let outcome = cache.prepare_for_scan("new-bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
+        assert_eq!(cache.info.name, "new-bucket");
+        assert!(cache.info.pending_heals.is_empty());
+    }
+
+    #[test]
+    fn data_usage_cache_prepare_for_scan_resets_unknown_key_format() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT + 1,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.replace(
+            "bucket",
+            "",
+            DataUsageEntry {
+                objects: 3,
+                ..Default::default()
+            },
+        );
+
+        let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
+    #[test]
+    fn data_usage_cache_prepare_for_scan_resets_persisted_windows_key_mismatch() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let root_key = hash_path("bucket").key();
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                snapshot_complete: true,
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.cache.insert(
+            root_key,
+            DataUsageEntry {
+                children: HashSet::from(["bucket/prefix".to_string()]),
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            "bucket\\prefix".to_string(),
+            DataUsageEntry {
+                objects: 3,
+                ..Default::default()
+            },
+        );
+
+        let encoded = cache.marshal_msg().expect("legacy Windows cache should serialize");
+        let mut decoded = DataUsageCache::unmarshal(&encoded).expect("legacy Windows cache should deserialize");
+        let outcome = decoded.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
+        assert!(decoded.cache.is_empty());
+        assert_eq!(decoded.info.name, "bucket");
+        assert_eq!(decoded.info.next_cycle, 8);
+        assert_eq!(decoded.info.source, Some(source));
+        assert!(!decoded.info.snapshot_complete);
+    }
+
+    #[test]
+    fn data_usage_cache_prepare_for_scan_resets_current_cache_with_dangling_child() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.cache.insert(
+            hash_path("bucket").key(),
+            DataUsageEntry {
+                children: HashSet::from([hash_path("bucket/missing").key()]),
+                ..Default::default()
+            },
+        );
+
+        let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
+    #[test]
+    fn data_usage_cache_prepare_for_scan_resets_complete_bucket_cache_with_detached_entry() {
+        let source = DataUsageCacheSource::new(1, 0);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                snapshot_complete: true,
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.cache.insert(
+            hash_path("bucket").key(),
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            hash_path("bucket/detached").key(),
+            DataUsageEntry {
+                objects: 2,
+                ..Default::default()
+            },
+        );
+        assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
+        assert!(
+            cache.checked_flatten_complete("bucket").is_none(),
+            "complete bucket cache reuse must reject detached entries"
+        );
+
+        let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
+
+        assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
     #[test]
     fn data_usage_cache_prepare_for_scan_rejects_legacy_cache_without_a_bucket_plan() {
         let source = DataUsageCacheSource::new(0, 0);
@@ -2836,6 +3144,98 @@ mod tests {
         );
     }
 
+    #[test]
+    fn checked_flatten_complete_rejects_detached_entries() {
+        let mut cache = DataUsageCache::default();
+        cache.cache.insert(
+            hash_path("bucket").key(),
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            hash_path("bucket/detached").key(),
+            DataUsageEntry {
+                objects: 2,
+                ..Default::default()
+            },
+        );
+
+        assert_eq!(
+            cache.checked_flatten("bucket").map(|entry| entry.objects),
+            Some(1),
+            "subtree flattening may ignore entries outside the requested subtree"
+        );
+        assert!(
+            cache.checked_flatten_complete("bucket").is_none(),
+            "an authoritative cache root must reach every persisted entry"
+        );
+    }
+
+    #[test]
+    fn checked_flatten_rejects_compacted_entries_with_children() {
+        let root_key = hash_path("bucket").key();
+        let child_key = hash_path("bucket/prefix").key();
+        let mut cache = DataUsageCache::default();
+        cache.cache.insert(
+            root_key,
+            DataUsageEntry {
+                children: HashSet::from([child_key.clone()]),
+                compacted: true,
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            child_key,
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+
+        assert!(
+            cache.checked_flatten_complete("bucket").is_none(),
+            "a compacted entry cannot retain child links without double-counting"
+        );
+    }
+
+    #[test]
+    fn checked_flatten_rejects_compacted_descendants_with_children() {
+        let root_key = hash_path("bucket").key();
+        let child_key = hash_path("bucket/prefix").key();
+        let grandchild_key = hash_path("bucket/prefix/object").key();
+        let mut cache = DataUsageCache::default();
+        cache.cache.insert(
+            root_key,
+            DataUsageEntry {
+                children: HashSet::from([child_key.clone()]),
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            child_key,
+            DataUsageEntry {
+                objects: 1,
+                children: HashSet::from([grandchild_key.clone()]),
+                compacted: true,
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            grandchild_key,
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+
+        assert!(
+            cache.checked_flatten("bucket").is_none(),
+            "a compacted descendant cannot retain child links without double-counting"
+        );
+    }
+
     #[test]
     fn checked_flatten_accepts_depth_limit_and_rejects_deeper_tree() {
         let root_key = hash_path("bucket").key();

crates/scanner/src/remote_scanner.rs

@@ -14,8 +14,8 @@
 
 use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
 use crate::scanner_io::{
-    ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, cache_snapshot_is_current, scanner_cache_lock_resource,
-    scanner_cache_lock_timeout, scanner_set_disk_inventory,
+    DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, current_cache_root_or_prepare,
+    scanner_cache_lock_resource, scanner_cache_lock_timeout, scanner_set_disk_inventory,
 };
 use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
 use crate::storage_api::scan::NamespaceLocking as _;
@@ -966,32 +966,34 @@ async fn scan_and_persist_local_bucket(
     let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
         RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
     })?;
-    if cache_snapshot_is_current(&cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest) {
-        if guard.is_lock_lost() {
-            return Err(RemoteScannerServerError::worker(
-                "remote namespace scanner cache lock was lost before reusing the current snapshot",
-            ));
+    let scan_state = current_cache_root_or_prepare(&mut cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest, true);
+    match scan_state {
+        DataUsageCacheScanState::Current(usage) => {
+            if guard.is_lock_lost() {
+                return Err(RemoteScannerServerError::worker(
+                    "remote namespace scanner cache lock was lost before reusing the current snapshot",
+                ));
+            }
+            return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
+                source,
+                scan_plan_digest,
+                usage: *usage,
+                pending_maintenance_work: !cache.info.pending_heals.is_empty(),
+            })));
         }
-        return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
-            source,
-            scan_plan_digest,
-            usage: cache_root_entry_info(&cache)
-                .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache is corrupt: {err}")))?,
-            pending_maintenance_work: !cache.info.pending_heals.is_empty(),
-        })));
-    }
-    match cache.prepare_for_scan(&bucket, next_cycle, leader_epoch, source, scan_plan_digest, true) {
-        DataUsageCachePrepareOutcome::RejectedNewerCycle => {
-            return Ok(RemoteScannerFrameResult::CycleAhead {
-                required_cycle: cache.info.next_cycle,
-            });
-        }
-        DataUsageCachePrepareOutcome::RejectedNewerLeader => {
-            return Err(RemoteScannerServerError::worker(
-                "remote namespace scanner rejected work from an older leader epoch",
-            ));
-        }
-        DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
+        DataUsageCacheScanState::Prepared { outcome, .. } => match outcome {
+            DataUsageCachePrepareOutcome::RejectedNewerCycle => {
+                return Ok(RemoteScannerFrameResult::CycleAhead {
+                    required_cycle: cache.info.next_cycle,
+                });
+            }
+            DataUsageCachePrepareOutcome::RejectedNewerLeader => {
+                return Err(RemoteScannerServerError::worker(
+                    "remote namespace scanner rejected work from an older leader epoch",
+                ));
+            }
+            DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
+        },
     }
     cache.info.skip_healing = skip_healing;
 
@@ -1971,6 +1973,7 @@ mod tests {
 
     #[test]
     fn request_rejects_empty_truncated_oversized_and_wrong_version_payloads() {
+        assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
         assert!(decode_remote_scanner_request(&[]).is_err());
 
         let mut body = rmp_serde::to_vec_named(&test_request(Uuid::new_v4())).expect("request should encode");
@@ -1980,7 +1983,7 @@ mod tests {
         let oversized = vec![0_u8; NS_SCANNER_MAX_REQUEST_BODY_SIZE + 1];
         assert!(decode_remote_scanner_request(&oversized).is_err());
 
-        for version in [NS_SCANNER_PROTOCOL_VERSION - 1, NS_SCANNER_PROTOCOL_VERSION + 1] {
+        for version in [2, 4] {
             let mut wrong_version = test_request(Uuid::new_v4());
             wrong_version.version = version;
             let body = rmp_serde::to_vec_named(&wrong_version).expect("request should encode");
@@ -2885,14 +2888,15 @@ mod tests {
 
     #[tokio::test]
     async fn wrong_frame_version_and_sequence_are_rejected() {
+        assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
         let request_id = Uuid::new_v4();
         let auth = FrameAuthenticator::for_test(request_id);
         let frame = RemoteScannerFrame::progress(RemoteScannerProgress::default());
         let payload = rmp_serde::to_vec_named(&frame).expect("frame should encode");
 
         for (version, sequence, expected_error) in [
-            (NS_SCANNER_PROTOCOL_VERSION - 1, 0, "unsupported remote namespace scanner frame version"),
-            (NS_SCANNER_PROTOCOL_VERSION + 1, 0, "unsupported remote namespace scanner frame version"),
+            (2, 0, "unsupported remote namespace scanner frame version"),
+            (4, 0, "unsupported remote namespace scanner frame version"),
             (NS_SCANNER_PROTOCOL_VERSION, 1, "frame sequence is invalid"),
         ] {
             let envelope = RemoteScannerFrameEnvelope {

crates/scanner/src/scanner.rs

@@ -14,6 +14,8 @@
 
 use std::collections::BTreeMap;
 use std::future::Future;
+#[cfg(test)]
+use std::sync::Mutex as StdMutex;
 use std::sync::{Arc, LazyLock, RwLock};
 
 use crate::ScannerObjectIO;
@@ -38,8 +40,8 @@ use bytes::Bytes;
 use chrono::{DateTime, Utc};
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{
-    CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
-    emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
+    CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
+    emit_scan_cycle_complete, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
 };
 use rustfs_config::ScannerSpeed;
 #[cfg(test)]
@@ -50,9 +52,12 @@ use rustfs_config::{
 use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest as _, Sha256};
+#[cfg(test)]
+use tokio::sync::Notify;
 use tokio::sync::mpsc;
 use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
+use tokio_util::task::AbortOnDropHandle;
 use tracing::{debug, error, info, instrument, warn};
 
 use crate::storage_api::scan::{
@@ -93,6 +98,44 @@ const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
 const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
 #[cfg(test)]
 const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
+#[cfg(test)]
+type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
+#[cfg(test)]
+static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
+    LazyLock::new(|| StdMutex::new(None));
+
+#[cfg(test)]
+struct ScannerCycleStatePersistTestHookGuard;
+
+#[cfg(test)]
+impl Drop for ScannerCycleStatePersistTestHookGuard {
+    fn drop(&mut self) {
+        *SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
+    }
+}
+
+#[cfg(test)]
+fn set_scanner_cycle_state_persist_test_hook(leader_epoch: u64, reached: Arc<Notify>) -> ScannerCycleStatePersistTestHookGuard {
+    *SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((leader_epoch, reached));
+    ScannerCycleStatePersistTestHookGuard
+}
+
+#[cfg(test)]
+fn notify_scanner_cycle_state_persist_test_hook(leader_epoch: u64) {
+    let reached = SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+        .as_ref()
+        .filter(|(expected_epoch, _)| *expected_epoch == leader_epoch)
+        .map(|(_, reached)| reached.clone());
+    if let Some(reached) = reached {
+        reached.notify_one();
+    }
+}
 
 #[derive(Debug, thiserror::Error)]
 enum ScannerCycleStateError {
@@ -1691,10 +1734,10 @@ fn data_usage_persist_timeout() -> Duration {
     DataUsageCache::persistence_timeout()
 }
 
-async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
+async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
     cycle_info.current = 0;
     global_metrics().clear_current_scan_mode();
-    global_metrics().set_cycle(Some(cycle_info.clone())).await;
+    cycle_metrics_guard.finish(cycle_info.clone()).await;
 }
 
 fn encode_scanner_cycle_state(cycle_info: &CurrentCycle, leader_epoch: u64) -> Result<Vec<u8>, ScannerCycleStateError> {
@@ -2218,6 +2261,8 @@ async fn persist_scanner_cycle_state(
             return false;
         }
 
+        #[cfg(test)]
+        notify_scanner_cycle_state_persist_test_hook(leader_epoch);
         match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
             .await
         {
@@ -2340,7 +2385,6 @@ async fn persist_scanner_cycle_state(
 
                     if persisted_cycle.next >= cycle_info.next {
                         *cycle_info = persisted_cycle;
-                        global_metrics().set_cycle(Some(cycle_info.clone())).await;
                         debug!(
                             target: "rustfs::scanner",
                             event = EVENT_SCANNER_PERSIST_STATE,
@@ -2406,6 +2450,7 @@ async fn finalize_partial_scan_cycle(
     cycle_info: &mut CurrentCycle,
     revision: &mut DataUsageCacheRevision,
     leader_epoch: u64,
+    cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
 ) -> bool {
     // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
     // must still advance (and persist) because per-bucket next_cycle is stamped from
@@ -2422,11 +2467,14 @@ async fn finalize_partial_scan_cycle(
             error = %err,
             "Scanner partial cycle could not advance"
         );
-        mark_scan_cycle_idle(cycle_info).await;
+        mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await;
         return false;
     }
-    mark_scan_cycle_idle(cycle_info).await;
-    persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await
+    cycle_info.current = 0;
+    global_metrics().clear_current_scan_mode();
+    let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
+    cycle_metrics_guard.finish(cycle_info.clone()).await;
+    persisted
 }
 
 async fn persist_required_scanner_cycle_floor(
@@ -2436,6 +2484,7 @@ async fn persist_required_scanner_cycle_floor(
     revision: &mut DataUsageCacheRevision,
     leader_epoch: u64,
     required_cycle: u64,
+    cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
 ) -> bool {
     if required_cycle <= cycle_info.current || required_cycle == u64::MAX {
         error!(
@@ -2448,13 +2497,16 @@ async fn persist_required_scanner_cycle_floor(
             state = "invalid_cache_cycle_floor",
             "Scanner cache cycle floor is invalid"
         );
-        mark_scan_cycle_idle(cycle_info).await;
+        mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await;
         return false;
     }
 
     cycle_info.next = cycle_info.next.max(required_cycle);
-    mark_scan_cycle_idle(cycle_info).await;
-    persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await
+    cycle_info.current = 0;
+    global_metrics().clear_current_scan_mode();
+    let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
+    cycle_metrics_guard.finish(cycle_info.clone()).await;
+    persisted
 }
 
 async fn await_scanner_cycle_with_lock_fence<Cycle, LockLost>(
@@ -2513,7 +2565,7 @@ async fn run_data_scanner_cycle(
     let now = Instant::now();
     cycle_info.started = Utc::now();
 
-    global_metrics().set_cycle(Some(cycle_info.clone())).await;
+    let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
 
     let mut background_heal_info = read_background_heal_info(storeapi.clone()).await;
 
@@ -2564,14 +2616,14 @@ async fn run_data_scanner_cycle(
                 "Scanner cycle could not capture the data usage persistence baseline"
             );
             emit_scan_cycle_complete(false, cycle_start.elapsed());
-            mark_scan_cycle_idle(cycle_info).await;
+            mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
             return ScannerCycleOutcome::Failed;
         }
     };
     let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
     let storeapi_clone = storeapi.clone();
     let ctx_clone = ctx.clone();
-    let mut usage_persist_task = tokio::spawn(async move {
+    let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
         store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
             ctx_clone,
             storeapi_clone,
@@ -2580,10 +2632,9 @@ async fn run_data_scanner_cycle(
             Some(usage_persist_baseline),
         )
         .await
-    });
+    }));
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
-    let cycle_work_start = global_metrics().start_scan_cycle_work();
     let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
     let scan_result = storeapi
         .clone()
@@ -2640,7 +2691,6 @@ async fn run_data_scanner_cycle(
         }
     };
     let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
-    global_metrics().finish_scan_cycle_work(cycle_work_start);
 
     let scan_cycle_result = match scan_result {
         Ok(result) => result,
@@ -2663,7 +2713,7 @@ async fn run_data_scanner_cycle(
             {
                 save_background_heal_info(storeapi.clone(), new_heal_info).await;
             }
-            mark_scan_cycle_idle(cycle_info).await;
+            mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
             return ScannerCycleOutcome::Failed;
         }
     };
@@ -2678,7 +2728,7 @@ async fn run_data_scanner_cycle(
             "Scanner cycle stopped before committing cycle state"
         );
         emit_scan_cycle_complete(false, cycle_start.elapsed());
-        mark_scan_cycle_idle(cycle_info).await;
+        mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         return ScannerCycleOutcome::Failed;
     }
     if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
@@ -2700,6 +2750,7 @@ async fn run_data_scanner_cycle(
             cycle_revision,
             leader_epoch,
             required_cycle,
+            &mut cycle_metrics_guard,
         )
         .await
         {
@@ -2719,7 +2770,7 @@ async fn run_data_scanner_cycle(
             "Scanner cycle completed without a durable data usage snapshot"
         );
         emit_scan_cycle_complete(false, cycle_start.elapsed());
-        mark_scan_cycle_idle(cycle_info).await;
+        mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         return ScannerCycleOutcome::Failed;
     }
     if budget_elapsed {
@@ -2743,7 +2794,16 @@ async fn run_data_scanner_cycle(
             scan_cycle_partial_reason(budget_reason),
             scan_cycle_partial_source(budget_reason),
         );
-        return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
+        return if finalize_partial_scan_cycle(
+            ctx,
+            storeapi.clone(),
+            cycle_info,
+            cycle_revision,
+            leader_epoch,
+            &mut cycle_metrics_guard,
+        )
+        .await
+        {
             ScannerCycleOutcome::Partial
         } else {
             ScannerCycleOutcome::Failed
@@ -2792,7 +2852,7 @@ async fn run_data_scanner_cycle(
                 "Scanner cycle completed without a durable data usage snapshot"
             );
             emit_scan_cycle_complete(false, cycle_start.elapsed());
-            mark_scan_cycle_idle(cycle_info).await;
+            mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
             return ScannerCycleOutcome::Failed;
         }
         ScannerCycleOutcome::Partial => {
@@ -2818,7 +2878,16 @@ async fn run_data_scanner_cycle(
                 );
             }
             emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
-            return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
+            return if finalize_partial_scan_cycle(
+                ctx,
+                storeapi.clone(),
+                cycle_info,
+                cycle_revision,
+                leader_epoch,
+                &mut cycle_metrics_guard,
+            )
+            .await
+            {
                 ScannerCycleOutcome::Partial
             } else {
                 ScannerCycleOutcome::Failed
@@ -2834,7 +2903,16 @@ async fn run_data_scanner_cycle(
                 state = "superseded",
                 "Scanner cycle usage snapshot was superseded by concurrent namespace activity"
             );
-            if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
+            if finalize_partial_scan_cycle(
+                ctx,
+                storeapi.clone(),
+                cycle_info,
+                cycle_revision,
+                leader_epoch,
+                &mut cycle_metrics_guard,
+            )
+            .await
+            {
                 emit_scan_cycle_superseded(cycle_start.elapsed());
                 return ScannerCycleOutcome::Superseded;
             }
@@ -2853,7 +2931,7 @@ async fn run_data_scanner_cycle(
             error = %err,
             "Scanner completed cycle could not advance"
         );
-        mark_scan_cycle_idle(cycle_info).await;
+        mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         emit_scan_cycle_complete(false, cycle_start.elapsed());
         return ScannerCycleOutcome::Failed;
     }
@@ -2862,9 +2940,8 @@ async fn run_data_scanner_cycle(
     global_metrics().clear_current_scan_mode();
 
     retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
-    global_metrics().set_cycle(Some(cycle_info.clone())).await;
     if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
-        mark_scan_cycle_idle(cycle_info).await;
+        cycle_metrics_guard.finish(cycle_info.clone()).await;
         emit_scan_cycle_complete(false, cycle_start.elapsed());
         return ScannerCycleOutcome::Failed;
     }
@@ -2888,9 +2965,37 @@ async fn run_data_scanner_cycle(
         "Scanner cycle completed"
     );
 
+    cycle_metrics_guard.finish(cycle_info.clone()).await;
     scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending_maintenance_work)
 }
 
+struct ScannerCycleMetricsGuard {
+    start: Option<ScanCycleWorkSnapshot>,
+}
+
+impl ScannerCycleMetricsGuard {
+    async fn new(cycle: CurrentCycle) -> Self {
+        Self {
+            start: Some(global_metrics().start_scan_cycle_work_with_cycle(cycle).await),
+        }
+    }
+
+    async fn finish(&mut self, cycle: CurrentCycle) {
+        if let Some(start) = self.start {
+            global_metrics().finish_scan_cycle_work_with_cycle(start, cycle).await;
+            self.start = None;
+        }
+    }
+}
+
+impl Drop for ScannerCycleMetricsGuard {
+    fn drop(&mut self) {
+        if let Some(start) = self.start.take() {
+            global_metrics().finish_scan_cycle_work(start);
+        }
+    }
+}
+
 async fn record_scanner_leader_lock_lost(message: &'static str) {
     reset_scanner_cycle_schedule();
     record_scanner_leader_lock_state("lost");
@@ -3443,7 +3548,7 @@ enum DataUsagePersistTaskResult {
 
 async fn wait_for_data_usage_persist_task(
     ctx: &CancellationToken,
-    task: &mut tokio::task::JoinHandle<DataUsagePersistOutcome>,
+    task: &mut AbortOnDropHandle<DataUsagePersistOutcome>,
     timeout: Duration,
 ) -> DataUsagePersistTaskResult {
     tokio::select! {
@@ -3840,8 +3945,9 @@ mod tests {
     use super::*;
     use crate::EcstoreResult;
     use crate::{
-        ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
-        ScannerPutObjReader as PutObjReader,
+        Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader,
+        ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
+        init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
     };
     use serial_test::serial;
     use std::collections::HashMap;
@@ -3853,6 +3959,47 @@ mod tests {
 
     const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
 
+    async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
+        init_ecstore_config_for_scanner_tests();
+        let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
+        let mut endpoints = Vec::new();
+        for disk_index in 0..4 {
+            let disk_path = temp_dir.path().join(format!("disk{disk_index}"));
+            tokio::fs::create_dir_all(&disk_path)
+                .await
+                .expect("scanner cycle test disk should be created");
+            let mut endpoint =
+                Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
+            endpoint.set_pool_index(0);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            endpoints.push(endpoint);
+        }
+        let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 4,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: "scanner-cycle-metrics".to_string(),
+            platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+        }]);
+        let instance_ctx = Arc::new(InstanceContext::new());
+        init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+            .await
+            .expect("scanner cycle test disks should initialize");
+        let store = ECStore::new_with_instance_ctx(
+            "127.0.0.1:0".parse().expect("test address should parse"),
+            endpoint_pools,
+            CancellationToken::new(),
+            instance_ctx,
+        )
+        .await
+        .expect("scanner cycle test ECStore should initialize");
+        init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
+
+        (temp_dir, store)
+    }
+
     fn assert_run_data_scanner_signature<F, Fut>(_run: F)
     where
         F: Fn(CancellationToken, Arc<ECStore>) -> Fut,
@@ -4312,9 +4459,9 @@ mod tests {
         };
 
         global_metrics().set_current_scan_mode(HealScanMode::Deep);
-        global_metrics().set_cycle(Some(cycle_info.clone())).await;
+        let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
 
-        mark_scan_cycle_idle(&mut cycle_info).await;
+        mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await;
 
         let published = global_metrics()
             .get_cycle()
@@ -4330,6 +4477,123 @@ mod tests {
         global_metrics().set_cycle(None).await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
+        let cycle_started = Utc::now() - chrono::Duration::seconds(5);
+        let mut cycle_info = CurrentCycle {
+            current: 0,
+            next: 1,
+            started: cycle_started,
+            ..Default::default()
+        };
+        let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
+        let setup_report = global_metrics().report().await;
+        assert!(setup_report.current_cycle_active);
+        assert_eq!(setup_report.current_cycle, 0);
+        assert_eq!(setup_report.current_started, cycle_started);
+
+        mark_scan_cycle_idle(&mut cycle_info, &mut guard).await;
+        let idle_report = global_metrics().report().await;
+        assert!(!idle_report.current_cycle_active);
+
+        global_metrics().set_cycle(None).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
+        let mut cycle_info = CurrentCycle {
+            current: 12,
+            next: 13,
+            started: Utc::now(),
+            ..Default::default()
+        };
+        let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
+
+        cycle_info.current = 0;
+        tokio::task::yield_now().await;
+        let finalizing_report = global_metrics().report().await;
+        assert!(finalizing_report.current_cycle_active);
+        assert_eq!(finalizing_report.current_cycle, 12);
+
+        guard.finish(cycle_info).await;
+        let idle_report = global_metrics().report().await;
+        assert!(!idle_report.current_cycle_active);
+        assert_eq!(idle_report.current_cycle, 0);
+
+        global_metrics().set_cycle(None).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn scanner_cycle_metrics_guard_drop_clears_activity() {
+        let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
+            current: 12,
+            next: 13,
+            started: Utc::now(),
+            ..Default::default()
+        })
+        .await;
+        assert!(global_metrics().report().await.current_cycle_active);
+
+        drop(guard);
+
+        assert!(!global_metrics().report().await.current_cycle_active);
+        global_metrics().set_cycle(None).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
+        let (_temp_dir, store) = setup_scanner_cycle_store().await;
+        let ctx = CancellationToken::new();
+        let mut cycle_info = CurrentCycle::default();
+        let mut revision = DataUsageCacheRevision::Missing;
+        let leader_epoch = u64::MAX - 1;
+        let state_persist_reached = Arc::new(Notify::new());
+        let _state_persist_hook = set_scanner_cycle_state_persist_test_hook(leader_epoch, state_persist_reached.clone());
+        let state_lock = store
+            .new_ns_lock(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str())
+            .await
+            .expect("scanner cycle state lock should be created");
+        let state_guard = state_lock
+            .get_write_lock(Duration::from_secs(1))
+            .await
+            .expect("scanner cycle state lock should be acquired");
+        let mut cycle = Box::pin(run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch));
+        let waker = std::task::Waker::noop();
+        let mut context = std::task::Context::from_waker(waker);
+
+        assert!(cycle.as_mut().poll(&mut context).is_pending());
+        let active = global_metrics().report().await;
+        assert!(active.current_cycle_active);
+        assert_eq!(active.current_cycle, 0);
+
+        tokio::time::timeout(Duration::from_secs(30), async {
+            tokio::select! {
+                outcome = &mut cycle => panic!("scanner cycle finished before state persistence was released: {outcome:?}"),
+                _ = state_persist_reached.notified() => {}
+            }
+        })
+        .await
+        .expect("scanner cycle should reach state persistence");
+        let finalizing = global_metrics().report().await;
+        assert!(finalizing.current_cycle_active);
+
+        drop(state_guard);
+        let outcome = tokio::time::timeout(Duration::from_secs(30), cycle)
+            .await
+            .expect("scanner cycle should finish");
+        assert!(matches!(
+            outcome,
+            ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
+        ));
+        assert!(!global_metrics().report().await.current_cycle_active);
+
+        global_metrics().set_cycle(None).await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
@@ -4342,8 +4606,11 @@ mod tests {
             cycle_completed: vec![],
             started: Utc::now(),
         };
+        let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
 
-        assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1).await);
+        assert!(
+            finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await
+        );
 
         assert_eq!(cycle_info.next, 13);
         assert_eq!(cycle_info.current, 0);
@@ -4377,8 +4644,20 @@ mod tests {
             cycle_completed: vec![],
             started: Utc::now(),
         };
+        let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
 
-        assert!(persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 19).await);
+        assert!(
+            persist_required_scanner_cycle_floor(
+                &ctx,
+                store.clone(),
+                &mut cycle_info,
+                &mut revision,
+                7,
+                19,
+                &mut cycle_metrics_guard,
+            )
+            .await
+        );
         assert_eq!(cycle_info.current, 0);
         assert_eq!(cycle_info.next, 19);
 
@@ -4405,22 +4684,37 @@ mod tests {
             cycle_completed: vec![],
             started: Utc::now(),
         };
+        let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
 
-        assert!(!persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 12).await);
-        assert_eq!(cycle_info.next, 12);
-        assert_eq!(revision, DataUsageCacheRevision::Missing);
         assert!(
             !persist_required_scanner_cycle_floor(
                 &ctx,
                 store.clone(),
-                &mut CurrentCycle {
-                    current: 12,
-                    next: 12,
-                    ..Default::default()
-                },
+                &mut cycle_info,
+                &mut revision,
+                7,
+                12,
+                &mut cycle_metrics_guard,
+            )
+            .await
+        );
+        assert_eq!(cycle_info.next, 12);
+        assert_eq!(revision, DataUsageCacheRevision::Missing);
+        let mut max_cycle_info = CurrentCycle {
+            current: 12,
+            next: 12,
+            ..Default::default()
+        };
+        let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await;
+        assert!(
+            !persist_required_scanner_cycle_floor(
+                &ctx,
+                store.clone(),
+                &mut max_cycle_info,
                 &mut revision,
                 7,
                 u64::MAX,
+                &mut max_cycle_metrics_guard,
             )
             .await
         );
@@ -4685,8 +4979,9 @@ mod tests {
             cycle_completed: vec![],
             started: Utc::now(),
         };
+        let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
 
-        assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1).await);
+        assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await);
         assert_eq!(cycle_info.next, 13);
         assert_eq!(cycle_info.current, 0);
         assert_eq!(revision, DataUsageCacheRevision::Missing);
@@ -5943,10 +6238,10 @@ mod tests {
     #[tokio::test]
     async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() {
         let ctx = CancellationToken::new();
-        let mut task = tokio::spawn(async {
+        let mut task = AbortOnDropHandle::new(tokio::spawn(async {
             std::future::pending::<()>().await;
             DataUsagePersistOutcome::Saved
-        });
+        }));
         ctx.cancel();
 
         let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await;
@@ -5958,10 +6253,10 @@ mod tests {
     #[tokio::test(start_paused = true)]
     async fn data_usage_persist_wait_aborts_after_timeout() {
         let ctx = CancellationToken::new();
-        let mut task = tokio::spawn(async {
+        let mut task = AbortOnDropHandle::new(tokio::spawn(async {
             std::future::pending::<()>().await;
             DataUsagePersistOutcome::Saved
-        });
+        }));
 
         let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await;
 

crates/scanner/src/scanner_folder.rs

@@ -1508,6 +1508,9 @@ impl FolderScanner {
             if entry.children.contains(&child) {
                 continue;
             }
+            if !self.old_cache.cache.contains_key(&child) {
+                continue;
+            }
 
             let child_hash = DataUsageHash(child.clone());
             self.new_cache
@@ -2320,7 +2323,6 @@ impl FolderScanner {
                     }
                     tokio::task::yield_now().await;
 
-                    let h = DataUsageHash(folder_item.name.clone());
                     into.add_child(&h);
                     self.record_scan_resume_hint(&folder_item.name);
                     // We scanned a folder, optionally send update.
@@ -2646,7 +2648,7 @@ impl FolderScanner {
                         tokio::task::yield_now().await;
                     } else {
                         let mut dst = DataUsageEntry::default();
-                        let h = DataUsageHash(folder_item.name.clone());
+                        let h = hash_path(&folder_item.name);
 
                         // Use Box::pin for recursive async call
                         let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
@@ -2911,7 +2913,7 @@ mod tests {
     use serial_test::serial;
     #[cfg(unix)]
     use std::os::unix::fs::{PermissionsExt, symlink};
-    use std::sync::atomic::AtomicBool;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
     use temp_env::{with_var, with_var_unset};
     use uuid::Uuid;
 
@@ -3864,11 +3866,15 @@ mod tests {
     }
 
     async fn write_test_object_metadata(root: &std::path::Path, bucket: &str, object: &str) {
+        write_test_object_metadata_bytes(root, bucket, object, &metadata_for_object(bucket, object)).await;
+    }
+
+    async fn write_test_object_metadata_bytes(root: &std::path::Path, bucket: &str, object: &str, metadata: &[u8]) {
         let object_dir = root.join(bucket).join(object);
         tokio::fs::create_dir_all(&object_dir)
             .await
             .expect("failed to create test object directory");
-        tokio::fs::write(object_dir.join("xl.meta"), metadata_for_object(bucket, object))
+        tokio::fs::write(object_dir.join("xl.meta"), metadata)
             .await
             .expect("failed to write test object metadata");
     }
@@ -4165,27 +4171,28 @@ mod tests {
     async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
         let (mut scanner, temp_dir) = build_test_scanner().await;
         let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
-        let _heal_responder = rustfs_common::heal_channel::init_heal_channel().ok().map(|mut heal_rx| {
-            tokio::spawn(async move {
-                while let Some(command) = heal_rx.recv().await {
-                    if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
-                        let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
-                    }
+        let heal_starts = Arc::new(AtomicUsize::new(0));
+        let heal_starts_clone = heal_starts.clone();
+        let mut heal_rx =
+            rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
+        let _heal_responder = tokio::spawn(async move {
+            while let Some(command) = heal_rx.recv().await {
+                if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
+                    heal_starts_clone.fetch_add(1, Ordering::Relaxed);
+                    let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
                 }
-            })
+            }
         });
 
         let bucket = "src-archive";
-        tokio::fs::create_dir_all(temp_dir.join(bucket))
-            .await
-            .expect("failed to create bucket directory");
+        let object = "snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4";
+        let metadata = metadata_for_object(bucket, object);
+        write_test_object_metadata_bytes(&temp_dir, bucket, object, &metadata).await;
 
         let mut disks = vec![scanner.local_disk.clone()];
         for disk_name in ["disk2", "disk3", "disk4"] {
             let disk_root = temp_dir.join(disk_name);
-            tokio::fs::create_dir_all(disk_root.join(bucket))
-                .await
-                .expect("failed to create extra disk bucket directory");
+            write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
             let endpoint =
                 Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint");
             let disk = new_disk(
@@ -4204,7 +4211,7 @@ mod tests {
         scanner.disks = disks;
         scanner.disks_quorum = 2;
         scanner.old_cache.replace(
-            "src-archive/snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4",
+            &format!("{bucket}/{object}"),
             bucket,
             DataUsageEntry {
                 objects: 1,
@@ -4219,13 +4226,17 @@ mod tests {
             object_heal_prob_div: 1,
         };
 
-        tokio::time::timeout(
-            Duration::from_millis(200),
-            scanner.scan_folder(CancellationToken::new(), folder, &mut into),
-        )
-        .await
-        .expect("scan_folder should not hang after list_path_raw finishes")
-        .expect("scan_folder should finish successfully");
+        tokio::time::timeout(Duration::from_secs(2), scanner.scan_folder(CancellationToken::new(), folder, &mut into))
+            .await
+            .expect("scan_folder should not hang after list_path_raw finishes")
+            .expect("scan_folder should finish successfully");
+
+        let root = scanner
+            .new_cache
+            .checked_flatten(bucket)
+            .expect("healed cache must contain canonical child links");
+        assert_eq!(root.objects, 1);
+        assert!(heal_starts.load(Ordering::Relaxed) > 0, "test must execute the heal child-link path");
     }
 
     #[tokio::test]
@@ -4774,7 +4785,7 @@ mod tests {
         .expect("unbounded scan should finish after partial progress");
 
         let root = result
-            .size_recursive("bucket")
+            .checked_flatten("bucket")
             .expect("completed cache should retain bucket usage");
         assert_eq!(root.objects, 5);
         assert!(result.info.snapshot_complete);
@@ -4828,6 +4839,106 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn test_partial_entry_does_not_carry_missing_old_child() {
+        let (mut scanner, temp_dir) = build_test_scanner().await;
+        let _guard = TestGuard {
+            temp_dir: Some(temp_dir),
+        };
+        let root_hash = hash_path("bucket");
+        scanner.old_cache.cache.insert(
+            root_hash.key(),
+            DataUsageEntry {
+                children: HashSet::from([hash_path("bucket/missing").key()]),
+                ..Default::default()
+            },
+        );
+
+        let mut partial = DataUsageEntry {
+            objects: 2,
+            size: 2,
+            ..Default::default()
+        };
+        scanner.carry_forward_old_children(&root_hash, &mut partial);
+        scanner.new_cache.replace_hashed(&root_hash, &None, &partial);
+
+        assert!(partial.children.is_empty());
+        let flattened = scanner
+            .new_cache
+            .checked_flatten("bucket")
+            .expect("a partial cache must not retain dangling child links");
+        assert_eq!(flattened.objects, 2);
+        assert_eq!(flattened.size, 2);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
+        let (scanner, temp_dir) = build_test_scanner().await;
+        let _guard = TestGuard {
+            temp_dir: Some(temp_dir.clone()),
+        };
+        write_test_object_metadata(&temp_dir, "bucket", "prefix/object").await;
+
+        let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
+        let scan_plan_digest = crate::data_usage_define::DataUsageScanPlanDigest([9; 32]);
+        let mut legacy = DataUsageCache {
+            info: crate::data_usage_define::DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 7,
+                source: Some(source),
+                scan_plan_digest: Some(scan_plan_digest),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        legacy.cache.insert(
+            "bucket".to_string(),
+            DataUsageEntry {
+                children: HashSet::from(["bucket\\prefix".to_string()]),
+                ..Default::default()
+            },
+        );
+        legacy.cache.insert(
+            "bucket\\prefix".to_string(),
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        let encoded = legacy.marshal_msg().expect("legacy cache should serialize");
+        let mut migrated = DataUsageCache::unmarshal(&encoded).expect("legacy cache should deserialize");
+        assert_eq!(
+            migrated.prepare_for_scan("bucket", 8, 0, source, scan_plan_digest, true),
+            crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
+        );
+
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new(&parent, Default::default());
+        let rebuilt = scan_data_folder(
+            budget.token(),
+            budget,
+            vec![scanner.local_disk.clone()],
+            scanner.local_disk,
+            migrated,
+            None,
+            HealScanMode::Normal,
+            SCANNER_SLEEPER.clone(),
+        )
+        .await
+        .expect("portable cache rebuild should complete");
+        let persisted = rebuilt.marshal_msg().expect("rebuilt cache should serialize");
+        let decoded = DataUsageCache::unmarshal(&persisted).expect("rebuilt cache should deserialize");
+
+        assert_eq!(decoded.info.cache_key_format, crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT);
+        assert!(decoded.cache.keys().all(|key| !key.contains('\\')));
+        let root = decoded
+            .checked_flatten("bucket")
+            .expect("rebuilt persisted cache should have a complete root");
+        assert_eq!(root.objects, 1);
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_scan_data_folder_success_clears_resume_hint() {

crates/scanner/src/scanner_io.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
 use crate::scanner_budget::ScannerCycleBudget;
 use crate::scanner_folder::{ScannerItem, scan_data_folder};
 use crate::sleeper::SCANNER_SLEEPER;
@@ -830,7 +831,7 @@ pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Resu
         return Err(ScannerError::Other("scanner cache root name is empty".to_string()));
     }
     let entry = cache
-        .checked_flatten(&cache.info.name)
+        .checked_flatten_complete_scope(&cache.info.name)
         .ok_or_else(|| ScannerError::Other(format!("scanner cache root is missing or corrupt: {}", cache.info.name)))?;
 
     Ok(DataUsageEntryInfo {
@@ -852,7 +853,7 @@ fn should_publish_completed_snapshot(completed_count: usize, total_count: usize,
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum NamespaceScannerWorkerMode {
     Coordinator,
-    RemoteV3(uuid::Uuid),
+    RemoteV4(uuid::Uuid),
 }
 
 fn namespace_scanner_workers<T>(
@@ -868,7 +869,7 @@ fn namespace_scanner_workers<T>(
     workers.extend(
         remote_disks
             .into_iter()
-            .map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV3(server_epoch))),
+            .map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV4(server_epoch))),
     );
     workers
 }
@@ -958,7 +959,57 @@ where
     let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await;
 }
 
-pub(crate) fn cache_snapshot_is_current(
+pub(crate) fn current_cache_root_entry(
+    cache: &DataUsageCache,
+    name: &str,
+    source: DataUsageCacheSource,
+    next_cycle: u64,
+    leader_epoch: u64,
+    scan_plan_digest: DataUsageScanPlanDigest,
+) -> std::result::Result<Option<DataUsageEntryInfo>, ScannerError> {
+    let metadata_is_current = cache.info.name == name
+        && cache.info.source == Some(source)
+        && cache.info.snapshot_complete
+        && cache.info.scan_plan_digest == Some(scan_plan_digest)
+        && cache.info.last_update.is_some()
+        && cache.info.next_cycle == next_cycle
+        && cache.info.leader_epoch == leader_epoch
+        && cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
+    if !metadata_is_current {
+        return Ok(None);
+    }
+
+    cache_root_entry_info(cache).map(Some)
+}
+
+pub(crate) enum DataUsageCacheScanState {
+    Current(Box<DataUsageEntryInfo>),
+    Prepared {
+        outcome: DataUsageCachePrepareOutcome,
+        invalid_current: Option<ScannerError>,
+    },
+}
+
+pub(crate) fn current_cache_root_or_prepare(
+    cache: &mut DataUsageCache,
+    name: &str,
+    source: DataUsageCacheSource,
+    next_cycle: u64,
+    leader_epoch: u64,
+    scan_plan_digest: DataUsageScanPlanDigest,
+    require_source: bool,
+) -> DataUsageCacheScanState {
+    match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) {
+        Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
+        current => DataUsageCacheScanState::Prepared {
+            invalid_current: current.err(),
+            outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source),
+        },
+    }
+}
+
+#[cfg(test)]
+fn cache_snapshot_is_current(
     cache: &DataUsageCache,
     name: &str,
     source: DataUsageCacheSource,
@@ -966,13 +1017,10 @@ pub(crate) fn cache_snapshot_is_current(
     leader_epoch: u64,
     scan_plan_digest: DataUsageScanPlanDigest,
 ) -> bool {
-    cache.info.name == name
-        && cache.info.source == Some(source)
-        && cache.info.snapshot_complete
-        && cache.info.scan_plan_digest == Some(scan_plan_digest)
-        && cache.info.last_update.is_some()
-        && cache.info.next_cycle == next_cycle
-        && cache.info.leader_epoch == leader_epoch
+    matches!(
+        current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest),
+        Ok(Some(_))
+    )
 }
 
 fn completed_data_usage_info(
@@ -1061,6 +1109,7 @@ mod publish_gate_tests {
                 source: Some(source),
                 snapshot_complete: false,
                 scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                 ..Default::default()
             },
             ..Default::default()
@@ -1102,6 +1151,7 @@ mod publish_gate_tests {
                 source: Some(source),
                 snapshot_complete: true,
                 scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                 ..Default::default()
             },
             ..Default::default()
@@ -1423,17 +1473,145 @@ mod publish_gate_tests {
         assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
     }
 
+    #[test]
+    fn current_cache_snapshot_rejects_persisted_windows_key_mismatch() {
+        let source = DataUsageCacheSource::new(1, 2);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 10,
+                last_update: Some(SystemTime::UNIX_EPOCH),
+                source: Some(source),
+                snapshot_complete: true,
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.cache.insert(
+            "bucket".to_string(),
+            DataUsageEntry {
+                children: HashSet::from(["bucket/prefix".to_string()]),
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            "bucket\\prefix".to_string(),
+            DataUsageEntry {
+                objects: 3,
+                ..Default::default()
+            },
+        );
+
+        assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
+        match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
+            DataUsageCacheScanState::Prepared {
+                outcome: DataUsageCachePrepareOutcome::Reset,
+                invalid_current: Some(_),
+            } => {}
+            _ => panic!("an invalid current cache must enter the rebuild path"),
+        }
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
+    #[test]
+    fn current_cache_snapshot_rejects_structurally_valid_legacy_key_format() {
+        let source = DataUsageCacheSource::new(1, 2);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 10,
+                last_update: Some(SystemTime::UNIX_EPOCH),
+                source: Some(source),
+                snapshot_complete: true,
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.cache.insert(
+            "bucket".to_string(),
+            DataUsageEntry {
+                children: HashSet::from(["bucket\\prefix".to_string()]),
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            "bucket\\prefix".to_string(),
+            DataUsageEntry {
+                objects: 3,
+                ..Default::default()
+            },
+        );
+        assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(3));
+
+        match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
+            DataUsageCacheScanState::Prepared {
+                outcome: DataUsageCachePrepareOutcome::Reset,
+                invalid_current: None,
+            } => {}
+            _ => panic!("a legacy key format must enter the rebuild path"),
+        }
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
+    #[test]
+    fn current_cache_snapshot_rejects_current_bucket_cache_with_detached_entry() {
+        let source = DataUsageCacheSource::new(1, 2);
+        let mut cache = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                next_cycle: 10,
+                last_update: Some(SystemTime::UNIX_EPOCH),
+                source: Some(source),
+                snapshot_complete: true,
+                scan_plan_digest: Some(TEST_PLAN_DIGEST),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        cache.cache.insert(
+            "bucket".to_string(),
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        cache.cache.insert(
+            "bucket/detached".to_string(),
+            DataUsageEntry {
+                objects: 2,
+                ..Default::default()
+            },
+        );
+        assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
+
+        match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
+            DataUsageCacheScanState::Prepared {
+                outcome: DataUsageCachePrepareOutcome::Reset,
+                invalid_current: Some(_),
+            } => {}
+            _ => panic!("a detached complete bucket cache must enter the rebuild path"),
+        }
+        assert!(cache.cache.is_empty());
+        assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
+    }
+
     #[test]
     fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() {
         let server_epoch = uuid::Uuid::new_v4();
-        let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v3", server_epoch)]);
+        let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v4", server_epoch)]);
 
         assert_eq!(
             workers,
             vec![
                 ("local", NamespaceScannerWorkerMode::Coordinator),
                 ("legacy-remote", NamespaceScannerWorkerMode::Coordinator),
-                ("v3", NamespaceScannerWorkerMode::RemoteV3(server_epoch)),
+                ("v4", NamespaceScannerWorkerMode::RemoteV4(server_epoch)),
             ]
         );
         assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty());
@@ -1507,6 +1685,7 @@ mod publish_gate_tests {
                 source: Some(source),
                 snapshot_complete: true,
                 scan_plan_digest: Some(first),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                 ..Default::default()
             },
             ..Default::default()
@@ -1595,6 +1774,15 @@ async fn send_cache_root_entry_info(
     pending_maintenance_work: &AtomicBool,
 ) -> std::result::Result<(), ScannerError> {
     let root = cache_root_entry_info(cache)?;
+    send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await
+}
+
+async fn send_cache_root_entry(
+    bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
+    root: DataUsageEntryInfo,
+    cache: &DataUsageCache,
+    pending_maintenance_work: &AtomicBool,
+) -> std::result::Result<(), ScannerError> {
     record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
     bucket_result_tx
         .send(root)
@@ -1690,13 +1878,16 @@ async fn persist_and_publish_cache_snapshot(
         );
         return None;
     }
-    if cache_snapshot_is_current(
-        &persisted,
-        DATA_USAGE_ROOT,
-        source,
-        cache_snapshot.info.next_cycle,
-        cache_snapshot.info.leader_epoch,
-        scan_plan_digest,
+    if matches!(
+        current_cache_root_entry(
+            &persisted,
+            DATA_USAGE_ROOT,
+            source,
+            cache_snapshot.info.next_cycle,
+            cache_snapshot.info.leader_epoch,
+            scan_plan_digest,
+        ),
+        Ok(Some(_))
     ) {
         cache_snapshot = persisted;
     } else {
@@ -2329,6 +2520,7 @@ impl ScannerIOCache for SetDisks {
                     source: Some(source),
                     snapshot_complete: true,
                     scan_plan_digest: Some(scan_plan_digest),
+                    cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                     ..Default::default()
                 },
                 cache: HashMap::new(),
@@ -2450,7 +2642,7 @@ impl ScannerIOCache for SetDisks {
                 subsystem = LOG_SUBSYSTEM_IO,
                 pool = self.pool_index,
                 set = self.set_index,
-                v3_disks = remote_disk_count,
+                v4_disks = remote_disk_count,
                 unsupported_remote_disks,
                 state = "unsupported_remote_disks_using_coordinator",
                 "Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers"
@@ -2547,6 +2739,7 @@ impl ScannerIOCache for SetDisks {
                 source: Some(source),
                 snapshot_complete: false,
                 scan_plan_digest: Some(scan_plan_digest),
+                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                 ..Default::default()
             },
             cache: HashMap::new(),
@@ -2638,7 +2831,7 @@ impl ScannerIOCache for SetDisks {
             let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
             let cache_cycle_floor_clone = cache_cycle_floor.clone();
             let remote_server_epoch = match worker_mode {
-                NamespaceScannerWorkerMode::RemoteV3(server_epoch) => Some(server_epoch),
+                NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
                 NamespaceScannerWorkerMode::Coordinator => None,
             };
             futs.push(tokio::spawn(async move {
@@ -2948,48 +3141,71 @@ impl ScannerIOCache for SetDisks {
                             continue;
                         }
                     };
-                    if cache_snapshot_is_current(&cache, &bucket.name, source, want_cycle, leader_epoch, bucket_scan_plan_digest)
-                    {
-                        if cache_guard.is_lock_lost() {
-                            record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
-                            error!(
-                                target: "rustfs::scanner::io",
-                                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                                component = LOG_COMPONENT_SCANNER,
-                                subsystem = LOG_SUBSYSTEM_IO,
-                                bucket = %bucket.name,
-                                cache_name = %cache_name,
-                                state = "lock_lost_before_reuse",
-                                "Current scanner bucket cache root publish skipped after lock loss"
-                            );
-                            continue;
-                        }
-                        if let Err(e) =
-                            send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await
-                        {
-                            record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
-                            error!(
-                                target: "rustfs::scanner::io",
-                                event = EVENT_SCANNER_DATA_USAGE_STREAM,
-                                component = LOG_COMPONENT_SCANNER,
-                                subsystem = LOG_SUBSYSTEM_IO,
-                                bucket = %bucket.name,
-                                state = "send_current_root_failed",
-                                error = %e,
-                                "Current scanner bucket cache root entry publish failed"
-                            );
-                        }
-                        continue;
-                    }
-
-                    match cache.prepare_for_scan(
+                    let scan_state = current_cache_root_or_prepare(
+                        &mut cache,
                         &bucket.name,
+                        source,
                         want_cycle,
                         leader_epoch,
-                        source,
                         bucket_scan_plan_digest,
                         require_cache_source,
-                    ) {
+                    );
+                    let outcome = match scan_state {
+                        DataUsageCacheScanState::Current(root) => {
+                            if cache_guard.is_lock_lost() {
+                                record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
+                                error!(
+                                    target: "rustfs::scanner::io",
+                                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                                    component = LOG_COMPONENT_SCANNER,
+                                    subsystem = LOG_SUBSYSTEM_IO,
+                                    bucket = %bucket.name,
+                                    cache_name = %cache_name,
+                                    state = "lock_lost_before_reuse",
+                                    "Current scanner bucket cache root publish skipped after lock loss"
+                                );
+                                continue;
+                            }
+                            if let Err(e) =
+                                send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
+                                    .await
+                            {
+                                record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
+                                error!(
+                                    target: "rustfs::scanner::io",
+                                    event = EVENT_SCANNER_DATA_USAGE_STREAM,
+                                    component = LOG_COMPONENT_SCANNER,
+                                    subsystem = LOG_SUBSYSTEM_IO,
+                                    bucket = %bucket.name,
+                                    state = "send_current_root_failed",
+                                    error = %e,
+                                    "Current scanner bucket cache root entry publish failed"
+                                );
+                            }
+                            continue;
+                        }
+                        DataUsageCacheScanState::Prepared {
+                            outcome,
+                            invalid_current,
+                        } => {
+                            if let Some(e) = invalid_current {
+                                warn!(
+                                    target: "rustfs::scanner::io",
+                                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                                    component = LOG_COMPONENT_SCANNER,
+                                    subsystem = LOG_SUBSYSTEM_IO,
+                                    bucket = %bucket.name,
+                                    cache_name = %cache_name,
+                                    state = "current_cache_invalid",
+                                    error = %e,
+                                    "Current scanner bucket cache is invalid; rebuilding"
+                                );
+                            }
+                            outcome
+                        }
+                    };
+
+                    match outcome {
                         DataUsageCachePrepareOutcome::RejectedNewerCycle => {
                             cache_cycle_floor_clone.fetch_max(cache.info.next_cycle, Ordering::AcqRel);
                             record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
@@ -3361,6 +3577,7 @@ impl ScannerIOCache for SetDisks {
                     source: Some(source),
                     snapshot_complete: false,
                     scan_plan_digest: Some(scan_plan_digest),
+                    cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
                     ..Default::default()
                 },
                 cache: HashMap::new(),
@@ -4668,6 +4885,67 @@ mod tests {
         root.add_child(&crate::hash_path("bucket/missing"));
         dangling.replace("bucket", DATA_USAGE_ROOT, root);
         assert!(cache_root_entry_info(&dangling).is_err());
+
+        let mut detached = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: DATA_USAGE_ROOT.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        detached.replace("bucket", DATA_USAGE_ROOT, DataUsageEntry::default());
+        detached.replace(
+            "bucket/detached",
+            "",
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        assert!(cache_root_entry_info(&detached).is_err());
+
+        let mut detached_bucket = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        detached_bucket.replace(
+            "bucket",
+            DATA_USAGE_ROOT,
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        detached_bucket.replace(
+            "bucket/detached",
+            "",
+            DataUsageEntry {
+                objects: 1,
+                ..Default::default()
+            },
+        );
+        assert!(cache_root_entry_info(&detached_bucket).is_err());
+
+        let mut compacted_with_child = DataUsageCache {
+            info: DataUsageCacheInfo {
+                name: "bucket".to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        compacted_with_child.replace(
+            "bucket",
+            DATA_USAGE_ROOT,
+            DataUsageEntry {
+                compacted: true,
+                ..Default::default()
+            },
+        );
+        compacted_with_child.replace("bucket/prefix", "bucket", DataUsageEntry::default());
+        assert!(cache_root_entry_info(&compacted_with_child).is_err());
     }
 
     #[test]

crates/storage-api/src/bucket.rs

@@ -53,7 +53,7 @@ pub struct DeleteBucketOptions {
     pub no_recreate: bool,
     /// Force deletion even if bucket is not empty.
     pub force: bool,
-    /// Force deletion only after the local peer verifies the bucket is empty.
+    /// Delete empty directory remnants after the local peer verifies no object metadata exists.
     pub force_if_empty: bool,
     /// Site replication delete operation.
     pub srdelete_op: SRBucketDeleteOp,

crates/utils/Cargo.toml

@@ -58,11 +58,16 @@ url = { workspace = true, optional = true }
 zstd = { workspace = true, optional = true }
 
 [dev-dependencies]
+criterion = { workspace = true, features = ["html_reports"] }
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt"] }
 temp-env = { workspace = true }
 proptest = "1"
 
+[[bench]]
+name = "hash_hotpath_benchmark"
+harness = false
+
 [target.'cfg(windows)'.dependencies]
 windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem"] }
 

crates/utils/benches/hash_hotpath_benchmark.rs

@@ -0,0 +1,55 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use rustfs_utils::HashAlgorithm;
+use std::hint::black_box;
+
+fn generate_payload(size: usize) -> Vec<u8> {
+    (0..size)
+        .map(|i| u8::try_from(i % 251).expect("modulo output fits in u8"))
+        .collect()
+}
+
+fn bench_hash_hotpaths(c: &mut Criterion) {
+    let payloads = [
+        ("64KiB", generate_payload(64 * 1024)),
+        ("1MiB", generate_payload(1024 * 1024)),
+    ];
+    let algorithms = [
+        ("md5", HashAlgorithm::Md5),
+        ("sha256", HashAlgorithm::SHA256),
+        ("highwayhash256s", HashAlgorithm::HighwayHash256S),
+        ("highwayhash256s_legacy", HashAlgorithm::HighwayHash256SLegacy),
+    ];
+
+    let mut group = c.benchmark_group("hash_hotpath");
+    for (payload_name, payload) in &payloads {
+        let payload_len = u64::try_from(payload.len()).expect("benchmark payload length fits in u64");
+        group.throughput(Throughput::Bytes(payload_len));
+        for (algo_name, algorithm) in &algorithms {
+            let algorithm = algorithm.clone();
+            group.bench_with_input(BenchmarkId::new(*algo_name, payload_name), payload.as_slice(), move |b, payload| {
+                b.iter(|| {
+                    let hash = algorithm.hash_encode(black_box(payload));
+                    black_box(hash.as_ref()[0]);
+                });
+            });
+        }
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_hash_hotpaths);
+criterion_main!(benches);

crates/utils/src/hash.rs

@@ -29,14 +29,24 @@ const LEGACY_HIGHWAY_HASH256_KEY: [u8; 32] = [
     3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
 ];
 
-fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] {
-    let mut key = [0u64; 4];
-    for (i, chunk) in bytes.chunks_exact(8).enumerate() {
-        key[i] = u64::from_le_bytes(chunk.try_into().unwrap());
-    }
-    key
+const fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] {
+    [
+        u64::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]]),
+        u64::from_le_bytes([
+            bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15],
+        ]),
+        u64::from_le_bytes([
+            bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], bytes[22], bytes[23],
+        ]),
+        u64::from_le_bytes([
+            bytes[24], bytes[25], bytes[26], bytes[27], bytes[28], bytes[29], bytes[30], bytes[31],
+        ]),
+    ]
 }
 
+const MAGIC_HIGHWAY_HASH256_PARSED_KEY: Key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
+const LEGACY_HIGHWAY_HASH256_PARSED_KEY: Key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY));
+
 #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)]
 /// Supported hash algorithms for bitrot protection.
 pub enum HashAlgorithm {
@@ -100,25 +110,23 @@ impl HashAlgorithm {
     /// # Returns
     /// A byte slice containing the hash of the input data
     ///
+    #[inline]
     pub fn hash_encode(&self, data: &[u8]) -> impl AsRef<[u8]> {
         match self {
             HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
             HashAlgorithm::HighwayHash256 => {
-                let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
-                let mut hasher = HighwayHasher::new(key);
+                let mut hasher = HighwayHasher::new(MAGIC_HIGHWAY_HASH256_PARSED_KEY);
                 hasher.append(data);
                 HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256()))
             }
             HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()),
             HashAlgorithm::HighwayHash256S => {
-                let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
-                let mut hasher = HighwayHasher::new(key);
+                let mut hasher = HighwayHasher::new(MAGIC_HIGHWAY_HASH256_PARSED_KEY);
                 hasher.append(data);
                 HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256()))
             }
             HashAlgorithm::HighwayHash256SLegacy => {
-                let key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY));
-                let mut hasher = HighwayHasher::new(key);
+                let mut hasher = HighwayHasher::new(LEGACY_HIGHWAY_HASH256_PARSED_KEY);
                 hasher.append(data);
                 HashEncoded::HighwayHash256SLegacy(u8x32_from_u64x4(hasher.finalize256()))
             }

crates/utils/src/http/metadata_compat.rs

@@ -40,6 +40,7 @@ pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
 pub const SUFFIX_TRANSITIONED_VERSION_STATE: &str = "transitioned-version-state";
 pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
 pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id";
+pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id";
 pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id";
 pub const SUFFIX_FREE_VERSION: &str = "free-version";
 pub const SUFFIX_PURGESTATUS: &str = "purgestatus";