Merge branch 'main' into reatang/fix-replication-switches

crates/ecstore/Cargo.toml

@@ -153,6 +153,10 @@ metrics = { workspace = true }
 [target.'cfg(target_os = "linux")'.dependencies]
 rustfs-uring = "0.2.1"
 
+[target.'cfg(windows)'.dependencies]
+winapi-util.workspace = true
+windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
+
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
 criterion = { workspace = true, features = ["html_reports"] }

crates/ecstore/src/api/mod.rs

@@ -67,6 +67,14 @@ pub mod bucket {
             };
         }
 
+        pub mod transition_transaction {
+            pub use crate::bucket::lifecycle::transition_transaction::{
+                TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator,
+            };
+        }
+
         pub mod evaluator {
             pub use crate::bucket::lifecycle::evaluator::Evaluator;
         }
@@ -377,6 +385,7 @@ pub mod metrics {
 pub mod notification {
     pub use crate::services::notification_sys::{
         NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
+        start_remote_version_state_fleet_probe,
     };
 }
 
@@ -407,13 +416,15 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
-        SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
-        TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
-        node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
-        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
-        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature,
+        AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
+        PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
+        ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
+        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
+        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -23,6 +23,7 @@ use uuid::Uuid;
 use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
+    delete_confirmed_transition_candidate_exact_with_lease_idempotent,
     delete_confirmed_transition_candidate_exact_with_manager_and_identity,
     delete_object_from_remote_tier_idempotent_with_manager_and_identity,
 };
@@ -616,6 +617,199 @@ pub enum TransitionTransactionRecoveryOutcome {
     Retained,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TransitionOperatorProbe {
+    Missing,
+    UnversionedPresent,
+    VersionedPresent(String),
+    Ambiguous,
+    Unsupported,
+}
+
+impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
+    fn from(value: TransitionCandidateProbe) -> Self {
+        match value {
+            TransitionCandidateProbe::Missing => Self::Missing,
+            TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
+            TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
+            TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
+            TransitionCandidateProbe::Unsupported => Self::Unsupported,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TransitionOperatorStatus {
+    pub transaction_id: Uuid,
+    pub state: TransitionTransactionState,
+    pub tier_name: String,
+    pub remote_object: String,
+    pub not_after_unix_nanos: i64,
+    pub probe: TransitionOperatorProbe,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TransitionOperatorDeleteResult {
+    pub status: TransitionOperatorStatus,
+    pub journal_observed_after_delete: bool,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum TransitionOperatorError {
+    #[error("transition transaction was not found")]
+    NotFound,
+    #[error("transition transaction is still inside its active ownership window")]
+    NotExpired,
+    #[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
+    InvalidState(TransitionTransactionState),
+    #[error("an exact non-empty remote version is required")]
+    RemoteVersionRequired,
+    #[error("remote candidate is not proven missing: {0:?}")]
+    CandidateNotMissing(TransitionOperatorProbe),
+    #[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
+    CandidateVersionMismatch {
+        expected: String,
+        actual: TransitionOperatorProbe,
+    },
+    #[error("transition transaction store failed: {0}")]
+    Store(#[source] Error),
+    #[error("remote tier reconciliation failed: {0}")]
+    Remote(#[source] std::io::Error),
+}
+
+type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
+
+fn validate_operator_reconcile_transaction(
+    transaction: &TransitionTransaction,
+    now_unix_nanos: i128,
+) -> TransitionOperatorResult<()> {
+    transaction
+        .validate()
+        .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
+    if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
+        return Err(TransitionOperatorError::InvalidState(transaction.state));
+    }
+    if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
+        return Err(TransitionOperatorError::NotExpired);
+    }
+    Ok(())
+}
+
+async fn load_operator_reconcile_transaction(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<TransitionTransaction> {
+    match load_transition_transaction_record(api, transaction_id).await {
+        Ok(transaction) => Ok(transaction),
+        Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
+        Err(err) => Err(TransitionOperatorError::Store(err)),
+    }
+}
+
+async fn operator_probe_transition_candidate(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> TransitionOperatorResult<TransitionOperatorProbe> {
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
+    lease
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
+        .await
+        .map(TransitionOperatorProbe::from)
+        .map_err(TransitionOperatorError::Remote)
+}
+
+pub async fn inspect_transition_transaction_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<TransitionOperatorStatus> {
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let probe = operator_probe_transition_candidate(api, &transaction).await?;
+    Ok(TransitionOperatorStatus {
+        transaction_id,
+        state: transaction.state,
+        tier_name: transaction.tier_name,
+        remote_object: transaction.remote_object,
+        not_after_unix_nanos: transaction.not_after_unix_nanos,
+        probe,
+    })
+}
+
+pub async fn delete_transition_candidate_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+    remote_version_id: &str,
+) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
+    if remote_version_id.is_empty() {
+        return Err(TransitionOperatorError::RemoteVersionRequired);
+    }
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
+    lease
+        .validate_remote_version_id(remote_version_id)
+        .map_err(TransitionOperatorError::Remote)?;
+    let before_delete_probe = lease
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
+        .await
+        .map(TransitionOperatorProbe::from)
+        .map_err(TransitionOperatorError::Remote)?;
+    if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
+        return Err(TransitionOperatorError::CandidateVersionMismatch {
+            expected: remote_version_id.to_string(),
+            actual: before_delete_probe,
+        });
+    }
+    delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
+        .await
+        .map_err(TransitionOperatorError::Remote)?;
+    let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
+    let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
+        Ok(_) => true,
+        Err(Error::ConfigNotFound) => false,
+        Err(err) => return Err(TransitionOperatorError::Store(err)),
+    };
+    Ok(TransitionOperatorDeleteResult {
+        status: TransitionOperatorStatus {
+            transaction_id,
+            state: transaction.state,
+            tier_name: transaction.tier_name,
+            remote_object: transaction.remote_object,
+            not_after_unix_nanos: transaction.not_after_unix_nanos,
+            probe,
+        },
+        journal_observed_after_delete,
+    })
+}
+
+pub async fn finalize_missing_transition_transaction_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<()> {
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
+    if probe != TransitionOperatorProbe::Missing {
+        return Err(TransitionOperatorError::CandidateNotMissing(probe));
+    }
+    delete_transition_transaction_record(api, transaction_id)
+        .await
+        .map_err(TransitionOperatorError::Store)
+}
+
 pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
     let transaction_id = transition_transaction_id_from_record_object_name(object)?;
     TransitionTransaction::decode(transaction_id, data)
@@ -700,7 +894,7 @@ async fn recover_unknown_upload_outcome(
     .map_err(Error::other)?;
 
     match lease
-        .probe_transition_candidate(&transaction.remote_object)
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
         .await
         .map_err(Error::other)?
     {
@@ -1097,6 +1291,27 @@ mod tests {
             .expect("upload state change should succeed")
     }
 
+    #[test]
+    fn operator_reconcile_requires_expired_unknown_upload_outcome() {
+        let mut transaction = new_transaction();
+        let active_deadline = transaction.not_after_unix_nanos;
+
+        assert!(matches!(
+            validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
+            Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
+        ));
+
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("unknown upload outcome should be recorded");
+        assert!(matches!(
+            validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
+            Err(TransitionOperatorError::NotExpired)
+        ));
+        validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
+            .expect("expired unknown upload outcome should be eligible");
+    }
+
     fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
         TransitionCleanupProof {
             transaction_id: transaction.transaction_id,

crates/ecstore/src/cluster/rpc/background_monitor.rs

@@ -84,44 +84,6 @@ where
     )
 }
 
-/// Stop a sibling test thread from poisoning tracing's process-global callsite
-/// interest cache while this test asserts on span or event context.
-///
-/// `tracing` caches every callsite's `Interest` in **process-global** state, and
-/// the first thread to reach a callsite fixes that value. While at most one
-/// dispatcher is registered, tracing-core takes a fast path
-/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
-/// newly-registered callsite's interest from whichever subscriber is current
-/// *on the registering thread*.
-///
-/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
-/// reaches a production callsite first, from a thread with no subscriber
-/// installed: the interest is derived from that thread's `NoSubscriber` and
-/// cached as `Interest::never()` for the whole process. From then on the
-/// callsite is dead for *every* later caller, including a test that installed
-/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`
-/// (so the monitor's log line lands under the caller's span instead of
-/// `recovery-monitor`), and a `warn!`/`info!` event never fires at all.
-///
-/// Registering a second, inert dispatcher closes both halves of that race:
-///
-/// * constructing it rebuilds every *already-registered* callsite's interest
-///   against the live dispatcher set — which includes the caller's subscriber —
-///   repairing whatever a sibling may already have poisoned; and
-/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
-///   a callsite registered *later* by any thread is resolved against that live
-///   set instead of the registering thread's `NoSubscriber`.
-///
-/// Call this **after** installing the test's subscriber, and hold the returned
-/// guard for the rest of the test. Only the `cargo test` fallback needs it:
-/// nextest runs each test in its own process, where there is no sibling thread
-/// to lose the race to (see `docs/testing/README.md`).
-#[cfg(test)]
-#[must_use = "callsite interest is only pinned while the returned guard is held"]
-pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
-    tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/ecstore/src/cluster/rpc/client.rs

@@ -12,11 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
-use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
+#[cfg(test)]
+use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
+use crate::cluster::rpc::http_auth::{
+    RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
+    RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
+};
+use crate::cluster::rpc::{
+    gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
+};
+#[cfg(test)]
+use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
 use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
 use crate::runtime::sources as runtime_sources;
-use http::Uri;
+use http::{Request as HttpRequest, Response as HttpResponse, Uri};
 use rustfs_protos::{
     ChannelClass, create_new_channel, get_channel_for_class,
     proto_gen::node_service::{
@@ -24,9 +33,19 @@ use rustfs_protos::{
         tier_mutation_control_service_client::TierMutationControlServiceClient,
     },
 };
-use std::{error::Error, io::ErrorKind};
+use std::{
+    collections::HashMap,
+    error::Error,
+    future::Future,
+    io::ErrorKind,
+    pin::Pin,
+    sync::{LazyLock, Mutex},
+    task::{Context, Poll},
+};
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
+use tower::Service;
 use tracing::debug;
+use uuid::Uuid;
 
 use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
 
@@ -35,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
 pub async fn node_service_time_out_client(
     addr: &String,
     interceptor: TonicInterceptor,
-) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     // Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
     // `_for_class` variant below (grpc-optimization P1).
     node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
@@ -44,13 +63,14 @@ pub async fn node_service_time_out_client(
 pub async fn heal_control_time_out_client(
     addr: &str,
     interceptor: TonicInterceptor,
-) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = match runtime_sources::cached_node_channel(addr).await {
         Some(channel) => channel,
         None => create_new_channel(addr).await?,
     };
     let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
     Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -59,13 +79,14 @@ pub async fn heal_control_time_out_client(
 pub async fn tier_mutation_control_time_out_client(
     addr: &str,
     interceptor: TonicInterceptor,
-) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = match runtime_sources::cached_node_channel(addr).await {
         Some(channel) => channel,
         None => create_new_channel(addr).await?,
     };
     let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
     Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -81,7 +102,7 @@ pub async fn node_service_time_out_client_for_class(
     addr: &String,
     interceptor: TonicInterceptor,
     class: ChannelClass,
-) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     let interceptor = interceptor.with_rpc_audience(addr)?;
     let channel = match class {
         ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
@@ -96,6 +117,7 @@ pub async fn node_service_time_out_client_for_class(
     };
 
     let max_message_size = rustfs_protos::internode_rpc_max_message_size();
+    let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
     Ok(NodeServiceClient::with_interceptor(channel, interceptor)
         .max_decoding_message_size(max_message_size)
         .max_encoding_message_size(max_message_size))
@@ -103,7 +125,7 @@ pub async fn node_service_time_out_client_for_class(
 
 pub async fn node_service_time_out_client_no_auth(
     addr: &String,
-) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
     node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
 }
 
@@ -199,6 +221,104 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
     }
 }
 
+/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
+/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
+/// old servers continue receiving precisely the metadata they understand.
+#[derive(Clone, Debug)]
+pub struct ReplayScopeChannel<S> {
+    inner: S,
+    audience: Option<String>,
+}
+
+/// The channel type used by internode clients after v2 authentication and replay-scope handling.
+pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
+
+static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+impl<S> ReplayScopeChannel<S> {
+    fn new(inner: S, audience: Option<String>) -> Self {
+        Self { inner, audience }
+    }
+}
+
+fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
+    PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
+}
+
+fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
+    if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
+        epochs.insert(audience, epoch);
+    }
+}
+
+impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
+where
+    S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
+    S::Error: Send + 'static,
+    S::Future: Send + 'static,
+    ReqBody: Send + 'static,
+    ResBody: Send + 'static,
+{
+    type Response = HttpResponse<ResBody>;
+    type Error = S::Error;
+    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
+        let authenticated = self.audience.as_ref().is_some_and(|_| {
+            request
+                .headers()
+                .get(RPC_AUTH_VERSION_HEADER)
+                .and_then(|value| value.to_str().ok())
+                == Some(RPC_AUTH_VERSION_V2)
+        });
+        let challenge = authenticated.then(Uuid::new_v4);
+        if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
+            // The challenge is independently HMAC-authenticated by the response proof. It is not
+            // part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
+            request.headers_mut().insert(
+                RPC_BOOT_EPOCH_CHALLENGE_HEADER,
+                challenge.to_string().parse().expect("UUID must be a valid header value"),
+            );
+            if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
+                cached_peer_boot_epoch(audience),
+                request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
+                request
+                    .headers()
+                    .get(RPC_CONTENT_SHA256_HEADER)
+                    .and_then(|value| value.to_str().ok()),
+            ) {
+                match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
+                    Ok(headers) => request.headers_mut().extend(headers),
+                    Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
+                }
+            }
+        }
+
+        let audience = self.audience.clone();
+        let future = self.inner.call(request);
+        Box::pin(async move {
+            let response = future.await?;
+            if let (Some(audience), Some(challenge)) = (audience, challenge) {
+                match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
+                    Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
+                    Err(error)
+                        if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
+                            || response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
+                    {
+                        debug!(error = %error, "peer boot epoch response proof was rejected")
+                    }
+                    Err(_) => {}
+                }
+            }
+            Ok(response)
+        })
+    }
+}
+
 pub struct TonicSignatureInterceptor {
     audience: Option<String>,
 }
@@ -257,6 +377,13 @@ impl TonicInterceptor {
         }
         Ok(self)
     }
+
+    fn replay_scope_audience(&self) -> Option<String> {
+        match self {
+            Self::Signature(interceptor) => interceptor.audience.clone(),
+            Self::NoOp(_) => None,
+        }
+    }
 }
 
 impl tonic::service::Interceptor for TonicInterceptor {
@@ -279,6 +406,38 @@ mod tests {
     use tracing_opentelemetry::OpenTelemetrySpanExt;
     use tracing_subscriber::{Registry, layer::SubscriberExt};
 
+    #[derive(Clone)]
+    struct EpochProofService {
+        audience: String,
+        seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
+    }
+
+    impl Service<HttpRequest<()>> for EpochProofService {
+        type Response = HttpResponse<()>;
+        type Error = std::convert::Infallible;
+        type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
+
+        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
+            self.seen_headers
+                .lock()
+                .expect("test header capture lock must not be poisoned")
+                .push(request.headers().clone());
+            let challenge = tonic_boot_epoch_challenge(request.headers())
+                .expect("client challenge must be syntactically valid")
+                .expect("authenticated client request must carry a boot epoch challenge");
+            let mut response = HttpResponse::new(());
+            response.headers_mut().extend(
+                tonic_boot_epoch_response_headers(&self.audience, challenge)
+                    .expect("test server must be able to sign an epoch proof"),
+            );
+            std::future::ready(Ok(response))
+        }
+    }
+
     fn ensure_test_rpc_secret() {
         runtime_sources::ensure_test_rpc_secret();
     }
@@ -420,6 +579,52 @@ mod tests {
         assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
     }
 
+    #[test]
+    fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
+        ensure_test_rpc_secret();
+        let audience = "replay-scope-client-test:9000";
+        PEER_BOOT_EPOCHS
+            .lock()
+            .expect("peer epoch cache lock must not be poisoned")
+            .remove(audience);
+        let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
+        let service = EpochProofService {
+            audience: audience.to_string(),
+            seen_headers: seen_headers.clone(),
+        };
+        let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
+        let make_request = || {
+            let mut request = HttpRequest::builder()
+                .uri("/node_service.NodeService/Ping")
+                .body(())
+                .expect("test RPC request must build");
+            request.headers_mut().extend(
+                gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
+                    .expect("v2 test headers must mint"),
+            );
+            request
+        };
+
+        futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
+        futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
+
+        let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
+        assert_eq!(headers.len(), 2);
+        assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
+        assert!(
+            !headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
+            "the first request must remain v2-compatible until the peer proves its epoch"
+        );
+        assert!(
+            headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
+            "the second request must carry the replay-scoped v3 signature"
+        );
+        PEER_BOOT_EPOCHS
+            .lock()
+            .expect("peer epoch cache lock must not be poisoned")
+            .remove(audience);
+    }
+
     #[test]
     fn test_signature_interceptor_requires_generated_method_metadata() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -20,8 +20,8 @@
 //! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
 //! below, plus the broader negative-signature suite. The advisory class is: a
 //! node must never accept an RPC whose auth is missing, malformed, or signed
-//! with the default/empty shared secret. Body-bound v2 requests additionally
-//! receive process-local replay protection. See
+//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
+//! requests additionally receive process-local replay protection. See
 //! `docs/testing/security-regressions.md` for the full advisory -> test map.
 //!
 //! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
@@ -50,13 +50,22 @@ use uuid::Uuid;
 type HmacSha256 = Hmac<Sha256>;
 
 const SIGNATURE_HEADER: &str = "x-rustfs-signature";
-const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
-const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
+pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
+pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
 const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
 const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
 pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
-const RPC_AUTH_VERSION_V2: &str = "2";
+pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
+pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
+pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
+pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
+pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
+pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
+pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
+const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
 const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
+const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
+const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
 const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
 const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
 const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
@@ -75,9 +84,15 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
         rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
     )
 });
-// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
-// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
-// cannot disable replay protection by rejecting every body-bound request.
+static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
+    get_env_bool(
+        rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
+        rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
+    )
+});
+// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
+// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
+// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
 static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
     rustfs_utils::get_env_usize(
         rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
@@ -86,6 +101,7 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
     .max(1)
 });
 static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
+static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
 
 #[derive(Default)]
 struct RpcNonceCache {
@@ -313,6 +329,189 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
     mac.verify_slice(&signature).is_ok()
 }
 
+#[derive(Clone, Copy)]
+struct ReplayScope<'a> {
+    audience: &'a str,
+    path: &'a str,
+    timestamp: &'a str,
+    nonce: Uuid,
+    content_sha256: &'a str,
+    boot_epoch: Uuid,
+}
+
+fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
+    mac.update(RPC_REPLAY_SCOPE_DOMAIN);
+    for part in [
+        scope.audience.as_bytes(),
+        b"|",
+        scope.path.as_bytes(),
+        b"|POST|",
+        scope.timestamp.as_bytes(),
+        b"|",
+        scope.nonce.as_bytes(),
+        b"|",
+        scope.content_sha256.as_bytes(),
+        b"|",
+        scope.boot_epoch.as_bytes(),
+    ] {
+        mac.update(part);
+    }
+}
+
+fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_replay_scope(&mut mac, scope);
+    Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
+}
+
+fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
+    let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
+        return false;
+    };
+    let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
+        return false;
+    };
+    update_replay_scope(&mut mac, scope);
+    mac.verify_slice(&signature).is_ok()
+}
+
+fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
+    mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
+    mac.update(audience.as_bytes());
+    mac.update(b"|");
+    mac.update(challenge.as_bytes());
+    mac.update(boot_epoch.as_bytes());
+}
+
+fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
+    if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
+        return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
+    }
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
+    Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
+}
+
+fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
+    if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
+        return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
+    }
+    let proof = general_purpose::STANDARD
+        .decode(proof)
+        .map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
+    let mut mac =
+        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
+    update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
+    mac.verify_slice(&proof)
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
+}
+
+fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
+    let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
+    (!value.is_nil())
+        .then_some(value)
+        .ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
+}
+
+fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
+    path.strip_prefix('/')
+        .and_then(|path| path.split_once('/'))
+        .filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
+        .ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
+}
+
+/// The process-unique epoch included in every replay-scoped server verification.
+///
+/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
+/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
+pub fn tonic_rpc_boot_epoch() -> Uuid {
+    *RPC_BOOT_EPOCH
+}
+
+/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
+/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
+/// old servers can continue validating the same request unchanged.
+pub fn gen_tonic_replay_scope_headers(
+    audience: &str,
+    path: &str,
+    timestamp: &str,
+    content_sha256: &str,
+    boot_epoch: Uuid,
+) -> std::io::Result<HeaderMap> {
+    if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
+        return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
+    }
+    parse_tonic_rpc_path(path)?;
+    timestamp
+        .parse::<i64>()
+        .map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
+
+    let nonce = Uuid::new_v4();
+    let signature = generate_replay_scope_signature(
+        &get_shared_secret()?,
+        ReplayScope {
+            audience,
+            path,
+            timestamp,
+            nonce,
+            content_sha256,
+            boot_epoch,
+        },
+    )?;
+    let mut headers = HeaderMap::new();
+    headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
+    headers.insert(
+        RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+        header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
+    );
+    headers.insert(
+        RPC_REPLAY_SCOPE_NONCE_HEADER,
+        header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
+    );
+    headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
+    Ok(headers)
+}
+
+/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
+pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
+    headers
+        .get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
+        .map(|value| {
+            value
+                .to_str()
+                .map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
+                .and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
+        })
+        .transpose()
+}
+
+/// Build the authenticated response headers for a client boot-epoch challenge.
+pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
+    let boot_epoch = tonic_rpc_boot_epoch();
+    let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
+    let mut headers = HeaderMap::new();
+    headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
+    headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
+    Ok(headers)
+}
+
+/// Verify the server boot-epoch response for a challenge generated by this client.
+pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
+    let boot_epoch = headers
+        .get(RPC_BOOT_EPOCH_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
+        .and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
+    let proof = headers
+        .get(RPC_BOOT_EPOCH_PROOF_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
+    verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
+    Ok(boot_epoch)
+}
+
 fn valid_content_sha256(value: &str) -> bool {
     value == UNSIGNED_PAYLOAD
         || (value.len() == 64
@@ -531,6 +730,17 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
     .any(|name| headers.contains_key(*name))
 }
 
+fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
+    [
+        RPC_REPLAY_SCOPE_VERSION_HEADER,
+        RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
+        RPC_REPLAY_SCOPE_NONCE_HEADER,
+        RPC_BOOT_EPOCH_HEADER,
+    ]
+    .iter()
+    .any(|name| headers.contains_key(*name))
+}
+
 /// Whether the server requires target-bound v2 authentication on every internode gRPC request,
 /// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
 /// lever gated on the v1-fallback counter reading zero fleet-wide; see
@@ -540,9 +750,127 @@ fn internode_rpc_signature_strict() -> bool {
     *INTERNODE_RPC_SIGNATURE_STRICT
 }
 
+fn internode_rpc_replay_scope_strict() -> bool {
+    *INTERNODE_RPC_REPLAY_SCOPE_STRICT
+}
+
+fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
+    if audience.is_empty() {
+        return Err(std::io::Error::other("Missing RPC audience"));
+    }
+    parse_tonic_rpc_path(path)?;
+
+    let version = headers
+        .get(RPC_REPLAY_SCOPE_VERSION_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
+    if version != RPC_REPLAY_SCOPE_VERSION_V3 {
+        return Err(std::io::Error::other("Unsupported RPC replay scope version"));
+    }
+    let signature = headers
+        .get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
+    let timestamp = headers
+        .get(TIMESTAMP_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
+    let signed_at = timestamp
+        .parse::<i64>()
+        .map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
+    check_timestamp(signed_at)?;
+    let nonce = headers
+        .get(RPC_REPLAY_SCOPE_NONCE_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
+        .and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
+    let content_sha256 = headers
+        .get(RPC_CONTENT_SHA256_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
+    if !valid_content_sha256(content_sha256) {
+        return Err(std::io::Error::other("Invalid RPC content SHA-256"));
+    }
+    let boot_epoch = headers
+        .get(RPC_BOOT_EPOCH_HEADER)
+        .and_then(|value| value.to_str().ok())
+        .ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
+        .and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
+    let secret = get_shared_secret()?;
+    if !verify_replay_scope_signature(
+        &secret,
+        ReplayScope {
+            audience,
+            path,
+            timestamp,
+            nonce,
+            content_sha256,
+            boot_epoch,
+        },
+        signature,
+    ) {
+        return Err(std::io::Error::other("Invalid RPC replay scope signature"));
+    }
+    if boot_epoch != tonic_rpc_boot_epoch() {
+        return Err(std::io::Error::other("RPC boot epoch is stale"));
+    }
+    check_and_record_nonce(nonce, signed_at)
+}
+
 /// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
 pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
-    verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
+    verify_tonic_rpc_signature_with_policy(
+        audience,
+        path,
+        headers,
+        internode_rpc_signature_strict(),
+        internode_rpc_replay_scope_strict(),
+        false,
+    )
+}
+
+/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
+/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
+pub fn verify_tonic_rpc_signature_with_bootstrap(
+    audience: &str,
+    path: &str,
+    headers: &HeaderMap,
+    allow_replay_scope_bootstrap: bool,
+) -> std::io::Result<()> {
+    verify_tonic_rpc_signature_with_policy(
+        audience,
+        path,
+        headers,
+        internode_rpc_signature_strict(),
+        internode_rpc_replay_scope_strict(),
+        allow_replay_scope_bootstrap,
+    )
+}
+
+fn verify_tonic_rpc_signature_with_policy(
+    audience: &str,
+    path: &str,
+    headers: &HeaderMap,
+    signature_strict: bool,
+    replay_scope_strict: bool,
+    allow_replay_scope_bootstrap: bool,
+) -> std::io::Result<()> {
+    if has_replay_scope_headers(headers) {
+        return verify_tonic_replay_scope_signature(audience, path, headers);
+    }
+
+    // Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
+    // client after its peer restarts. Legacy metadata never gets this exception.
+    let bootstrap = allow_replay_scope_bootstrap
+        && has_v2_auth_headers(headers)
+        && tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
+    if replay_scope_strict && !bootstrap {
+        return Err(std::io::Error::other("RPC replay-scoped authentication required"));
+    }
+
+    verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
+    global_internode_metrics().record_replay_scope_fallback();
+    Ok(())
 }
 
 /// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
@@ -1273,6 +1601,104 @@ mod tests {
         assert_eq!(error.to_string(), "Invalid RPC v2 signature");
     }
 
+    #[test]
+    fn replay_scope_binds_path_epoch_and_random_nonce() {
+        ensure_test_rpc_secret();
+        let path = "/node_service.NodeService/Ping";
+        let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("v2 compatibility headers should build");
+        let timestamp = headers
+            .get(TIMESTAMP_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 timestamp")
+            .to_string();
+        let content_sha256 = headers
+            .get(RPC_CONTENT_SHA256_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 content digest")
+            .to_string();
+        headers.extend(
+            gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, tonic_rpc_boot_epoch())
+                .expect("replay-scope headers should build"),
+        );
+
+        assert!(
+            verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
+            "the first replay-scoped request must be accepted"
+        );
+        let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
+            .expect_err("the random replay-scope nonce must be single-use");
+        assert_eq!(replay.to_string(), "RPC request replay detected");
+
+        let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
+            .expect_err("a replay-scoped signature must not move to another method");
+        assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
+    }
+
+    #[test]
+    fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
+        ensure_test_rpc_secret();
+        let path = "/node_service.NodeService/Ping";
+        let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("v2 compatibility headers should build");
+        partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
+        let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
+            .expect_err("partial replay-scope metadata must never downgrade to v2");
+        assert_eq!(error.to_string(), "Missing RPC replay scope signature");
+
+        let timestamp = partial
+            .get(TIMESTAMP_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 timestamp")
+            .to_string();
+        let content_sha256 = partial
+            .get(RPC_CONTENT_SHA256_HEADER)
+            .and_then(|value| value.to_str().ok())
+            .expect("v2 content digest")
+            .to_string();
+        let stale_epoch = Uuid::new_v4();
+        partial.extend(
+            gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, stale_epoch)
+                .expect("replay-scope headers should build"),
+        );
+        let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
+            .expect_err("a signature from a prior server boot epoch must be rejected");
+        assert_eq!(stale.to_string(), "RPC boot epoch is stale");
+    }
+
+    #[test]
+    fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
+        ensure_test_rpc_secret();
+        let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
+            .expect("v2 compatibility headers should build");
+        let rejected =
+            verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
+                .expect_err("strict replay scope must reject stripped new metadata");
+        assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
+
+        headers.insert(
+            RPC_BOOT_EPOCH_CHALLENGE_HEADER,
+            HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
+        );
+        assert!(
+            verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
+                .is_ok(),
+            "only the signed Ping bootstrap may obtain a new server epoch in strict mode"
+        );
+    }
+
+    #[test]
+    fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
+        ensure_test_rpc_secret();
+        let challenge = Uuid::new_v4();
+        let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
+        let epoch =
+            verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
+        assert_eq!(epoch, tonic_rpc_boot_epoch());
+        assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
+        assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
+    }
+
     #[test]
     fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
         ensure_test_rpc_secret();

crates/ecstore/src/cluster/rpc/mod.rs

@@ -23,18 +23,21 @@ pub(crate) mod remote_disk;
 pub(crate) mod remote_locker;
 pub(crate) mod runtime_sources;
 
-#[cfg(test)]
-pub(crate) use background_monitor::pin_callsite_interest_for_test;
 pub use background_monitor::shutdown_background_monitors;
 pub(crate) use background_monitor::spawn_background_monitor;
 pub use client::{
-    TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
+    node_service_time_out_client_no_auth,
 };
+// Re-exported through `api::rpc`; not every item is consumed inside this crate.
+#[allow(unused_imports)]
 pub use http_auth::{
-    TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
-    set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
-    verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
-    verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
+    normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
+    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
+    verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+    verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::cluster::rpc::client::{
-    TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
+    AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
     is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
 };
 use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
@@ -66,7 +66,6 @@ use std::{
 use tokio::{net::TcpStream, time::Duration};
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
-use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
@@ -222,6 +221,21 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
         .map_err(|_| Error::other("peer returned an invalid heal control response proof"))
 }
 
+fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
+    let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
+    if topology_member != expected_member {
+        return Err(Error::other(
+            "peer returned a remote version state capability for a different topology member",
+        ));
+    }
+    let server_epoch =
+        Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
+    if server_epoch.is_nil() {
+        return Err(Error::other("peer returned a nil remote version state epoch"));
+    }
+    Ok(server_epoch)
+}
+
 #[derive(Clone, Debug)]
 pub struct PeerLiveEventsBatch {
     pub events: Vec<u8>,
@@ -233,6 +247,7 @@ pub struct PeerLiveEventsBatch {
 pub struct PeerRestClient {
     pub host: XHost,
     pub grid_host: String,
+    topology_member: String,
     offline: Arc<AtomicBool>,
     recovery_running: Arc<AtomicBool>,
 }
@@ -325,9 +340,11 @@ impl PeerRestClient {
     }
 
     pub fn new(host: XHost, grid_host: String) -> Self {
+        let topology_member = host.to_string();
         Self {
             host,
             grid_host,
+            topology_member,
             offline: Arc::new(AtomicBool::new(false)),
             recovery_running: Arc::new(AtomicBool::new(false)),
         }
@@ -347,7 +364,11 @@ impl PeerRestClient {
 
             let client = match grid_host {
                 Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
-                    Ok(host) => Some(PeerRestClient::new(host, grid_host)),
+                    Ok(host) => {
+                        let mut client = PeerRestClient::new(host, grid_host);
+                        client.topology_member = peer_host_port.clone();
+                        Some(client)
+                    }
                     Err(err) => {
                         warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
                         None
@@ -390,7 +411,7 @@ impl PeerRestClient {
         (remote, all, remote_topology_hosts)
     }
 
-    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if self.offline.load(Ordering::Acquire) {
             self.mark_offline_and_spawn_recovery();
             return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -411,7 +432,7 @@ impl PeerRestClient {
         &self,
     ) -> Result<
         rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
-            InterceptedService<Channel, TonicInterceptor>,
+            InterceptedService<AuthenticatedChannel, TonicInterceptor>,
         >,
     > {
         if self.offline.load(Ordering::Acquire) {
@@ -432,7 +453,7 @@ impl PeerRestClient {
 
     async fn get_tier_mutation_control_client(
         &self,
-    ) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    ) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if self.offline.load(Ordering::Acquire) {
             self.mark_offline_and_spawn_recovery();
             return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -1154,6 +1175,15 @@ impl PeerRestClient {
         validate_heal_control_capability_proof(&canonical_ack, &proof)
     }
 
+    pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
+        let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
+        let result = self
+            .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
+            .await?;
+        let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
+        Ok((self.topology_member.clone(), epoch))
+    }
+
     pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
         self.finalize_result(
             async {
@@ -2470,6 +2500,22 @@ mod tests {
         }
     }
 
+    #[test]
+    fn remote_version_state_capability_decoder_fails_closed() {
+        let epoch = Uuid::new_v4();
+        let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
+            .expect("small capability response should encode");
+        assert_eq!(
+            decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
+            epoch
+        );
+        assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
+        assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
+        let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
+            .expect("small capability response should encode");
+        assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
+    }
+
     struct TierMutationResponseFixture<'a> {
         version: u32,
         phase: TierMutationRpcPhase,
@@ -2745,7 +2791,7 @@ mod tests {
         // `mark_offline_and_spawn_recovery` path that sibling tests exercise from
         // subscriber-less threads; without this the span can be cached as
         // `Interest::never()` and silently degrade to `Span::none()`.
-        let _callsite_pin = crate::cluster::rpc::pin_callsite_interest_for_test();
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
 
         let client = test_peer_client();
         let span = tracing::info_span!("request-span", request_id = "req-peer-rest");

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -14,7 +14,8 @@
 
 use crate::bucket::metadata_sys;
 use crate::cluster::rpc::client::{
-    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
+    node_service_time_out_client,
 };
 use crate::cluster::rpc::set_tonic_mutation_body_digest;
 use crate::disk::error::DiskError;
@@ -40,16 +41,84 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
 use rustfs_protos::proto_gen::node_service::{
     DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
 };
+#[cfg(test)]
+use std::sync::{
+    Mutex as StdMutex,
+    atomic::{AtomicBool, Ordering},
+};
 use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
+#[cfg(test)]
+use tokio::sync::Notify;
 use tokio::{net::TcpStream, sync::RwLock, time};
 use tokio_util::sync::CancellationToken;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
-use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 
 type Client = Arc<Box<dyn PeerS3Client>>;
 
+#[cfg(test)]
+#[derive(Default)]
+pub(crate) struct DeleteBucketEmptyScanBarrier {
+    arrived: AtomicBool,
+    arrived_notify: Notify,
+    released: AtomicBool,
+    release_notify: Notify,
+}
+
+#[cfg(test)]
+impl DeleteBucketEmptyScanBarrier {
+    pub(crate) async fn wait_until_paused(&self) {
+        loop {
+            let notified = self.arrived_notify.notified();
+            if self.arrived.load(Ordering::Acquire) {
+                return;
+            }
+            notified.await;
+        }
+    }
+
+    pub(crate) fn release(&self) {
+        self.released.store(true, Ordering::Release);
+        self.release_notify.notify_waiters();
+    }
+
+    async fn pause(&self) {
+        self.arrived.store(true, Ordering::Release);
+        self.arrived_notify.notify_waiters();
+        loop {
+            let notified = self.release_notify.notified();
+            if self.released.load(Ordering::Acquire) {
+                return;
+            }
+            notified.await;
+        }
+    }
+}
+
+#[cfg(test)]
+static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
+
+#[cfg(test)]
+pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
+    let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
+    *DELETE_BUCKET_EMPTY_SCAN_BARRIER
+        .lock()
+        .expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
+    barrier
+}
+
+#[cfg(test)]
+async fn pause_after_delete_bucket_empty_scan() {
+    let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
+        .lock()
+        .expect("empty scan barrier lock should not be poisoned")
+        .take();
+    if let Some(barrier) = barrier {
+        barrier.pause().await;
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct ScannerBucketListing {
     pub buckets: Vec<BucketInfo>,
@@ -650,24 +719,22 @@ impl PeerS3Client for LocalPeerS3Client {
             return Err(Error::ErasureWriteQuorum);
         }
 
-        let force = if opts.force_if_empty && !opts.force {
+        if opts.force_if_empty && !opts.force {
             for disk in local_disks.iter() {
                 if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
                     return Err(Error::VolumeNotEmpty);
                 }
             }
-            true
-        } else {
-            opts.force
-        };
+            #[cfg(test)]
+            pause_after_delete_bucket_empty_scan().await;
+        }
 
         let mut futures = Vec::with_capacity(local_disks.len());
 
         for disk in local_disks.iter() {
-            // Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
-            // the recreate loop below turns into BucketNotEmpty; only an explicit
-            // force delete removes recursively (backlog#799 B1).
-            futures.push(disk.delete_volume(bucket, force));
+            // `force_if_empty` is validation-only. Passing it as force would let
+            // a PutObject committed after the scan be removed recursively.
+            futures.push(disk.delete_volume(bucket, opts.force));
         }
 
         let results = join_all(futures).await;
@@ -730,6 +797,15 @@ pub struct RemotePeerS3Client {
 }
 
 impl RemotePeerS3Client {
+    fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
+        let mut remote_opts = opts.clone();
+        // Older peers promote `force_if_empty` to recursive force after their
+        // metadata scan. Keep this coordinator-only hint off the wire so a
+        // mixed-version delete fails closed on non-empty directory remnants.
+        remote_opts.force_if_empty = false;
+        serde_json::to_string(&remote_opts).map_err(Into::into)
+    }
+
     fn recovery_monitor_span(addr: &str) -> tracing::Span {
         tracing::info_span!(
             "recovery-monitor",
@@ -756,7 +832,7 @@ impl RemotePeerS3Client {
         client
     }
 
-    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
             .await
             .map_err(|err| Error::other(format!("can not get client, err: {err}")))
@@ -1040,7 +1116,7 @@ impl PeerS3Client for RemotePeerS3Client {
     async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
         self.execute_with_timeout(
             || async {
-                let options = serde_json::to_string(opts)?;
+                let options = Self::encode_delete_bucket_options(opts)?;
                 let mut client = self.get_client().await?;
 
                 let mut request = Request::new(DeleteBucketRequest {
@@ -1414,6 +1490,49 @@ mod tests {
         }
     }
 
+    #[test]
+    fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
+        let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
+            no_lock: true,
+            no_recreate: true,
+            force_if_empty: true,
+            ..Default::default()
+        })
+        .expect("remote delete options should serialize");
+        let legacy_opts: DeleteBucketOptions =
+            serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
+
+        assert!(legacy_opts.no_lock);
+        assert!(legacy_opts.no_recreate);
+        assert!(!legacy_opts.force);
+        assert!(!legacy_opts.force_if_empty);
+
+        let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
+            true
+        } else {
+            legacy_opts.force
+        };
+        assert!(
+            !legacy_recursive_force,
+            "legacy peer must not upgrade empty-only delete to recursive force"
+        );
+    }
+
+    #[test]
+    fn remote_delete_bucket_options_preserve_explicit_force() {
+        let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
+            force: true,
+            force_if_empty: true,
+            ..Default::default()
+        })
+        .expect("remote force-delete options should serialize");
+        let remote_opts: DeleteBucketOptions =
+            serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
+
+        assert!(remote_opts.force);
+        assert!(!remote_opts.force_if_empty);
+    }
+
     #[tokio::test]
     async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
         let client = test_remote_peer("http://peer-network-error:9000");
@@ -1571,6 +1690,54 @@ mod tests {
         reset_local_disk_test_state().await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
+        let disks = init_test_local_disks_for_pools(
+            &temp_dir,
+            &[(0, 1), (1, 1)],
+            "local-peer-force-if-empty-preserves-unclassified-file",
+        )
+        .await;
+        let bucket = "empty-only-delete-bucket";
+        let marker = "object/commit-marker";
+        let data = bytes::Bytes::from_static(b"committed object data");
+
+        disks[1]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should be created in the selected pool");
+        disks[1]
+            .write_all(bucket, marker, data.clone())
+            .await
+            .expect("unclassified committed file should be written");
+
+        let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
+            .delete_bucket(
+                bucket,
+                &DeleteBucketOptions {
+                    force_if_empty: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("empty-only delete must not recursively remove an unclassified file");
+
+        assert_eq!(err, Error::VolumeNotEmpty);
+        assert_eq!(
+            disks[1]
+                .read_all(bucket, marker)
+                .await
+                .expect("unclassified committed file should be preserved"),
+            data
+        );
+
+        reset_local_disk_test_state().await;
+    }
+
     #[tokio::test]
     #[serial]
     async fn heal_bucket_local_recreates_missing_bucket_volumes() {

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use crate::cluster::rpc::client::{
-    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
-    node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
+    node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
 };
 use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
 use crate::cluster::rpc::internode_data_transport::{
@@ -71,7 +71,7 @@ use tokio::{
     time::timeout,
 };
 use tokio_util::sync::CancellationToken;
-use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
+use tonic::{Code, Request, service::interceptor::InterceptedService};
 use tracing::{debug, trace, warn};
 use uuid::Uuid;
 
@@ -1083,7 +1083,7 @@ impl RemoteDisk {
         internode_offline_bypass_reason(&self.addr).map(Error::other)
     }
 
-    async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if let Some(err) = self.offline_bypass_error() {
             return Err(err);
         }
@@ -1096,7 +1096,7 @@ impl RemoteDisk {
     /// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
     /// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
     /// is disabled.
-    async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         if let Some(err) = self.offline_bypass_error() {
             return Err(err);
         }
@@ -5336,7 +5336,7 @@ mod tests {
         // production callsites that sibling tests exercise from subscriber-less
         // threads; without this they can be cached as `Interest::never()` and go
         // silently missing here.
-        let _callsite_pin = crate::cluster::rpc::pin_callsite_interest_for_test();
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
 
         let endpoint = Endpoint {
             url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
@@ -5395,7 +5395,7 @@ mod tests {
         // production callsites that sibling tests exercise from subscriber-less
         // threads; without this they can be cached as `Interest::never()` and go
         // silently missing here.
-        let _callsite_pin = crate::cluster::rpc::pin_callsite_interest_for_test();
+        let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
 
         let addr = "http://127.0.0.1:59997".to_string();
         let endpoint = Endpoint {

crates/ecstore/src/cluster/rpc/remote_locker.rs

@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::cluster::rpc::client::{
+    AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
+};
 use crate::cluster::rpc::set_tonic_mutation_body_digest;
 use async_trait::async_trait;
 use bytes::Bytes;
@@ -29,7 +31,6 @@ use std::time::Duration;
 use tokio::time::timeout;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
-use tonic::transport::Channel;
 use tracing::{debug, info, warn};
 
 /// Remote lock client implementation
@@ -78,7 +79,7 @@ impl RemoteClient {
         }
     }
 
-    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
+    pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
         // P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
         // offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
         // change quorum; the self-healing re-probe keeps the peer recoverable.

crates/ecstore/src/disk/local.rs

@@ -381,6 +381,291 @@ fn classify_delete_volume_error(err: std::io::Error) -> DiskError {
     }
 }
 
+#[cfg(unix)]
+struct EmptyDirectoryFrame {
+    path: PathBuf,
+    name_in_parent: std::ffi::CString,
+    entries: rustix::fs::Dir,
+}
+
+#[cfg(unix)]
+fn empty_tree_io_error(err: rustix::io::Errno) -> std::io::Error {
+    match err {
+        rustix::io::Errno::NOTDIR | rustix::io::Errno::LOOP => std::io::Error::from(ErrorKind::DirectoryNotEmpty),
+        _ => err.into(),
+    }
+}
+
+#[cfg(unix)]
+fn remove_empty_directory_tree_unix_with(
+    root: &Path,
+    mut before_descend: impl FnMut(&Path) -> std::io::Result<()>,
+    mut before_remove: impl FnMut(&Path) -> std::io::Result<()>,
+) -> std::io::Result<()> {
+    use rustix::{
+        fs::{AtFlags, Dir, Mode, OFlags, fstat, open, openat, statat, unlinkat},
+        io::Errno,
+    };
+    use std::os::fd::AsFd;
+    use std::os::unix::ffi::OsStrExt;
+
+    let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
+    let root_parent_path = root
+        .parent()
+        .ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?;
+    let root_name = root
+        .file_name()
+        .ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?
+        .as_bytes();
+    let root_name = std::ffi::CString::new(root_name).map_err(|_| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?;
+    let root_parent = open(root_parent_path, flags, Mode::empty()).map_err(empty_tree_io_error)?;
+    let root_fd = openat(&root_parent, root_name.as_c_str(), flags, Mode::empty()).map_err(empty_tree_io_error)?;
+    // Each frame owns one directory iterator/FD, so memory and descriptors are
+    // bounded by path depth rather than by the number of empty remnants.
+    let mut stack = vec![EmptyDirectoryFrame {
+        path: root.to_path_buf(),
+        name_in_parent: root_name,
+        entries: Dir::new(root_fd).map_err(empty_tree_io_error)?,
+    }];
+
+    while let Some(mut frame) = stack.pop() {
+        let next_child = loop {
+            let Some(entry) = frame.entries.next() else {
+                break None;
+            };
+            let entry = entry.map_err(std::io::Error::from)?;
+            let name = entry.file_name();
+            if name.to_bytes() == b"." || name.to_bytes() == b".." {
+                continue;
+            }
+
+            let name = name.to_owned();
+            let child_path = frame.path.join(std::ffi::OsStr::from_bytes(name.as_bytes()));
+            before_descend(&child_path)?;
+            break Some((child_path, name));
+        };
+
+        if let Some((child_path, name)) = next_child {
+            let parent = frame.entries.fd().map_err(std::io::Error::from)?;
+            let child = match openat(parent, name.as_c_str(), flags, Mode::empty()) {
+                Ok(child) => child,
+                // A concurrent cleanup may remove an empty child after readdir
+                // returns it. Resume the parent instead of treating the whole
+                // bucket as missing and leaving its root behind.
+                Err(Errno::NOENT) => {
+                    stack.push(frame);
+                    continue;
+                }
+                Err(err) => return Err(empty_tree_io_error(err)),
+            };
+            stack.push(frame);
+            stack.push(EmptyDirectoryFrame {
+                path: child_path,
+                name_in_parent: name,
+                entries: Dir::new(child).map_err(empty_tree_io_error)?,
+            });
+            continue;
+        }
+        before_remove(&frame.path)?;
+        let parent = if let Some(parent) = stack.last() {
+            parent.entries.fd().map_err(std::io::Error::from)?
+        } else {
+            root_parent.as_fd()
+        };
+        let expected = fstat(frame.entries.fd().map_err(std::io::Error::from)?).map_err(empty_tree_io_error)?;
+        let current = statat(parent, frame.name_in_parent.as_c_str(), AtFlags::SYMLINK_NOFOLLOW).map_err(empty_tree_io_error)?;
+        if current.st_dev != expected.st_dev || current.st_ino != expected.st_ino {
+            return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
+        }
+        match unlinkat(parent, frame.name_in_parent.as_c_str(), AtFlags::REMOVEDIR).map_err(empty_tree_io_error) {
+            Ok(()) => {}
+            Err(err) if err.kind() == ErrorKind::NotFound => {}
+            Err(err) => return Err(err),
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(unix)]
+async fn remove_empty_directory_tree_with(
+    root: &Path,
+    before_descend: impl FnMut(&Path) -> std::io::Result<()>,
+    before_remove: impl FnMut(&Path) -> std::io::Result<()>,
+) -> std::io::Result<()> {
+    remove_empty_directory_tree_unix_with(root, before_descend, before_remove)
+}
+
+#[cfg(unix)]
+async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
+    let root = root.to_path_buf();
+    tokio::task::spawn_blocking(move || remove_empty_directory_tree_unix_with(&root, |_| Ok(()), |_| Ok(()))).await?
+}
+
+#[cfg(windows)]
+#[derive(Debug)]
+struct LockedEmptyDirectory {
+    handle: winapi_util::Handle,
+}
+
+#[cfg(windows)]
+fn validate_windows_empty_directory(file_attributes: u64) -> std::io::Result<()> {
+    const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
+    const FILE_ATTRIBUTE_REPARSE_POINT: u64 = 0x400;
+
+    if file_attributes & FILE_ATTRIBUTE_DIRECTORY == 0 || file_attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
+        return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
+    }
+    Ok(())
+}
+
+#[cfg(windows)]
+async fn lock_windows_empty_directory(path: &Path, canonical_root: Option<&Path>) -> std::io::Result<LockedEmptyDirectory> {
+    use std::os::windows::fs::OpenOptionsExt;
+    use windows_sys::Win32::{
+        Foundation::GENERIC_READ,
+        Storage::FileSystem::{DELETE, FILE_SHARE_READ},
+    };
+
+    const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
+    const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
+
+    let path = path.to_path_buf();
+    let canonical_root = canonical_root.map(Path::to_path_buf);
+    tokio::task::spawn_blocking(move || {
+        let file = std::fs::OpenOptions::new()
+            .access_mode(GENERIC_READ | DELETE)
+            .share_mode(FILE_SHARE_READ)
+            .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
+            .open(&path)?;
+        let handle = winapi_util::Handle::from_file(file);
+        let info = winapi_util::file::information(&handle)?;
+        validate_windows_empty_directory(info.file_attributes())?;
+        if let Some(canonical_root) = canonical_root {
+            let canonical_path = std::fs::canonicalize(path)?;
+            if !canonical_path.starts_with(canonical_root) {
+                return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
+            }
+        }
+        Ok::<_, std::io::Error>(LockedEmptyDirectory { handle })
+    })
+    .await?
+}
+
+#[cfg(windows)]
+// SAFETY: This helper only passes an owned live handle and one initialized
+// FILE_DISPOSITION_INFO to the synchronous Windows deletion API.
+#[allow(unsafe_code)]
+async fn remove_windows_empty_directory(directory: LockedEmptyDirectory) -> std::io::Result<()> {
+    tokio::task::spawn_blocking(move || {
+        use std::os::windows::io::AsRawHandle;
+        use windows_sys::Win32::Storage::FileSystem::{FILE_DISPOSITION_INFO, FileDispositionInfo, SetFileInformationByHandle};
+
+        let disposition = FILE_DISPOSITION_INFO { DeleteFile: true };
+        let disposition_size = u32::try_from(std::mem::size_of_val(&disposition))
+            .map_err(|_| std::io::Error::other("FILE_DISPOSITION_INFO size exceeds the Win32 API limit"))?;
+        let handle = directory.handle.as_raw_handle();
+        // SAFETY: `handle` is owned by `directory` and stays live for this synchronous
+        // call. `disposition` is initialized with the exact structure and byte size
+        // required by `FileDispositionInfo`; Windows does not retain the pointer.
+        let deleted = unsafe {
+            SetFileInformationByHandle(handle, FileDispositionInfo, std::ptr::from_ref(&disposition).cast(), disposition_size)
+        };
+        if deleted == 0 {
+            Err(std::io::Error::last_os_error())
+        } else {
+            Ok(())
+        }
+    })
+    .await?
+}
+
+#[cfg(windows)]
+struct WindowsEmptyDirectoryFrame {
+    path: PathBuf,
+    directory: LockedEmptyDirectory,
+    entries: fs::ReadDir,
+}
+
+#[cfg(windows)]
+async fn remove_empty_directory_tree_with(
+    root: &Path,
+    mut before_descend: impl FnMut(&Path) -> std::io::Result<()>,
+    mut before_remove: impl FnMut(&Path) -> std::io::Result<()>,
+) -> std::io::Result<()> {
+    let root_directory = lock_windows_empty_directory(root, None).await?;
+    let canonical_root = fs::canonicalize(root).await?;
+    let root_entries = fs::read_dir(root).await?;
+
+    // Holding each validated directory without delete sharing keeps its path
+    // generation stable until handle-relative deletion. State is O(depth).
+    let mut stack = vec![WindowsEmptyDirectoryFrame {
+        path: root.to_path_buf(),
+        directory: root_directory,
+        entries: root_entries,
+    }];
+
+    while let Some(mut frame) = stack.pop() {
+        match frame.entries.next_entry().await {
+            Ok(Some(entry)) => {
+                let child = entry.path();
+                before_descend(&child)?;
+                let child_directory = match lock_windows_empty_directory(&child, Some(&canonical_root)).await {
+                    Ok(directory) => directory,
+                    Err(err) if err.kind() == ErrorKind::NotFound => {
+                        stack.push(frame);
+                        continue;
+                    }
+                    Err(err) => return Err(err),
+                };
+                let child_entries = match fs::read_dir(&child).await {
+                    Ok(entries) => entries,
+                    Err(err) if err.kind() == ErrorKind::NotFound => {
+                        stack.push(frame);
+                        continue;
+                    }
+                    Err(err) if err.kind() == ErrorKind::NotADirectory => {
+                        return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
+                    }
+                    Err(err) => return Err(err),
+                };
+                stack.push(frame);
+                stack.push(WindowsEmptyDirectoryFrame {
+                    path: child,
+                    directory: child_directory,
+                    entries: child_entries,
+                });
+            }
+            Ok(None) => {
+                before_remove(&frame.path)?;
+                drop(frame.entries);
+                match remove_windows_empty_directory(frame.directory).await {
+                    Ok(()) => {}
+                    Err(err) if err.kind() == ErrorKind::NotFound => {}
+                    Err(err) if err.kind() == ErrorKind::NotADirectory => {
+                        return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
+                    }
+                    Err(err) => return Err(err),
+                }
+            }
+            Err(err) if err.kind() == ErrorKind::NotFound => {}
+            Err(err) => return Err(err),
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(windows)]
+async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
+    remove_empty_directory_tree_with(root, |_| Ok(()), |_| Ok(())).await
+}
+
+#[cfg(all(not(unix), not(windows)))]
+async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
+    fs::remove_dir(root).await
+}
+
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_DISK_LOCAL: &str = "disk_local";
 const EVENT_DISK_LOCAL_STARTUP_CLEANUP: &str = "disk_local_startup_cleanup";
@@ -1556,6 +1841,8 @@ static RENAME_DATA_FAIL_COMMIT_RENAME: std::sync::Mutex<Option<String>> = std::s
 #[cfg(test)]
 static LOCAL_INLINE_ROLLBACK_HARDLINK_FAILURE: std::sync::Mutex<Option<PathBuf>> = std::sync::Mutex::new(None);
 #[cfg(test)]
+static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex<Option<(String, PathBuf)>> = std::sync::Mutex::new(None);
+#[cfg(test)]
 static DELETE_VERSION_FAIL_AFTER_DATA_STAGED: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
 #[cfg(test)]
 static DELETE_VERSION_FAIL_AFTER_COMMIT: std::sync::Mutex<Vec<(PathBuf, String)>> = std::sync::Mutex::new(Vec::new());
@@ -1588,6 +1875,13 @@ fn set_local_inline_rollback_hardlink_failure(dst_path: &Path) {
         .expect("test failpoint lock should not be poisoned") = Some(dst_path.to_path_buf());
 }
 
+#[cfg(test)]
+fn set_rename_data_remove_dst_base_before_commit(dst_path: &str, dst_base: &Path) {
+    *RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT
+        .lock()
+        .expect("test failpoint lock should not be poisoned") = Some((dst_path.to_string(), dst_base.to_path_buf()));
+}
+
 #[cfg(test)]
 fn set_delete_version_fail_after_data_staged(path: &str) {
     DELETE_VERSION_FAIL_AFTER_DATA_STAGED
@@ -1656,6 +1950,19 @@ fn should_fail_local_inline_rollback_hardlink(dst_path: &Path) -> bool {
     }
 }
 
+#[cfg(test)]
+fn remove_dst_base_before_commit(dst_path: &str) -> std::io::Result<()> {
+    let mut target = RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT
+        .lock()
+        .expect("test failpoint lock should not be poisoned");
+    let Some((_, base)) = target.as_ref().filter(|(target_path, _)| target_path == dst_path) else {
+        return Ok(());
+    };
+    std::fs::remove_dir_all(base)?;
+    target.take();
+    Ok(())
+}
+
 #[cfg(test)]
 fn should_fail_after_delete_data_staged(path: &str) -> bool {
     let mut targets = DELETE_VERSION_FAIL_AFTER_DATA_STAGED
@@ -1705,6 +2012,11 @@ fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool {
     false
 }
 
+#[cfg(not(test))]
+fn remove_dst_base_before_commit(_dst_path: &str) -> std::io::Result<()> {
+    Ok(())
+}
+
 #[cfg(not(test))]
 fn should_fail_after_delete_data_staged(_path: &str) -> bool {
     false
@@ -4243,9 +4555,11 @@ impl LocalDisk {
         let tmp_path = Self::meta_path(root, RUSTFS_META_TMP_BUCKET);
         let tmp_old_path = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET).join(Uuid::new_v4().to_string());
 
-        rename_all(&tmp_path, &tmp_old_path, root).await.inspect_err(|err| {
-            log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
-        })?;
+        rename_all_ignore_missing_source(&tmp_path, &tmp_old_path, root)
+            .await
+            .inspect_err(|err| {
+                log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
+            })?;
 
         let tmp_deleted_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET);
         tokio::fs::create_dir_all(&tmp_deleted_path).await.inspect_err(|err| {
@@ -7359,6 +7673,10 @@ impl DiskAPI for LocalDisk {
         dst_path: &str,
     ) -> Result<RenameDataResp> {
         crate::hp_guard!("LocalDisk::rename_data");
+        // A non-force DeleteBucket must not remove a directory while a local
+        // object commit is publishing into it. The peer's empty scan remains
+        // optimistic; this guard establishes the local commit/delete order.
+        let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, dst_volume).read_owned().await;
         if fi.is_legacy_indexed_delete_marker() {
             fi.erasure.index = 0;
         }
@@ -7555,6 +7873,7 @@ impl DiskAPI for LocalDisk {
             // sequential version did.
             tmp_meta_res?;
             shard_sync_res?;
+            remove_dst_base_before_commit(dst_path).map_err(to_file_error)?;
 
             if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref()
                 && let Err(err) = rename_all(src_data_path, dst_data_path, &skip_parent).await
@@ -7814,6 +8133,7 @@ impl DiskAPI for LocalDisk {
                 xlmeta.add_version(fi)?;
                 let version_signature = rename_data_versions_signature(&xlmeta);
                 let new_buf = xlmeta.marshal_msg()?;
+                remove_dst_base_before_commit(&dst_path_for_failpoint).map_err(to_file_error)?;
 
                 // Write new xl.meta + rename. Inline objects carry their data
                 // inside xl.meta, so this whole sequence is a metadata commit:
@@ -7838,9 +8158,11 @@ impl DiskAPI for LocalDisk {
                 {
                     let old_path = dst_parent.join(old_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
                     let old_parent = old_path.parent().map(|p| p.to_path_buf());
-                    if let Some(ref old_parent) = old_parent {
-                        std::fs::create_dir_all(old_parent)?;
-                    }
+                    let _old_parent_guard = old_parent
+                        .as_deref()
+                        .map(|parent| os::mkdir_all_below_existing_base_std(parent, &bucket_dir))
+                        .transpose()
+                        .map_err(to_file_error)?;
                     // This rollback backup is the sole restore source for a later
                     // undo_write when the set-level write quorum fails. Persist it as
                     // durably as the new xl.meta written above (and as the non-inline
@@ -7866,6 +8188,12 @@ impl DiskAPI for LocalDisk {
                     local_rollback_path = Some(create_local_inline_rollback_backup(&dst, &src, old_metadata)?);
                 }
 
+                #[cfg(windows)]
+                let _commit_parent_guard = if let Some(parent) = dst.parent() {
+                    Some(os::mkdir_all_below_existing_base_std(parent, &bucket_dir).map_err(to_file_error)?)
+                } else {
+                    None
+                };
                 let commit_result = if should_fail_commit_rename(&dst_path_for_failpoint) {
                     Err(std::io::Error::other("test fail during metadata commit rename"))
                 } else {
@@ -7874,7 +8202,8 @@ impl DiskAPI for LocalDisk {
                         Err(err) if err.kind() == ErrorKind::NotFound && !src.exists() => Ok(()),
                         Err(err) if err.kind() == ErrorKind::NotFound => {
                             if let Some(parent) = dst.parent() {
-                                std::fs::create_dir_all(parent)?;
+                                let _parent_guard =
+                                    os::mkdir_all_below_existing_base_std(parent, &bucket_dir).map_err(to_file_error)?;
                             }
                             std::fs::rename(&src, &dst).map_err(to_file_error)?;
                             Ok(())
@@ -8791,18 +9120,16 @@ impl DiskAPI for LocalDisk {
     #[tracing::instrument(level = "trace", skip_all)]
     async fn delete_volume(&self, volume: &str, force_delete: bool) -> Result<()> {
         let p = self.get_bucket_path(volume)?;
+        let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await;
 
-        // Non-force is non-recursive: `remove_dir` (rmdir) fails atomically with
-        // `DirectoryNotEmpty` -> VolumeNotEmpty if the bucket still holds any
-        // object data, so a misclassified "dangling" bucket on the heal path
-        // (or a non-force S3 DeleteBucket on a populated bucket) can never be
-        // recursively wiped. Only an explicit `force_delete` (e.g. S3 force
-        // bucket delete) removes recursively. Mirrors MinIO's
-        // xlStorage.DeleteVol (Remove vs RemoveAll). (backlog#799 B1)
+        // Non-force removes empty directory remnants children-first with
+        // non-recursive rmdir calls. A file that exists during the scan, or
+        // appears before its parent is removed, fails closed with
+        // VolumeNotEmpty. Only an explicit force delete removes recursively.
         let res = if force_delete {
             fs::remove_dir_all(&p).await
         } else {
-            fs::remove_dir(&p).await
+            remove_empty_directory_tree(&p).await
         };
 
         if let Err(err) = res {
@@ -9061,6 +9388,35 @@ mod test {
         }
     }
 
+    #[cfg(windows)]
+    #[test]
+    fn windows_empty_tree_requires_non_reparse_directory() {
+        validate_windows_empty_directory(0x10).expect("ordinary directories should be accepted");
+        assert!(validate_windows_empty_directory(0).is_err());
+        assert!(validate_windows_empty_directory(0x10 | 0x400).is_err());
+    }
+
+    #[cfg(windows)]
+    #[tokio::test]
+    async fn windows_empty_tree_blocks_replacement_at_final_delete_boundary() {
+        let root = tempfile::tempdir().expect("temporary root should be created");
+        let bucket_path = root.path().join("bucket");
+        let child_path = bucket_path.join("child");
+        fs::create_dir_all(&child_path).await.expect("bucket child should be created");
+        let canonical_root = fs::canonicalize(&bucket_path).await.expect("bucket path should canonicalize");
+        let directory = lock_windows_empty_directory(&child_path, Some(&canonical_root))
+            .await
+            .expect("child directory should be locked");
+
+        std::fs::rename(&child_path, bucket_path.join("replacement"))
+            .expect_err("the locked directory must not be replaceable at the final deletion boundary");
+        remove_windows_empty_directory(directory)
+            .await
+            .expect("handle-relative deletion should remove the locked directory");
+
+        assert!(!child_path.exists(), "the exact locked directory should be removed");
+    }
+
     async fn ensure_test_volume(disk: &LocalDisk, volume: &str) {
         match disk.make_volume(volume).await {
             Ok(()) | Err(DiskError::VolumeExists) => {}
@@ -10787,6 +11143,72 @@ mod test {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial(rename_data_deleted_bucket)]
+    async fn rename_data_non_inline_does_not_recreate_bucket_deleted_before_commit() {
+        let dir = tempfile::tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        let bucket = "deleted-before-non-inline-commit";
+        let object = "prefix/object";
+        let tmp_object = "tmp-non-inline-delete-race";
+        let data_dir = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").expect("data dir should parse");
+        ensure_test_volume(&disk, bucket).await;
+        ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
+
+        let staged_data = disk
+            .get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1"))
+            .expect("staged data path should resolve");
+        fs::create_dir_all(staged_data.parent().expect("staged data should have a parent"))
+            .await
+            .expect("staged data parent should be created");
+        fs::write(&staged_data, b"payload")
+            .await
+            .expect("staged shard should be written");
+
+        let bucket_path = disk.get_bucket_path(bucket).expect("bucket path should resolve");
+        set_rename_data_remove_dst_base_before_commit(object, &bucket_path);
+        let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None);
+        let err = disk
+            .rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
+            .await
+            .expect_err("commit must fail after the destination bucket is deleted");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(!bucket_path.exists(), "rename_data must not recreate the deleted bucket");
+        assert!(staged_data.exists(), "failed commit must preserve the staged shard");
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(rename_data_deleted_bucket)]
+    async fn rename_data_inline_does_not_recreate_bucket_deleted_before_commit() {
+        let dir = tempfile::tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        let bucket = "deleted-before-inline-commit";
+        let object = "prefix/object";
+        let tmp_object = "tmp-inline-delete-race";
+        ensure_test_volume(&disk, bucket).await;
+        ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
+
+        let bucket_path = disk.get_bucket_path(bucket).expect("bucket path should resolve");
+        set_rename_data_remove_dst_base_before_commit(object, &bucket_path);
+        let fi = test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline payload")));
+        let err = disk
+            .rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
+            .await
+            .expect_err("inline commit must fail after the destination bucket is deleted");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(!bucket_path.exists(), "inline rename_data must not recreate the deleted bucket");
+        assert!(
+            disk.get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}"))
+                .expect("staged metadata path should resolve")
+                .exists(),
+            "failed inline commit must preserve staged metadata"
+        );
+    }
+
     #[test]
     fn test_resolve_durability_mode_mapping() {
         // Default: nothing set -> strict (current main behavior).
@@ -12645,6 +13067,19 @@ mod test {
         assert!(format_info.last_check.is_none(), "cached format timestamp should be cleared");
     }
 
+    #[tokio::test]
+    async fn cleanup_tmp_on_startup_allows_missing_tmp_directory() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().expect("operation should succeed");
+
+        LocalDisk::cleanup_tmp_on_startup(dir.path(), Arc::new(AtomicU32::new(0)), Arc::new(Notify::new()))
+            .await
+            .expect("missing temporary directory should already be clean");
+
+        assert!(LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_DELETED_BUCKET).exists());
+    }
+
     #[tokio::test]
     async fn cleanup_tmp_on_startup_moves_existing_tmp_and_recreates_trash() {
         use tempfile::tempdir;
@@ -14517,6 +14952,154 @@ mod test {
         let _ = fs::remove_dir_all(&test_dir).await;
     }
 
+    #[tokio::test]
+    async fn delete_volume_non_force_removes_nested_empty_directories() {
+        let root = tempfile::tempdir().expect("temporary disk root should be created");
+        let endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        let bucket = "nested-empty-bucket";
+
+        disk.make_volume(bucket).await.expect("bucket should be created");
+        fs::create_dir_all(disk.path().join(bucket).join("a/b/c"))
+            .await
+            .expect("nested empty directories should be created");
+
+        disk.delete_volume(bucket, false)
+            .await
+            .expect("non-force delete should remove an empty directory tree");
+
+        assert!(matches!(disk.stat_volume(bucket).await, Err(DiskError::VolumeNotFound)));
+    }
+
+    #[tokio::test]
+    async fn empty_tree_delete_preserves_xlmeta_published_after_scan() {
+        let root = tempfile::tempdir().expect("temporary disk root should be created");
+        let bucket_path = root.path().join("bucket");
+        let object_path = bucket_path.join("object").join(STORAGE_FORMAT_FILE);
+        fs::create_dir_all(object_path.parent().expect("object path should have a parent"))
+            .await
+            .expect("empty object directory should be created");
+
+        let data = b"committed object metadata";
+        let mut published = false;
+        let err = remove_empty_directory_tree_with(
+            &bucket_path,
+            |_| Ok(()),
+            |directory| {
+                if !published && directory == object_path.parent().expect("object path should have a parent") {
+                    std::fs::write(&object_path, data)?;
+                    published = true;
+                }
+                Ok(())
+            },
+        )
+        .await
+        .expect_err("rmdir should refuse metadata published after the directory scan");
+
+        assert!(published, "test barrier should publish metadata before rmdir");
+        assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
+        assert_eq!(std::fs::read(&object_path).expect("object metadata should be preserved"), data);
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn empty_tree_delete_rejects_child_replaced_with_external_symlink() {
+        use std::os::unix::fs::symlink;
+
+        let root = tempfile::tempdir().expect("temporary root should be created");
+        let bucket_path = root.path().join("bucket");
+        let child_path = bucket_path.join("child");
+        let outside_path = root.path().join("outside");
+        let outside_empty = outside_path.join("must-remain");
+        fs::create_dir_all(&child_path).await.expect("bucket child should be created");
+        fs::create_dir_all(&outside_empty)
+            .await
+            .expect("outside directory should be created");
+
+        let mut replaced = false;
+        let err = remove_empty_directory_tree_with(
+            &bucket_path,
+            |child| {
+                if !replaced && child == child_path {
+                    std::fs::remove_dir(&child_path)?;
+                    symlink(&outside_path, &child_path)?;
+                    replaced = true;
+                }
+                Ok(())
+            },
+            |_| Ok(()),
+        )
+        .await
+        .expect_err("a replaced child must fail closed");
+
+        assert!(replaced, "test barrier should replace the child before it is opened");
+        assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
+        assert!(outside_empty.exists(), "bucket deletion must not remove directories outside the bucket");
+        assert!(
+            std::fs::symlink_metadata(&child_path)
+                .expect("replacement symlink should remain")
+                .file_type()
+                .is_symlink()
+        );
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn empty_tree_delete_rechecks_parent_after_child_disappears() {
+        let root = tempfile::tempdir().expect("temporary root should be created");
+        let bucket_path = root.path().join("bucket");
+        let child_path = bucket_path.join("child");
+        fs::create_dir_all(&child_path).await.expect("bucket child should be created");
+
+        let mut removed = false;
+        remove_empty_directory_tree_with(
+            &bucket_path,
+            |child| {
+                if !removed && child == child_path {
+                    std::fs::remove_dir(&child_path)?;
+                    removed = true;
+                }
+                Ok(())
+            },
+            |_| Ok(()),
+        )
+        .await
+        .expect("a vanished empty child should not leave the bucket root behind");
+
+        assert!(removed, "test hook should remove the child before openat");
+        assert!(!bucket_path.exists(), "parent should be rechecked and removed after the child disappears");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn empty_tree_delete_rejects_root_generation_replacement() {
+        let root = tempfile::tempdir().expect("temporary root should be created");
+        let bucket_path = root.path().join("bucket");
+        let moved_path = root.path().join("bucket-before-replacement");
+        fs::create_dir(&bucket_path).await.expect("bucket should be created");
+
+        let mut replaced = false;
+        let err = remove_empty_directory_tree_with(
+            &bucket_path,
+            |_| Ok(()),
+            |directory| {
+                if !replaced && directory == bucket_path {
+                    std::fs::rename(&bucket_path, &moved_path)?;
+                    std::fs::create_dir(&bucket_path)?;
+                    replaced = true;
+                }
+                Ok(())
+            },
+        )
+        .await
+        .expect_err("a replacement root directory must fail closed");
+
+        assert!(replaced, "test hook should replace the root generation before rmdir");
+        assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
+        assert!(moved_path.exists(), "the originally scanned root should not be removed by its old name");
+        assert!(bucket_path.exists(), "the replacement root should remain after identity validation fails");
+    }
+
     #[tokio::test]
     async fn test_local_disk_volume_operations() {
         let test_dir = "./test_local_disk_volumes";

crates/ecstore/src/disk/os.rs

@@ -25,7 +25,7 @@ use std::{
     sync::{Arc, LazyLock, Weak},
 };
 use tokio::fs;
-use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
+use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
 use tracing::warn;
 
 /// Check path length according to OS limits.
@@ -123,6 +123,8 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
 
 fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
     let cpu_scaled = cpu_count
@@ -157,6 +159,24 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
     limiter
 }
 
+/// Serialize a bucket's local metadata commits with physical bucket removal.
+///
+/// The key includes the canonical disk root, so independently reconnected
+/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
+/// disconnected disks do not keep the registry alive.
+pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
+    let key = root.join(volume);
+    let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
+    locks.retain(|_, lock| lock.strong_count() > 0);
+    if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
+        return lock;
+    }
+
+    let lock = Arc::new(RwLock::new(()));
+    locks.insert(key, Arc::downgrade(&lock));
+    lock
+}
+
 /// Always acquire the per-disk permit before the process-wide permit. Keeping
 /// this order uniform prevents one slow disk from reserving global capacity
 /// while it waits for its own concurrency slot.
@@ -572,15 +592,14 @@ async fn reliable_rename_inner(
     base_dir: impl AsRef<Path>,
     warn_on_missing_source: bool,
 ) -> io::Result<()> {
-    if let Some(parent) = dst_file_path.as_ref().parent()
-        && !file_exists(parent)
-    {
-        reliable_mkdir_all(parent, base_dir.as_ref()).await?;
-    }
+    let parent_guard = match dst_file_path.as_ref().parent() {
+        Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
+        None => None,
+    };
 
     let mut i = 0;
     loop {
-        if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
+        if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
             if should_retry_rename(&e, i) {
                 i += 1;
                 continue;
@@ -597,6 +616,159 @@ async fn reliable_rename_inner(
     Ok(())
 }
 
+#[cfg(unix)]
+fn rename_into_existing_parent(
+    src_file_path: &Path,
+    dst_file_path: &Path,
+    parent_guard: Option<&ExistingBaseDirectoryGuard>,
+) -> io::Result<()> {
+    use rustix::fs::{Mode, OFlags, open, renameat};
+
+    let Some(parent_guard) = parent_guard else {
+        return super::fs::rename_std(src_file_path, dst_file_path);
+    };
+    let src_parent = src_file_path
+        .parent()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
+    let src_name = src_file_path
+        .file_name()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
+    let dst_name = dst_file_path
+        .file_name()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
+    let src_parent = open(
+        src_parent,
+        OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+        Mode::empty(),
+    )
+    .map_err(io::Error::from)?;
+    let dst_parent = parent_guard
+        .last()
+        .ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
+
+    renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
+}
+
+#[cfg(not(unix))]
+fn rename_into_existing_parent(
+    src_file_path: &Path,
+    dst_file_path: &Path,
+    _parent_guard: Option<&ExistingBaseDirectoryGuard>,
+) -> io::Result<()> {
+    super::fs::rename_std(src_file_path, dst_file_path)
+}
+
+async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
+    let dir_path = dir_path.to_path_buf();
+    let base_dir = base_dir.to_path_buf();
+
+    tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
+}
+
+#[cfg(windows)]
+pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
+
+#[cfg(unix)]
+pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
+
+#[cfg(all(not(unix), not(windows)))]
+pub(crate) type ExistingBaseDirectoryGuard = ();
+
+#[cfg(windows)]
+fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
+    use std::os::windows::fs::OpenOptionsExt;
+
+    const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
+    const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
+    const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
+    const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
+    const FILE_SHARE_READ: u32 = 0x1;
+
+    let file = std::fs::OpenOptions::new()
+        .read(true)
+        .share_mode(FILE_SHARE_READ)
+        .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
+        .open(path)?;
+    let handle = winapi_util::Handle::from_file(file);
+    let info = winapi_util::file::information(&handle)?;
+    if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
+        || info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
+    {
+        return Err(io::Error::from(io::ErrorKind::NotADirectory));
+    }
+    Ok(handle)
+}
+
+pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
+    let relative = dir_path
+        .strip_prefix(base_dir)
+        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
+    for component in relative.components() {
+        if !matches!(component, Component::Normal(_) | Component::CurDir) {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "rename destination contains an invalid path component",
+            ));
+        }
+    }
+
+    #[cfg(unix)]
+    {
+        use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
+        use rustix::io::Errno;
+
+        let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
+        let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
+        let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
+
+        for component in relative.components() {
+            let Component::Normal(component) = component else {
+                continue;
+            };
+            let parent = parents
+                .last()
+                .expect("base directory guard should contain the base directory");
+            match mkdirat(parent, component, mode) {
+                Ok(()) => {}
+                Err(Errno::EXIST) => {}
+                Err(err) => return Err(err.into()),
+            }
+            parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
+        }
+
+        Ok(parents)
+    }
+
+    #[cfg(windows)]
+    {
+        let mut handles = vec![lock_windows_directory(base_dir)?];
+        let mut current = base_dir.to_path_buf();
+        for component in relative.components() {
+            let Component::Normal(component) = component else {
+                continue;
+            };
+            current.push(component);
+            match std::fs::create_dir(&current) {
+                Ok(()) => {}
+                Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
+                Err(err) => return Err(err),
+            }
+            handles.push(lock_windows_directory(&current)?);
+        }
+
+        Ok(handles)
+    }
+
+    #[cfg(all(not(unix), not(windows)))]
+    {
+        let _ = relative;
+        Err(io::Error::new(
+            io::ErrorKind::Unsupported,
+            "safe recursive directory creation is unavailable on this platform",
+        ))
+    }
+}
+
 fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
     warn!(
         "reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
@@ -727,6 +899,20 @@ mod tests {
         Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
     }
 
+    #[tokio::test]
+    async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
+        let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
+        let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
+
+        assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
+        assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
+
+        let _write_guard = first.write().await;
+        assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
+    }
+
     #[derive(Clone, Default)]
     struct CapturedLogs {
         buffer: Arc<Mutex<Vec<u8>>>,
@@ -771,9 +957,26 @@ mod tests {
         }
     }
 
+    /// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
+    /// the pin that keeps tracing's process-global callsite-interest cache from
+    /// being decided by some other test's thread.
+    struct WarnCaptureGuard {
+        _subscriber: tracing::subscriber::DefaultGuard,
+        _callsite_pin: tracing::Dispatch,
+    }
+
     /// Capture WARN-level output on the current thread; tokio tests here run on
     /// the current-thread runtime, so the guard covers the whole test body.
-    fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
+    ///
+    /// The callsite pin matters because `warn_reliable_rename_failure` is a
+    /// single production callsite shared with tests that call `rename_all`
+    /// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
+    /// is one. Whichever thread reaches it first fixes its `Interest`
+    /// process-wide, so without the pin that sibling can cache
+    /// `Interest::never()` and the WARN never fires here at all, leaving the
+    /// "must keep the WARN" assertions staring at empty output. See
+    /// [`crate::test_tracing::pin_callsite_interest_for_test`].
+    fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
         let logs = CapturedLogs::default();
         let subscriber = tracing_subscriber::fmt()
             .with_max_level(tracing::Level::WARN)
@@ -781,7 +984,10 @@ mod tests {
             .with_ansi(false)
             .without_time()
             .finish();
-        let guard = tracing::subscriber::set_default(subscriber);
+        let guard = WarnCaptureGuard {
+            _subscriber: tracing::subscriber::set_default(subscriber),
+            _callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
+        };
         (logs, guard)
     }
 
@@ -968,6 +1174,112 @@ mod tests {
         assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
     }
 
+    #[tokio::test]
+    async fn rename_all_does_not_recreate_missing_base_directory() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        std::fs::create_dir(&base).expect("create destination base");
+        let src = temp_dir.path().join("staged-object");
+        std::fs::write(&src, b"payload").expect("write staged object");
+        let dst = base.join("object").join("xl.meta");
+        std::fs::remove_dir(&base).expect("delete destination base before commit");
+
+        let err = rename_all(&src, &dst, &base)
+            .await
+            .expect_err("rename must not recreate a deleted destination base");
+
+        assert!(matches!(err, DiskError::FileNotFound));
+        assert!(src.exists(), "failed commit must preserve the staged source");
+        assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir(&base).expect("create destination base");
+        std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
+        let src = temp_dir.path().join("staged-object");
+        std::fs::write(&src, b"payload").expect("write staged object");
+        let dst = base.join("object").join("xl.meta");
+
+        std::fs::remove_dir(&base).expect("remove destination base before replacement");
+        symlink(&outside, &base).expect("replace destination base with a symlink");
+
+        rename_all(&src, &dst, &base)
+            .await
+            .expect_err("rename must reject an existing destination parent below a replaced base");
+
+        assert!(src.exists(), "rejected rename must preserve the staged source");
+        assert!(
+            !outside.join("object/xl.meta").exists(),
+            "rename must not publish through the replacement symlink"
+        );
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        std::fs::create_dir(&base).expect("create destination base");
+        let parent = base.join("object").join("nested");
+        let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
+
+        std::fs::rename(&base, temp_dir.path().join("replacement-base"))
+            .expect_err("the locked base must not be replaceable before commit");
+        std::fs::rename(base.join("object"), base.join("replacement-object"))
+            .expect_err("a locked intermediate directory must not be replaceable before commit");
+
+        drop(guard);
+        std::fs::rename(base.join("object"), base.join("replacement-object"))
+            .expect("replacement should succeed after the commit guard is released");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_parent_creation_rejects_symlinked_base() {
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir(&outside).expect("create outside directory");
+        let base = temp_dir.path().join("bucket");
+        symlink(&outside, &base).expect("create symlinked base");
+
+        mkdir_all_below_existing_base(&base.join("object"), &base)
+            .await
+            .expect_err("symlinked base must be rejected");
+
+        assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn rename_parent_creation_rejects_symlink_below_base() {
+        use std::os::unix::fs::symlink;
+
+        let temp_dir = tempdir().expect("create temp dir");
+        let base = temp_dir.path().join("bucket");
+        let outside = temp_dir.path().join("outside");
+        std::fs::create_dir(&base).expect("create destination base");
+        std::fs::create_dir(&outside).expect("create outside directory");
+        symlink(&outside, base.join("linked")).expect("create symlink below base");
+
+        mkdir_all_below_existing_base(&base.join("linked/object"), &base)
+            .await
+            .expect_err("symlink below base must be rejected");
+
+        assert!(
+            !outside.join("object").exists(),
+            "parent creation must not follow a symlink outside the base"
+        );
+    }
+
     #[tokio::test]
     async fn fsync_dir_succeeds_on_directory() {
         let temp_dir = tempdir().expect("create temp dir");

crates/ecstore/src/lib.rs

@@ -93,3 +93,6 @@ pub(crate) mod ecstore_validation_blackbox;
 
 #[cfg(test)]
 pub(crate) mod test_metrics;
+
+#[cfg(test)]
+pub(crate) mod test_tracing;

crates/ecstore/src/services/notification_sys.rs

@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use rustfs_utils::XHost;
-use std::collections::{HashMap, hash_map::DefaultHasher};
+use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
 use std::future::Future;
 use std::hash::{Hash, Hasher};
 use std::sync::{Arc, Mutex, OnceLock};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
@@ -47,6 +47,9 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
 const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
 const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
+const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
+const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
+const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 
 /// Cached result from the last successful admin call to a peer.
 struct PeerAdminCache {
@@ -91,6 +94,180 @@ lazy_static! {
     pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
 }
 
+#[derive(Clone)]
+struct RemoteVersionStateFleetProof {
+    topology_fingerprint: String,
+    peer_epochs: Arc<BTreeMap<String, Uuid>>,
+    expires_at: Instant,
+}
+
+impl RemoteVersionStateFleetProof {
+    fn token(&self) -> RemoteVersionStateFleetProofToken {
+        RemoteVersionStateFleetProofToken {
+            topology_fingerprint: self.topology_fingerprint.clone(),
+            peer_epochs: self.peer_epochs.clone(),
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub(crate) struct RemoteVersionStateFleetProofToken {
+    topology_fingerprint: String,
+    peer_epochs: Arc<BTreeMap<String, Uuid>>,
+}
+
+#[derive(Default)]
+struct RemoteVersionStateFleetProofState {
+    proof: Option<RemoteVersionStateFleetProof>,
+    topology_conflict: bool,
+}
+
+static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
+static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
+
+fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
+    REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
+}
+
+fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
+    replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
+}
+
+fn replace_remote_version_state_fleet_proof_in(
+    slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
+    proof: Option<RemoteVersionStateFleetProof>,
+) {
+    slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
+}
+
+fn publish_remote_version_state_probe_result(
+    slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
+    topology_fingerprint: &str,
+    result: Result<BTreeMap<String, Uuid>>,
+    observed_at: Instant,
+) -> Option<Error> {
+    match result {
+        Ok(peer_epochs) => {
+            let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+            let peer_epochs = state
+                .proof
+                .as_ref()
+                .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
+                .map(|proof| Arc::clone(&proof.peer_epochs))
+                .unwrap_or_else(|| Arc::new(peer_epochs));
+            state.proof = Some(RemoteVersionStateFleetProof {
+                topology_fingerprint: topology_fingerprint.to_string(),
+                peer_epochs,
+                expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
+            });
+            None
+        }
+        Err(err) => {
+            replace_remote_version_state_fleet_proof_in(slot, None);
+            Some(err)
+        }
+    }
+}
+
+pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = remote_version_state_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
+}
+
+fn acquire_remote_version_state_fleet_proof_from(
+    state: &RemoteVersionStateFleetProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<RemoteVersionStateFleetProofToken> {
+    if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
+        return None;
+    }
+    state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
+}
+
+pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    let state = remote_version_state_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    if state.topology_conflict {
+        return false;
+    }
+    state.proof.as_ref().is_some_and(|current| {
+        current.topology_fingerprint == *expected_topology
+            && current.topology_fingerprint == proof.topology_fingerprint
+            && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
+            && Instant::now() < current.expires_at
+    })
+}
+
+fn remote_version_state_fleet_proof_valid_at(
+    proof: Option<&RemoteVersionStateFleetProof>,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
+}
+
+fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
+    if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
+        return Err(Error::other("remote version state capability peer identity is invalid"));
+    }
+    Ok(())
+}
+
+pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
+    if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
+        if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
+            let mut state = remote_version_state_fleet_proof_slot()
+                .write()
+                .unwrap_or_else(std::sync::PoisonError::into_inner);
+            state.topology_conflict = true;
+            state.proof = None;
+        }
+        return;
+    }
+
+    tokio::spawn(async move {
+        loop {
+            let result = match get_global_notification_sys() {
+                Some(notification_sys) => {
+                    match timeout(
+                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                        notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
+                    )
+                    .await
+                    {
+                        Ok(result) => result,
+                        Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
+                    }
+                }
+                None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
+            };
+            let topology_conflict = remote_version_state_fleet_proof_slot()
+                .read()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .topology_conflict;
+            if topology_conflict {
+                replace_remote_version_state_fleet_proof(None);
+            } else if let Some(err) = publish_remote_version_state_probe_result(
+                remote_version_state_fleet_proof_slot(),
+                &topology_fingerprint,
+                result,
+                Instant::now(),
+            ) {
+                debug!(error = %err, "remote version state fleet capability probe failed closed");
+            }
+            sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
+        }
+    });
+}
+
 pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
     let _ = GLOBAL_NOTIFICATION_SYS
         .set(Arc::new(NotificationSys::new(eps).await))
@@ -115,7 +292,17 @@ pub struct NotificationSys {
 
 impl NotificationSys {
     pub async fn new(eps: EndpointServerPools) -> Self {
+        let expected_remote_hosts = eps
+            .peer_grid_host_slots_sorted()
+            .into_iter()
+            .filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
+            .collect::<Vec<_>>();
         let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
+        let peer_topology_hosts = if peer_topology_hosts.is_empty() {
+            expected_remote_hosts
+        } else {
+            peer_topology_hosts
+        };
         let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
         Self {
             peer_clients,
@@ -125,6 +312,24 @@ impl NotificationSys {
             tier_config_reload_workers: Default::default(),
         }
     }
+
+    async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
+        if self.peer_clients.len() != self.peer_topology_hosts.len() {
+            return Err(Error::other("remote version state capability fleet membership is incomplete"));
+        }
+        let probes = self.peer_clients.iter().map(|client| async {
+            let client = client
+                .as_ref()
+                .ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
+            client.probe_remote_version_state(topology_fingerprint.to_string()).await
+        });
+        let mut peer_epochs = BTreeMap::new();
+        for result in join_all(probes).await {
+            let (peer, epoch) = result?;
+            insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
+        }
+        Ok(peer_epochs)
+    }
 }
 
 pub struct NotificationPeerErr {
@@ -1890,6 +2095,183 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 mod tests {
     use super::*;
 
+    #[test]
+    fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
+        let now = Instant::now();
+        let mut peer_epochs = BTreeMap::new();
+        peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(peer_epochs),
+            expires_at: now + Duration::from_secs(1),
+        };
+
+        assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
+        assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
+        assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
+        assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
+        let mut peer_epochs = BTreeMap::new();
+
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
+        assert!(peer_epochs.is_empty());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_accepts_single_node_membership() {
+        let now = Instant::now();
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(BTreeMap::new()),
+            expires_at: now + Duration::from_secs(1),
+        };
+
+        assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
+        let now = Instant::now();
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            expires_at: now + Duration::from_secs(1),
+        };
+        let captured = proof.token();
+        let restarted = RemoteVersionStateFleetProof {
+            topology_fingerprint: proof.topology_fingerprint.clone(),
+            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            expires_at: proof.expires_at,
+        };
+
+        assert!(captured != restarted.token());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
+        let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
+        let now = Instant::now();
+        let epoch = Uuid::new_v4();
+        let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
+        let original = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("successful probe should publish proof")
+            .token();
+
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
+        );
+        let renewed = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("renewal should retain proof")
+            .token();
+        assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
+
+        let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
+                .is_none()
+        );
+        let replaced = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("restarted peer should publish a new proof")
+            .token();
+        assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
+        let now = Instant::now();
+        let mut state = RemoteVersionStateFleetProofState {
+            proof: Some(RemoteVersionStateFleetProof {
+                topology_fingerprint: "topology-a".to_string(),
+                peer_epochs: Arc::new(BTreeMap::new()),
+                expires_at: now + Duration::from_secs(1),
+            }),
+            topology_conflict: false,
+        };
+        assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
+
+        state.topology_conflict = true;
+        assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
+        let epoch = Uuid::new_v4();
+        let mut peer_epochs = BTreeMap::new();
+        insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
+            .expect("first member should be admitted");
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
+        let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
+        let now = Instant::now();
+        let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
+        assert!(slot.read().expect("proof slot should not poison").proof.is_some());
+
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
+        );
+        assert!(slot.read().expect("proof slot should not poison").proof.is_none());
+
+        let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
+        assert!(slot.read().expect("proof slot should not poison").proof.is_some());
+    }
+
+    #[tokio::test]
+    async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
+        let notification_sys = NotificationSys {
+            peer_clients: vec![None],
+            all_peer_clients: vec![None, None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+            tier_config_reload_workers: Default::default(),
+        };
+
+        let err = notification_sys
+            .probe_remote_version_state_fleet("topology-a")
+            .await
+            .expect_err("an unreachable configured member must fail the fleet proof");
+        assert!(err.to_string().contains("unreachable"));
+    }
+
+    #[tokio::test]
+    async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
+        let notification_sys = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: Vec::new(),
+            tier_config_reload_workers: Default::default(),
+        };
+
+        let err = notification_sys
+            .probe_remote_version_state_fleet("topology-a")
+            .await
+            .expect_err("a missing configured member slot must fail the fleet proof");
+        assert!(err.to_string().contains("incomplete"));
+    }
+
     fn build_props(endpoint: &str) -> ServerProperties {
         ServerProperties {
             endpoint: endpoint.to_string(),

crates/ecstore/src/services/tier/tier.rs

@@ -236,6 +236,7 @@ struct TierPublishTransition {
 
 struct PreparedTierDriver {
     tier_name: String,
+    tier_config: TierConfig,
     config_fingerprint: TierDriverFingerprint,
     backend_identity: TierDestinationId,
     exact_get_delete: bool,
@@ -1342,6 +1343,7 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
 
 struct TierDriverGeneration {
     tier_name: Arc<str>,
+    tier_config: TierConfig,
     generation: DriverRevision,
     // Process-local only: this may reflect credential changes and must never be persisted or logged.
     config_fingerprint: TierDriverFingerprint,
@@ -1349,6 +1351,9 @@ struct TierDriverGeneration {
     backend_identity: TierDestinationId,
     exact_get_delete: bool,
     driver: SharedWarmBackend,
+    reconciler: tokio::sync::OnceCell<
+        Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
+    >,
     accepting: AtomicBool,
     active_leases: AtomicUsize,
     drained: tokio::sync::Notify,
@@ -1473,6 +1478,35 @@ impl TierOperationLease {
         self.inner.backend_identity
     }
 
+    pub(crate) async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        transaction_id: uuid::Uuid,
+    ) -> io::Result<TransitionCandidateProbe> {
+        let Some(reconciler) = self
+            .inner
+            .reconciler
+            .get_or_try_init(|| async {
+                crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
+                    .await
+                    .map(|reconciler| reconciler.map(Arc::from))
+            })
+            .await
+            .map_err(|err| io::Error::other(err.message))?
+        else {
+            return self.inner.driver.probe_transition_candidate(object).await;
+        };
+        reconciler
+            .probe_transition_candidate_for(
+                object,
+                crate::services::tier::warm_backend::TransitionCandidateIdentity {
+                    transaction_id,
+                    destination_id: self.backend_identity(),
+                },
+            )
+            .await
+    }
+
     pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
         self.inner.driver.validate_remote_version_id(remote_version_id)?;
         if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
@@ -2833,6 +2867,7 @@ impl TierConfigMgr {
                     let exact_get_delete = tier_exact_get_delete(config);
                     Some(PreparedTierDriver {
                         tier_name: tier_name.to_string(),
+                        tier_config: config.clone(),
                         config_fingerprint,
                         backend_identity,
                         exact_get_delete,
@@ -2861,11 +2896,13 @@ impl TierConfigMgr {
             })?;
             let entry = Arc::new(TierDriverGeneration {
                 tier_name: Arc::from(prepared.tier_name.as_str()),
+                tier_config: prepared.tier_config.clone(),
                 generation,
                 config_fingerprint: prepared.config_fingerprint,
                 backend_identity: prepared.backend_identity,
                 exact_get_delete: prepared.exact_get_delete,
                 driver: prepared.driver.clone(),
+                reconciler: tokio::sync::OnceCell::new(),
                 accepting: AtomicBool::new(true),
                 active_leases: AtomicUsize::new(0),
                 drained: tokio::sync::Notify::new(),
@@ -3609,11 +3646,13 @@ impl TierConfigMgr {
         let driver: SharedWarmBackend = Arc::from(driver);
         let entry = Arc::new(TierDriverGeneration {
             tier_name: Arc::from(tier_name),
+            tier_config: config.clone(),
             generation,
             config_fingerprint,
             backend_identity,
             exact_get_delete,
             driver: driver.clone(),
+            reconciler: tokio::sync::OnceCell::new(),
             accepting: AtomicBool::new(true),
             active_leases: AtomicUsize::new(0),
             drained: tokio::sync::Notify::new(),

crates/ecstore/src/services/tier/warm_backend.rs

@@ -73,6 +73,21 @@ pub enum TransitionCandidateProbe {
     Unsupported,
 }
 
+#[derive(Clone, Copy)]
+pub(crate) struct TransitionCandidateIdentity {
+    pub transaction_id: uuid::Uuid,
+    pub destination_id: [u8; 32],
+}
+
+#[async_trait::async_trait]
+pub(crate) trait TransitionCandidateReconciler {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error>;
+}
+
 #[async_trait::async_trait]
 pub trait WarmBackend {
     async fn validate(&self) -> Result<(), std::io::Error> {
@@ -189,6 +204,20 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
         metadata.remove(key);
     }
 
+    for suffix in [
+        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+    ] {
+        for key in [
+            rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
+            format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
+        ] {
+            if let Some(value) = metadata.remove(&key) {
+                metadata.insert(format!("x-amz-meta-{key}"), value);
+            }
+        }
+    }
+
     opts.user_metadata = metadata;
     opts
 }
@@ -446,6 +475,51 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
     Ok(d)
 }
 
+pub(crate) async fn new_transition_candidate_reconciler(
+    tier: &TierConfig,
+) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
+    let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
+        TierType::S3 => Box::new(
+            WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::MinIO => Box::new(
+            WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::RustFS => Box::new(
+            WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::R2 => Box::new(
+            WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        _ => return Ok(None),
+    };
+    Ok(Some(reconciler))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -775,6 +849,37 @@ mod tests {
         assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
     }
 
+    #[test]
+    fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            "5a".repeat(32),
+        );
+
+        let opts = build_transition_put_options("COLD".to_string(), metadata);
+
+        for suffix in [
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+        ] {
+            assert!(opts.user_metadata.contains_key(&format!(
+                "x-amz-meta-{}",
+                rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
+            )));
+            assert!(opts.user_metadata.contains_key(&format!(
+                "x-amz-meta-{}{suffix}",
+                rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
+            )));
+        }
+    }
+
     #[test]
     fn build_transition_put_options_requests_no_checksum_and_content_md5() {
         // Regression for rustfs/rustfs#4811: transition uploads must leave the

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendMinIO {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_r2.rs

@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendR2 {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -132,6 +132,20 @@ impl WarmBackend for WarmBackendRustFS {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -37,7 +37,10 @@ use crate::error::ErrorResponse;
 use crate::error::error_resp_to_object_err;
 use crate::services::tier::{
     tier_config::TierS3,
-    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{
+        TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
+        build_transition_put_options,
+    },
 };
 use http::HeaderMap;
 use rustfs_utils::egress::validate_outbound_url;
@@ -219,6 +222,92 @@ impl WarmBackendS3 {
             advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
         }
     }
+
+    async fn probe_transition_candidate_identity(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+        bucket_versioning: RemoteBucketVersioning,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
+        let mut opts = ListObjectsOptions::default();
+        opts.set("prefix", &remote_object);
+        opts.set("max-keys", "1000");
+        let mut key_marker = String::new();
+        let mut version_id_marker = String::new();
+        let mut matched_version = None;
+        let mut saw_unproven_candidate = false;
+
+        loop {
+            let versions = self
+                .client
+                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
+                .await?;
+            for version in versions.versions.iter().filter(|version| version.key == remote_object) {
+                let mut stat_opts = GetObjectOptions::default();
+                stat_opts.version_id.clone_from(&version.version_id);
+                let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
+                let mut metadata = info.user_metadata;
+                for (name, value) in &info.metadata {
+                    if (name
+                        .as_str()
+                        .starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
+                        || name
+                            .as_str()
+                            .starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
+                        && let Ok(value) = value.to_str()
+                    {
+                        metadata.insert(name.as_str().to_string(), value.to_string());
+                    }
+                }
+                if transition_candidate_metadata_matches(&metadata, identity)? {
+                    if matched_version.is_some() {
+                        return Ok(TransitionCandidateProbe::Ambiguous);
+                    }
+                    matched_version = Some(version.version_id.clone());
+                } else {
+                    saw_unproven_candidate = true;
+                }
+            }
+            if !versions.is_truncated {
+                if matched_version.is_none() && saw_unproven_candidate {
+                    return Ok(TransitionCandidateProbe::Unsupported);
+                }
+                let candidates = TransitionCandidateVersions {
+                    version_id: matched_version,
+                    ambiguous: false,
+                };
+                return classify_transition_candidates(candidates, bucket_versioning);
+            }
+            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
+        }
+    }
+}
+
+fn transition_candidate_metadata_matches(
+    metadata: &HashMap<String, String>,
+    identity: TransitionCandidateIdentity,
+) -> Result<bool, std::io::Error> {
+    use rustfs_utils::http::metadata_compat::{
+        SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
+    };
+
+    let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
+    let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
+    if transaction_id.is_none() || destination_id.is_none() {
+        if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
+            || contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
+        {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                "transition candidate identity metadata is empty or conflicting",
+            ));
+        }
+        return Ok(false);
+    }
+    let expected_transaction_id = identity.transaction_id.to_string();
+    let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
+    Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
 }
 
 fn classify_transition_candidates(
@@ -342,6 +431,60 @@ mod tests {
         candidates.classify(bucket_versioning)
     }
 
+    fn candidate_identity() -> TransitionCandidateIdentity {
+        TransitionCandidateIdentity {
+            transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
+            destination_id: [0x5a; 32],
+        }
+    }
+
+    fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            identity.transaction_id.to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(identity.destination_id),
+        );
+        metadata
+    }
+
+    #[test]
+    fn transition_candidate_identity_requires_exact_compatible_metadata() {
+        let identity = candidate_identity();
+        let metadata = candidate_metadata(identity);
+        assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
+
+        let mut adjacent = metadata;
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut adjacent,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
+                .unwrap()
+                .to_string(),
+        );
+        assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
+
+        assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
+    }
+
+    #[test]
+    fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
+        let identity = candidate_identity();
+        let mut metadata = candidate_metadata(identity);
+        metadata.insert(
+            rustfs_utils::http::metadata_compat::internal_key_rustfs(
+                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            ),
+            uuid::Uuid::new_v4().to_string(),
+        );
+        assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
+    }
+
     #[test]
     fn transition_candidate_probe_classifier_is_fail_closed() {
         assert_eq!(
@@ -503,3 +646,16 @@ impl WarmBackend for WarmBackendS3 {
         Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
     }
 }
+
+#[async_trait::async_trait]
+impl TransitionCandidateReconciler for WarmBackendS3 {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let bucket_versioning = self.remote_bucket_versioning().await?;
+        self.probe_transition_candidate_identity(object, identity, bucket_versioning)
+            .await
+    }
+}

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -3379,85 +3379,98 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn complete_holds_object_then_upload_lock_through_commit() {
-        temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
-            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
-            let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
-            let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
-            let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
-            let bucket = "multipart-layout-lock-order-bucket";
-            let object = "object";
-            make_bucket_on_all(&disk_stores, bucket).await;
-            let create_opts = ObjectOptions {
-                user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
-                ..Default::default()
-            };
-            let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
-            let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
-            let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
-            let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
-            signaling.set_target(upload_resource.clone());
-            signaling.clear_observed();
-            let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
-            let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
+        temp_env::async_with_vars(
+            [
+                (crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
+                (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
+            ],
+            async {
+                let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
+                let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
+                let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
+                let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+                let bucket = "multipart-layout-lock-order-bucket";
+                let object = "object";
+                make_bucket_on_all(&disk_stores, bucket).await;
+                let create_opts = ObjectOptions {
+                    user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
+                    ..Default::default()
+                };
+                let (upload_id, parts) =
+                    stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
+                let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
+                let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
+                let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
+                signaling.set_target(upload_resource.clone());
+                signaling.clear_observed();
+                let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+                let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
 
-            let complete_store = set_disks.clone();
-            let complete_upload_id = upload_id.clone();
-            let complete = tokio::spawn(async move {
-                complete_store
-                    .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
+                let complete_store = set_disks.clone();
+                let complete_upload_id = upload_id.clone();
+                let complete = tokio::spawn(async move {
+                    complete_store
+                        .complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
+                        .await
+                });
+                barrier.wait_until_paused().await;
+
+                let observed = signaling.observed();
+                let object_position = observed
+                    .iter()
+                    .position(|resource| resource == &object_resource)
+                    .expect("completion should acquire the object lock");
+                let upload_position = observed
+                    .iter()
+                    .position(|resource| resource == &upload_resource)
+                    .expect("completion should acquire the upload lock");
+                assert!(object_position < upload_position, "completion must acquire object before upload");
+
+                let abort_store = set_disks.clone();
+                let abort_upload_id = upload_id.clone();
+                let abort = tokio::spawn(async move {
+                    abort_store
+                        .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
+                        .await
+                });
+                signaling.wait_for_attempts(2).await;
+                tokio::task::yield_now().await;
+                assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
+
+                let list_store = set_disks.clone();
+                let list_upload_id = upload_id.clone();
+                let list = tokio::spawn(async move {
+                    list_store
+                        .list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
+                        .await
+                });
+                signaling.wait_for_attempts(3).await;
+                tokio::task::yield_now().await;
+                assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
+
+                barrier.release();
+                complete
                     .await
-            });
-            barrier.wait_until_paused().await;
-
-            let observed = signaling.observed();
-            let object_position = observed
-                .iter()
-                .position(|resource| resource == &object_resource)
-                .expect("completion should acquire the object lock");
-            let upload_position = observed
-                .iter()
-                .position(|resource| resource == &upload_resource)
-                .expect("completion should acquire the upload lock");
-            assert!(object_position < upload_position, "completion must acquire object before upload");
-
-            let abort_store = set_disks.clone();
-            let abort_upload_id = upload_id.clone();
-            let abort = tokio::spawn(async move {
-                abort_store
-                    .abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
+                    .expect("completion task should not panic")
+                    .expect("completion should commit after the barrier is released");
+                let abort_err = abort
                     .await
-            });
-            signaling.wait_for_attempts(2).await;
-            tokio::task::yield_now().await;
-            assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
-
-            let list_store = set_disks.clone();
-            let list_upload_id = upload_id.clone();
-            let list = tokio::spawn(async move {
-                list_store
-                    .list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
+                    .expect("abort task should not panic")
+                    .expect_err("the committed upload should no longer exist when abort acquires the lock");
+                assert!(
+                    matches!(abort_err, StorageError::InvalidUploadID(..)),
+                    "abort should return InvalidUploadID after completion, got {abort_err:?}"
+                );
+                let list_err = list
                     .await
-            });
-            signaling.wait_for_attempts(3).await;
-            tokio::task::yield_now().await;
-            assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
-
-            barrier.release();
-            complete
-                .await
-                .expect("completion task should not panic")
-                .expect("completion should commit after the barrier is released");
-            let abort_err = abort
-                .await
-                .expect("abort task should not panic")
-                .expect_err("the committed upload should no longer exist when abort acquires the lock");
-            assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
-            let list_err = list
-                .await
-                .expect("ListParts task should not panic")
-                .expect_err("the committed upload should no longer exist when ListParts acquires the lock");
-            assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
-        })
+                    .expect("ListParts task should not panic")
+                    .expect_err("the committed upload should no longer exist when ListParts acquires the lock");
+                assert!(
+                    matches!(list_err, StorageError::InvalidUploadID(..)),
+                    "ListParts should return InvalidUploadID after completion, got {list_err:?}"
+                );
+            },
+        )
         .await;
     }
 

crates/ecstore/src/set_disk/ops/object.rs

@@ -2320,13 +2320,25 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
     }
 }
 
+#[cfg(test)]
 fn persisted_transition_version(
     remote_version: &str,
 ) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
     persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
 }
 
+#[cfg(test)]
 fn remote_version_state_writer_enabled() -> bool {
+    remote_version_state_writer_fleet_proof().is_some()
+}
+
+fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
+    remote_version_state_writer_requested()
+        .then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
+        .flatten()
+}
+
+fn remote_version_state_writer_requested() -> bool {
     remote_version_state_writer_enabled_for(
         rustfs_utils::get_env_bool(
             rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
@@ -2336,11 +2348,25 @@ fn remote_version_state_writer_enabled() -> bool {
             rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
             rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
         ),
+        true,
     )
 }
 
-fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
-    requested && fleet_confirmed
+fn remote_version_state_writer_fleet_proof_matches(
+    proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
+) -> bool {
+    remote_version_state_writer_fleet_proof_matches_for(
+        remote_version_state_writer_requested(),
+        crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
+    )
+}
+
+fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
+    requested && fleet_proof_matches
+}
+
+fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
+    requested && fleet_confirmed && fleet_proof_valid
 }
 
 fn persisted_transition_version_with_gate(
@@ -2359,7 +2385,7 @@ fn persisted_transition_version_with_gate(
         Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
         Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
             std::io::ErrorKind::Unsupported,
-            "opaque remote tier versions require the operator-attested fleet gate",
+            "opaque remote tier versions require the operator-attested live fleet capability gate",
         )),
         Err(_) if remote_version == "null" => {
             Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
@@ -2605,7 +2631,7 @@ mod transition_upload_completion_tests {
 mod transition_version_id_tests {
     use super::{
         TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
-        remote_version_state_writer_enabled_for,
+        remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
     };
     use rustfs_filemeta::TransitionVersionState;
     use uuid::Uuid;
@@ -2648,15 +2674,35 @@ mod transition_version_id_tests {
 
     #[test]
     fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
-        for (case, requested, fleet_confirmed, expected) in [
-            ("old defaults", false, false, false),
-            ("missing fleet confirmation", true, false, false),
-            ("missing local opt-in", false, true, false),
-            ("explicitly unconfirmed fleet", true, false, false),
-            ("rolled-back writer", false, true, false),
-            ("fully upgraded fleet", true, true, true),
+        for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
+            ("old defaults", false, false, false, false),
+            ("missing fleet confirmation", true, false, true, false),
+            ("missing local opt-in", false, true, true, false),
+            ("missing fleet proof", true, true, false, false),
+            ("explicitly unconfirmed fleet", true, false, true, false),
+            ("rolled-back writer", false, true, true, false),
+            ("fully upgraded fleet", true, true, true, true),
         ] {
-            assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
+            assert_eq!(
+                remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
+                expected,
+                "{case}"
+            );
+        }
+    }
+
+    #[test]
+    fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
+        for (case, requested, fleet_proof_matches, expected) in [
+            ("operator gate closed", false, true, false),
+            ("fleet proof changed", true, false, false),
+            ("current authorization", true, true, true),
+        ] {
+            assert_eq!(
+                remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
+                expected,
+                "{case}"
+            );
         }
     }
 
@@ -3841,6 +3887,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let dest_obj = transaction.remote_object.clone();
         let mut transition_meta = (*oi.user_defined).clone();
         transition_meta.insert("name".to_string(), object.to_string());
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut transition_meta,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            transaction.transaction_id.to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut transition_meta,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(transaction.backend_fingerprint),
+        );
 
         if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
             transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
@@ -3951,20 +4007,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
             return Err(err.into());
         }
-        let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
-            Ok(version) => version,
-            Err(err) => {
-                let cleanup_api = transition_cleanup_store(&self.ctx).await;
-                if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
-                    return Err(StorageError::Io(std::io::Error::other(format!(
-                        "{err}; rejected remote upload cleanup failed: {cleanup_err}"
-                    ))));
+        let fleet_proof = remote_version_state_writer_fleet_proof();
+        let remote_version_requires_fleet_proof =
+            !candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
+        let (transition_version_id, transition_version_state) =
+            match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
+                Ok(version) => version,
+                Err(err) => {
+                    let cleanup_api = transition_cleanup_store(&self.ctx).await;
+                    if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
+                        return Err(StorageError::Io(std::io::Error::other(format!(
+                            "{err}; rejected remote upload cleanup failed: {cleanup_err}"
+                        ))));
+                    }
+                    delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
+                        .await;
+                    return Err(err.into());
                 }
-                delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
-                    .await;
-                return Err(err.into());
-            }
-        };
+            };
         if let Err(err) = advance_and_save_transition_transaction(
             transaction_api.as_ref(),
             &mut transaction,
@@ -4080,6 +4140,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
         #[cfg(test)]
         pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
+        // This check is the fleet-proof lease linearization point. Revocation
+        // blocks later commits; an already-authorized local quorum commit is
+        // allowed to finish without holding a synchronous lock across I/O.
+        if remote_version_requires_fleet_proof
+            && !fleet_proof
+                .as_ref()
+                .is_some_and(remote_version_state_writer_fleet_proof_matches)
+        {
+            drop(transition_lock_guard);
+            if upload_cleanup.cleanup().await.is_ok() {
+                delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
+                    .await;
+            }
+            return Err(Error::other("remote version state fleet capability changed during transition"));
+        }
         if let Err(err) = advance_and_save_transition_transaction(
             transaction_api.as_ref(),
             &mut transaction,

crates/ecstore/src/store/bucket.rs

@@ -568,6 +568,7 @@ mod tests {
     };
     use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
     use crate::bucket::metadata_sys;
+    use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
     use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
     use crate::error::StorageError;
     use crate::object_api::{ObjectOptions, PutObjReader};
@@ -589,6 +590,7 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use std::time::{Duration, SystemTime};
     use time::OffsetDateTime;
+    use tokio::io::AsyncReadExt;
     use tokio::sync::{Notify, OnceCell};
     use tokio_util::sync::CancellationToken;
     use uuid::Uuid;
@@ -1048,7 +1050,7 @@ mod tests {
             .await
             .expect("bucket should be created");
         for disk_path in &disk_paths {
-            tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
+            tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
                 .await
                 .expect("empty directory remnant should be created");
         }
@@ -1257,6 +1259,55 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn bucket_delete_preserves_put_committed_after_empty_scan() {
+        let (_, ecstore) = setup_bucket_delete_test_env().await;
+        let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
+        let object = "committed-after-empty-scan";
+        let payload = b"object committed after DeleteBucket empty scan".to_vec();
+
+        ecstore
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        let barrier = install_delete_bucket_empty_scan_barrier();
+        let delete_store = ecstore.clone();
+        let delete_bucket = bucket.clone();
+        let delete = tokio::spawn(async move {
+            delete_store
+                .delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
+                .await
+        });
+        barrier.wait_until_paused().await;
+
+        let mut put_reader = PutObjReader::from_vec(payload.clone());
+        ecstore
+            .put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
+            .await
+            .expect("PUT should commit while DeleteBucket is paused after its empty scan");
+
+        barrier.release();
+        let err = delete
+            .await
+            .expect("DeleteBucket task should join")
+            .expect_err("DeleteBucket must reject a PUT committed after its empty scan");
+        assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
+
+        let mut reader = ecstore
+            .get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("committed object should remain readable after DeleteBucket fails");
+        let mut restored = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut restored)
+            .await
+            .expect("object body should remain readable");
+        assert_eq!(restored, payload);
+    }
+
     #[tokio::test]
     #[serial]
     async fn bucket_recreation_does_not_publish_unverified_usage() {

crates/ecstore/src/store/init.rs

@@ -562,10 +562,12 @@ mod tests {
             },
             tier_sweeper::Jentry,
             transition_transaction::{
-                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
-                TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
-                TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
-                save_transition_transaction_record,
+                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
+                TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
+                TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator, load_transition_transaction_record,
+                recover_transition_transaction_records, save_transition_transaction_record,
             },
         },
         client::transition_api::ReaderImpl,
@@ -575,6 +577,7 @@ mod tests {
         services::tier::{
             test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
             tier::{TIER_CONFIG_FILE, TierConfigMgr},
+            tier_config::{TierConfig, TierType, TierWasabi},
             tier_mutation_intent::{
                 TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
                 TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -1163,6 +1166,37 @@ mod tests {
             .len()
     }
 
+    #[cfg(feature = "test-util")]
+    async fn register_transition_reconcile_test_tier(
+        handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
+        tier_name: &str,
+    ) -> MockWarmBackend {
+        let backend = MockWarmBackend::new();
+        let mut manager = handle.write().await;
+        manager.tiers.insert(
+            tier_name.to_string(),
+            TierConfig {
+                version: "v1".to_string(),
+                tier_type: TierType::Wasabi,
+                name: tier_name.to_string(),
+                wasabi: Some(TierWasabi {
+                    name: tier_name.to_string(),
+                    endpoint: "https://s3.wasabisys.com".to_string(),
+                    access_key: "test-access-key".to_string(),
+                    secret_key: "test-secret-key".to_string(),
+                    bucket: "mock-tier".to_string(),
+                    prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
+                    region: "us-east-1".to_string(),
+                }),
+                ..Default::default()
+            },
+        );
+        manager
+            .install_test_driver(tier_name, Box::new(backend.clone()))
+            .expect("mock fallback tier driver should install");
+        backend
+    }
+
     #[cfg(feature = "test-util")]
     async fn wait_for_tier_delete_journal_recovery(
         store: Arc<crate::store::ECStore>,
@@ -2738,7 +2772,7 @@ mod tests {
             .await;
             crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
 
-            let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+            let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
             let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
                 .await
                 .expect("tier lease should resolve")
@@ -2809,6 +2843,157 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXOPERATOR";
+        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: None,
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Unversioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("transaction should enter unknown upload outcome state");
+
+        let remote_version = uuid::Uuid::new_v4().to_string();
+        backend.set_put_remote_version(Some(remote_version.clone())).await;
+        let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
+        backend
+            .put(
+                &transaction.remote_object,
+                ReaderImpl::Body(candidate.clone()),
+                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
+            )
+            .await
+            .expect("mock backend should accept candidate");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+
+        let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
+            .await
+            .expect("operator inspection should probe the candidate");
+        assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
+
+        let wrong_version = uuid::Uuid::new_v4().to_string();
+        let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
+            .await
+            .expect_err("a mismatched exact version must fail before deleting a candidate");
+        assert!(matches!(
+            err,
+            TransitionOperatorError::CandidateVersionMismatch {
+                expected,
+                actual: TransitionOperatorProbe::VersionedPresent(ref observed),
+            } if expected == wrong_version && observed == &remote_version
+        ));
+        assert!(backend.contains(&transaction.remote_object).await);
+        assert_eq!(backend.exact_remove_count(), 0);
+        load_transition_transaction_record(store.clone(), transaction.transaction_id)
+            .await
+            .expect("an incorrect exact version must retain the transaction journal");
+
+        let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
+            .await
+            .expect("operator-confirmed exact candidate should be deleted");
+        assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
+        assert!(result.journal_observed_after_delete);
+        assert_eq!(backend.exact_remove_count(), 1);
+        assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
+        load_transition_transaction_record(store.clone(), transaction.transaction_id)
+            .await
+            .expect("candidate deletion must retain the transaction journal");
+
+        finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
+            .await
+            .expect("a separately confirmed missing candidate should permit finalization");
+        assert!(matches!(
+            load_transition_transaction_record(store, transaction.transaction_id).await,
+            Err(Error::ConfigNotFound)
+        ));
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn operator_finalize_retains_record_without_missing_proof() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXOPERATORFAIL";
+        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: None,
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Unversioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("transaction should enter unknown upload outcome state");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+        backend
+            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
+            .await;
+
+        assert!(matches!(
+            finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
+            Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
+        ));
+        load_transition_transaction_record(store, transaction.transaction_id)
+            .await
+            .expect("an unsupported probe must retain the transaction journal");
+        assert_eq!(backend.remove_count().await, 0);
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -2819,7 +3004,7 @@ mod tests {
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
 
         let tier_name = "TXRESPONSELOSS";
-        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
         let bucket = "transition-response-loss-bucket";
         let object = "source.bin";
         store

crates/ecstore/src/store/list_objects.rs

@@ -6895,6 +6895,7 @@ mod test {
         )
         .await
         .expect("local disk should be created");
+        disk.make_volume("bucket").await.expect("test bucket should be created");
         let mut fi = FileInfo::new("object", 1, 1);
         fi.volume = "bucket".to_owned();
         fi.name = "object".to_owned();

crates/ecstore/src/test_tracing.rs

@@ -0,0 +1,43 @@
+//! Test-only guard against tracing's process-global callsite-interest cache.
+//!
+//! Any test that installs its own subscriber and then asserts on span or event
+//! context is asserting on process-global state. See
+//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
+//! guard fixes it.
+
+/// Stop a sibling test thread from poisoning tracing's process-global callsite
+/// interest cache while this test asserts on span or event context.
+///
+/// `tracing` caches every callsite's `Interest` in **process-global** state, and
+/// the first thread to reach a callsite fixes that value. While at most one
+/// dispatcher is registered, tracing-core takes a fast path
+/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
+/// newly-registered callsite's interest from whichever subscriber is current
+/// *on the registering thread*, and registration happens exactly once (guarded
+/// by a CAS in `DefaultCallsite::register`).
+///
+/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
+/// reaches a production callsite first, from a thread with no subscriber
+/// installed: the interest is derived from that thread's `NoSubscriber` and
+/// cached as `Interest::never()` for the whole process. From then on the
+/// callsite is dead for *every* later caller, including a test that installed
+/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
+/// a `warn!`/`info!` event never fires at all.
+///
+/// Registering a second, inert dispatcher closes both halves of that race:
+///
+/// * constructing it rebuilds every *already-registered* callsite's interest
+///   against the live dispatcher set — which includes the caller's subscriber —
+///   repairing whatever a sibling may already have poisoned; and
+/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
+///   a callsite registered *later* by any thread is resolved against that live
+///   set instead of the registering thread's `NoSubscriber`.
+///
+/// Call this **after** installing the test's subscriber, and hold the returned
+/// guard for the rest of the test. Only the `cargo test` fallback needs it:
+/// nextest runs each test in its own process, where there is no sibling thread
+/// to lose the race to (see `docs/testing/README.md`).
+#[must_use = "callsite interest is only pinned while the returned guard is held"]
+pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
+    tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
+}