fix: guard operator candidate delete by exact version

Co-Authored-By: heihutu <heihutu@gmail.com>

crates/e2e_test/src/notification_webhook_test.rs

@@ -21,22 +21,18 @@
 //! function, never as an S3 event sink.
 //!
 //! Coverage:
-//!   * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
+//!   * PUT / multipart-complete / DELETE each deliver one event with the correct
 //!     eventName, bucket, key, versionId and eTag.
 //!   * prefix/suffix filters drop non-matching keys (rule-engine gate).
 //!   * an event queued while the target endpoint is unreachable is redelivered
 //!     from the on-disk store once the endpoint recovers (store-and-forward).
-//!   * responseElements and the S3 response use the canonical request ID while
-//!     requestParameters preserve a conflicting client-supplied value.
 
 use crate::common::{RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
-use aws_sdk_s3::operation::RequestId;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
-    NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
-    VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
+    NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
 };
 use http::header::{CONTENT_TYPE, HOST};
 use local_ip_address::local_ip;
@@ -44,12 +40,10 @@ use reqwest::StatusCode;
 use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
-use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
 use s3s::Body;
 use serde_json::Value;
 use serial_test::serial;
 use std::error::Error;
-use std::io::Cursor;
 use std::path::Path;
 use std::sync::{
     Arc, Once,
@@ -69,8 +63,6 @@ type BoxError = Box<dyn Error + Send + Sync>;
 /// for target lookup (see `process_queue_configurations`), so the region is
 /// nominal, but it must be present for `ARN::parse` to succeed.
 const NOTIFY_REGION: &str = "us-east-1";
-const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
-const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
 
 /// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
 /// so the ARN a notification rule references is
@@ -587,36 +579,6 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
     value.map(|e| e.trim_matches('"').to_string())
 }
 
-fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
-    assert_eq!(
-        record["requestParameters"][REQUEST_ID_HEADER].as_str(),
-        Some(CLIENT_REQUEST_ID),
-        "notification request parameters should retain the actual client header: {record}"
-    );
-    assert_eq!(
-        record["requestParameters"][AMZ_REQUEST_ID].as_str(),
-        Some(CLIENT_AMZ_REQUEST_ID),
-        "notification request parameters should retain the actual client header: {record}"
-    );
-    assert_eq!(
-        record["responseElements"][AMZ_REQUEST_ID].as_str(),
-        Some(server_request_id),
-        "notification response elements should use the canonical request ID: {record}"
-    );
-}
-
-fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
-    assert!(
-        record["requestParameters"][AMZ_REQUEST_ID].is_null(),
-        "notification request parameters must not invent a client request header: {record}"
-    );
-    assert_eq!(
-        record["responseElements"][AMZ_REQUEST_ID].as_str(),
-        Some(request_id),
-        "notification response elements should match the S3 response request ID: {record}"
-    );
-}
-
 // ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
@@ -709,17 +671,8 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         .bucket(bucket)
         .key(put_key)
         .body(ByteStream::from_static(b"peri-1 put body"))
-        .customize()
-        .mutate_request(|request| {
-            request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
-            request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
-        })
         .send()
         .await?;
-    let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
-    assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
-    assert_ne!(put_request_id, CLIENT_REQUEST_ID);
-    assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
     let put_version = put
         .version_id()
         .ok_or("PUT response missing versionId (versioning not enabled?)")?;
@@ -739,7 +692,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         "record eventName: {record}"
     );
     assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
-    assert_conflicting_request_id_correlation(record, &put_request_id);
     assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
     assert_eq!(
         trimmed_etag(object["eTag"].as_str()),
@@ -792,35 +744,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         "multipart eTag in event: {mp_record}"
     );
 
-    // --- Snowball extract: direct notification path keeps response correlation
-    let snowball_key = "uploads/snowball.dat";
-    let snowball_body = b"snowball notification body";
-    let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
-    let mut archive_header = tokio_tar::Header::new_gnu();
-    archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
-    archive_header.set_mode(0o644);
-    archive_header.set_cksum();
-    archive_builder
-        .append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
-        .await?;
-    let archive = archive_builder.into_inner().await?.into_inner();
-    let snowball = client
-        .put_object()
-        .bucket(bucket)
-        .key("snowball-fixture.tar")
-        .body(ByteStream::from(archive))
-        .customize()
-        .mutate_request(|request| {
-            request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await?;
-    let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
-
-    let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
-    let snowball_record = &snowball_event["Records"][0];
-    assert_generated_request_id_correlation(snowball_record, snowball_request_id);
-
     // --- Filter: non-matching prefix and suffix must never be delivered ------
     let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
     let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
@@ -849,32 +772,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
         "wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
     );
 
-    // --- DeleteObjects: direct notification path keeps response correlation --
-    let delete_many_key = "uploads/delete-many.dat";
-    client
-        .put_object()
-        .bucket(bucket)
-        .key(delete_many_key)
-        .body(ByteStream::from_static(b"delete objects notification body"))
-        .send()
-        .await?;
-    wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
-
-    let delete_many = client
-        .delete_objects()
-        .bucket(bucket)
-        .delete(
-            Delete::builder()
-                .objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
-                .build()?,
-        )
-        .send()
-        .await?;
-    let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
-    let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
-    assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
-
-    // --- DeleteObject on a versioned bucket: delete-marker version ----------
+    // --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
     let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
     let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
     let removed_record = &removed["Records"][0];

crates/ecstore/src/api/mod.rs

@@ -67,6 +67,14 @@ pub mod bucket {
             };
         }
 
+        pub mod transition_transaction {
+            pub use crate::bucket::lifecycle::transition_transaction::{
+                TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator,
+            };
+        }
+
         pub mod evaluator {
             pub use crate::bucket::lifecycle::evaluator::Evaluator;
         }
@@ -122,13 +130,13 @@ pub mod bucket {
 
     pub mod metadata_sys {
         pub use crate::bucket::metadata_sys::{
-            BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
+            BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
             get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
             get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
             get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
             init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
-            update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
+            update, update_bucket_targets_under_transaction_lock, update_config_with,
         };
     }
 
@@ -377,6 +385,7 @@ pub mod metrics {
 pub mod notification {
     pub use crate::services::notification_sys::{
         NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
+        start_remote_version_state_fleet_probe,
     };
 }
 

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -23,6 +23,7 @@ use uuid::Uuid;
 use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
+    delete_confirmed_transition_candidate_exact_with_lease_idempotent,
     delete_confirmed_transition_candidate_exact_with_manager_and_identity,
     delete_object_from_remote_tier_idempotent_with_manager_and_identity,
 };
@@ -616,6 +617,199 @@ pub enum TransitionTransactionRecoveryOutcome {
     Retained,
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TransitionOperatorProbe {
+    Missing,
+    UnversionedPresent,
+    VersionedPresent(String),
+    Ambiguous,
+    Unsupported,
+}
+
+impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
+    fn from(value: TransitionCandidateProbe) -> Self {
+        match value {
+            TransitionCandidateProbe::Missing => Self::Missing,
+            TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
+            TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
+            TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
+            TransitionCandidateProbe::Unsupported => Self::Unsupported,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TransitionOperatorStatus {
+    pub transaction_id: Uuid,
+    pub state: TransitionTransactionState,
+    pub tier_name: String,
+    pub remote_object: String,
+    pub not_after_unix_nanos: i64,
+    pub probe: TransitionOperatorProbe,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TransitionOperatorDeleteResult {
+    pub status: TransitionOperatorStatus,
+    pub journal_observed_after_delete: bool,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum TransitionOperatorError {
+    #[error("transition transaction was not found")]
+    NotFound,
+    #[error("transition transaction is still inside its active ownership window")]
+    NotExpired,
+    #[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
+    InvalidState(TransitionTransactionState),
+    #[error("an exact non-empty remote version is required")]
+    RemoteVersionRequired,
+    #[error("remote candidate is not proven missing: {0:?}")]
+    CandidateNotMissing(TransitionOperatorProbe),
+    #[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
+    CandidateVersionMismatch {
+        expected: String,
+        actual: TransitionOperatorProbe,
+    },
+    #[error("transition transaction store failed: {0}")]
+    Store(#[source] Error),
+    #[error("remote tier reconciliation failed: {0}")]
+    Remote(#[source] std::io::Error),
+}
+
+type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
+
+fn validate_operator_reconcile_transaction(
+    transaction: &TransitionTransaction,
+    now_unix_nanos: i128,
+) -> TransitionOperatorResult<()> {
+    transaction
+        .validate()
+        .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
+    if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
+        return Err(TransitionOperatorError::InvalidState(transaction.state));
+    }
+    if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
+        return Err(TransitionOperatorError::NotExpired);
+    }
+    Ok(())
+}
+
+async fn load_operator_reconcile_transaction(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<TransitionTransaction> {
+    match load_transition_transaction_record(api, transaction_id).await {
+        Ok(transaction) => Ok(transaction),
+        Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
+        Err(err) => Err(TransitionOperatorError::Store(err)),
+    }
+}
+
+async fn operator_probe_transition_candidate(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> TransitionOperatorResult<TransitionOperatorProbe> {
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
+    lease
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
+        .await
+        .map(TransitionOperatorProbe::from)
+        .map_err(TransitionOperatorError::Remote)
+}
+
+pub async fn inspect_transition_transaction_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<TransitionOperatorStatus> {
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let probe = operator_probe_transition_candidate(api, &transaction).await?;
+    Ok(TransitionOperatorStatus {
+        transaction_id,
+        state: transaction.state,
+        tier_name: transaction.tier_name,
+        remote_object: transaction.remote_object,
+        not_after_unix_nanos: transaction.not_after_unix_nanos,
+        probe,
+    })
+}
+
+pub async fn delete_transition_candidate_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+    remote_version_id: &str,
+) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
+    if remote_version_id.is_empty() {
+        return Err(TransitionOperatorError::RemoteVersionRequired);
+    }
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
+        &api.tier_config_mgr(),
+        &transaction.tier_name,
+        transaction.backend_fingerprint,
+    )
+    .await
+    .map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
+    lease
+        .validate_remote_version_id(remote_version_id)
+        .map_err(TransitionOperatorError::Remote)?;
+    let before_delete_probe = lease
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
+        .await
+        .map(TransitionOperatorProbe::from)
+        .map_err(TransitionOperatorError::Remote)?;
+    if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
+        return Err(TransitionOperatorError::CandidateVersionMismatch {
+            expected: remote_version_id.to_string(),
+            actual: before_delete_probe,
+        });
+    }
+    delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
+        .await
+        .map_err(TransitionOperatorError::Remote)?;
+    let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
+    let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
+        Ok(_) => true,
+        Err(Error::ConfigNotFound) => false,
+        Err(err) => return Err(TransitionOperatorError::Store(err)),
+    };
+    Ok(TransitionOperatorDeleteResult {
+        status: TransitionOperatorStatus {
+            transaction_id,
+            state: transaction.state,
+            tier_name: transaction.tier_name,
+            remote_object: transaction.remote_object,
+            not_after_unix_nanos: transaction.not_after_unix_nanos,
+            probe,
+        },
+        journal_observed_after_delete,
+    })
+}
+
+pub async fn finalize_missing_transition_transaction_for_operator(
+    api: Arc<ECStore>,
+    transaction_id: Uuid,
+) -> TransitionOperatorResult<()> {
+    let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
+    validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
+    let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
+    if probe != TransitionOperatorProbe::Missing {
+        return Err(TransitionOperatorError::CandidateNotMissing(probe));
+    }
+    delete_transition_transaction_record(api, transaction_id)
+        .await
+        .map_err(TransitionOperatorError::Store)
+}
+
 pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
     let transaction_id = transition_transaction_id_from_record_object_name(object)?;
     TransitionTransaction::decode(transaction_id, data)
@@ -700,7 +894,7 @@ async fn recover_unknown_upload_outcome(
     .map_err(Error::other)?;
 
     match lease
-        .probe_transition_candidate(&transaction.remote_object)
+        .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
         .await
         .map_err(Error::other)?
     {
@@ -1097,6 +1291,27 @@ mod tests {
             .expect("upload state change should succeed")
     }
 
+    #[test]
+    fn operator_reconcile_requires_expired_unknown_upload_outcome() {
+        let mut transaction = new_transaction();
+        let active_deadline = transaction.not_after_unix_nanos;
+
+        assert!(matches!(
+            validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
+            Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
+        ));
+
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("unknown upload outcome should be recorded");
+        assert!(matches!(
+            validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
+            Err(TransitionOperatorError::NotExpired)
+        ));
+        validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
+            .expect("expired unknown upload outcome should be eligible");
+    }
+
     fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
         TransitionCleanupProof {
             transaction_id: transaction.transaction_id,

crates/ecstore/src/bucket/metadata_sys.rs

@@ -242,138 +242,73 @@ pub(crate) async fn remove_bucket_metadata_in(ctx: &crate::runtime::instance::In
     Ok(lock.remove(bucket).await)
 }
 
-/// Rewrite one config file of a bucket's metadata, serialized cluster-wide.
-///
-/// See [`acquire_bucket_metadata_transaction_lock`] for why every config
-/// write — not just the replication-targets one — has to hold that lock.
 pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
-    update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data).await
-}
-
-pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
-    delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
-}
-
-/// [`update`] against an explicitly supplied metadata system.
-///
-/// The free functions resolve the instance's own system; this variant takes
-/// it as an argument so a test can drive two independent systems over one
-/// backing store — the in-process stand-in for two nodes, which is the only
-/// configuration where the transaction lock is what does the serializing.
-async fn update_with_sys(
-    sys: Arc<RwLock<BucketMetadataSys>>,
-    bucket: &str,
-    config_file: &str,
-    data: Vec<u8>,
-) -> Result<OffsetDateTime> {
-    let (_transaction_guard, mut sys) = acquire_config_write_guards(sys, bucket).await?;
-    sys.update(bucket, config_file, data).await
-}
-
-/// [`delete`] against an explicitly supplied metadata system. See
-/// [`update_with_sys`].
-async fn delete_with_sys(sys: Arc<RwLock<BucketMetadataSys>>, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
-    let (_transaction_guard, mut sys) = acquire_config_write_guards(sys, bucket).await?;
-    sys.delete(bucket, config_file).await
-}
-
-/// Take, in the one order every config write uses, the two guards a
-/// read-modify-write of a bucket's metadata needs: the cluster-wide
-/// transaction lock first, then this process's metadata-system write guard.
-///
-/// The order is load-bearing. Acquiring the process-local guard first would
-/// park every local reader and writer of *every* bucket behind a lock whose
-/// holder may be another node, turning remote contention into a local stall.
-async fn acquire_config_write_guards(
-    sys: Arc<RwLock<BucketMetadataSys>>,
-    bucket: &str,
-) -> Result<(rustfs_lock::NamespaceLockGuard, tokio::sync::OwnedRwLockWriteGuard<BucketMetadataSys>)> {
-    let transaction_guard = acquire_transaction_lock_with_sys(&sys, bucket).await?;
-    let sys_guard = sys.write_owned().await;
-    Ok((transaction_guard, sys_guard))
-}
-
-/// Rewrite one config file while the caller already holds this bucket's
-/// transaction lock.
-///
-/// [`update`] would deadlock here: the lock is not reentrant, so a holder
-/// that called it would block until its own guard timed out.
-pub async fn update_under_transaction_lock(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let _targets_guard = if config_file == BUCKET_TARGETS_FILE {
+        Some(acquire_bucket_targets_transaction_lock(bucket).await?)
+    } else {
+        None
+    };
     let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
+
     bucket_meta_sys.update(bucket, config_file, data).await
 }
 
 pub async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
-    update_under_transaction_lock(bucket, BUCKET_TARGETS_FILE, data).await
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
+    bucket_meta_sys.update(bucket, BUCKET_TARGETS_FILE, data).await
 }
 
-/// Read-modify-write one bucket config file under both guards a config
-/// write takes.
+/// Read-modify-write one bucket config file under the metadata system's
+/// outer write guard.
 ///
 /// `mutate` sees the freshly loaded on-disk metadata and returns the
 /// replacement payload for `config_file` (empty clears it, like
 /// [`delete`]). Both the read and the persisted write happen inside the
-/// same guards [`update`] takes, so the rewrite can neither clobber a
-/// concurrent update to another config file nor lose a concurrent write to
-/// the same one — unlike caching a mutated clone of previously read
-/// metadata.
+/// same guard that [`update`] uses, so within this process the rewrite can
+/// neither clobber a concurrent update to another config file nor lose a
+/// concurrent write to the same one — unlike caching a mutated clone of
+/// previously read metadata.
 ///
-/// That exclusion is cluster-wide, not merely process-local: the transaction
-/// lock is now taken for every config file rather than only the replication
-/// targets one, so a writer on another node cannot land a whole-file save in
-/// the middle of this read-modify-write.
+/// This guard is process-local. Writers on other nodes still race, exactly
+/// as they do for [`update`]: each rewrites the whole metadata file, so the
+/// later save wins. What this narrows is the window — from "as stale as the
+/// local cache" down to a single metadata read plus write.
 pub async fn update_config_with<F>(bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
 where
     F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
 {
-    let (_transaction_guard, mut sys) = acquire_config_write_guards(get_bucket_metadata_sys()?, bucket).await?;
-    sys.update_config_with(bucket, config_file, mutate).await
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let _targets_guard = if config_file == BUCKET_TARGETS_FILE {
+        Some(acquire_bucket_targets_transaction_lock(bucket).await?)
+    } else {
+        None
+    };
+    let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
+    bucket_meta_sys.update_config_with(bucket, config_file, mutate).await
 }
 
-/// Acquire a bucket's metadata transaction lock, held across a whole
-/// read-modify-write of its metadata file.
-///
-/// Every config write loads the entire [`BucketMetadata`] blob, replaces one
-/// field, and saves the whole thing back. The namespace locks inside
-/// `read_config`/`save_config` are taken and released separately, so they do
-/// not span that cycle: two nodes updating *different* config files of one
-/// bucket both load the same blob, each set their own field, and the later
-/// save drops the other's — with both clients already told 2xx. This is not
-/// last-writer-wins on one document; an orthogonal config silently vanishes.
-///
-/// So the lock is per bucket, not per config file: a per-file key would let
-/// exactly that pair run concurrently.
-///
-/// Callers that hold this guard must use [`update_under_transaction_lock`]
-/// rather than [`update`] — see that function.
-pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
-    acquire_transaction_lock_with_sys(&get_bucket_metadata_sys()?, bucket).await
-}
-
-async fn acquire_transaction_lock_with_sys(
-    sys: &Arc<RwLock<BucketMetadataSys>>,
-    bucket: &str,
-) -> Result<rustfs_lock::NamespaceLockGuard> {
-    // Resolve the store under a short-lived read guard: this runs before the
-    // write guard in `acquire_config_write_guards`, and must not still hold a
-    // read guard when the namespace lock is awaited.
-    let api = sys.read().await.object_store();
+pub async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let api = bucket_meta_sys_lock.read().await.object_store();
     let lock = api
-        .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
+        .new_ns_lock(RUSTFS_META_BUCKET, &bucket_targets_transaction_lock_key(bucket))
         .await?;
     Ok(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
 }
 
-/// The lock resource name is deliberately still the `bucket-targets` one it
-/// had when only replication-target writes took it. The key is what nodes
-/// agree on, so renaming it would leave a mixed-version cluster with two
-/// disjoint keys — and old and new nodes would stop excluding each other on
-/// the very writes that are serialized today.
-fn bucket_metadata_transaction_lock_key(bucket: &str) -> String {
+fn bucket_targets_transaction_lock_key(bucket: &str) -> String {
     format!("bucket-targets/{bucket}/transaction.lock")
 }
 
+pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
+    let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
+
+    bucket_meta_sys.delete(bucket, config_file).await
+}
+
 pub async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -929,11 +864,11 @@ impl BucketMetadataSys {
     }
 
     async fn update_and_parse(&mut self, bucket: &str, config_file: &str, data: Vec<u8>, parse: bool) -> Result<OffsetDateTime> {
-        // Load through this system's own store, the one `save` persists to
-        // (backlog#1052 S7). Reading from the ambient handle instead made the
-        // read and the write of a single read-modify-write able to target
-        // different instances.
-        let mut bm = Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse).await?;
+        let Some(store) = runtime_sources::object_store_handle() else {
+            return Err(Error::other("errServerNotInitialized"));
+        };
+
+        let mut bm = Self::load_bucket_metadata_for_update(store, bucket, parse).await?;
 
         let updated = bm.update_config(config_file, data)?;
 
@@ -1880,160 +1815,6 @@ mod tests {
         );
     }
 
-    /// Two metadata systems over one backing store: the in-process stand-in
-    /// for two nodes. They share no `RwLock`, so nothing but the transaction
-    /// lock can serialize them — exactly the cross-node case.
-    async fn two_nodes_over_one_store() -> (Vec<tempfile::TempDir>, Arc<RwLock<BucketMetadataSys>>, Arc<RwLock<BucketMetadataSys>>)
-    {
-        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let node_a = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
-        let node_b = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
-        (dirs, node_a, node_b)
-    }
-
-    /// Writers on different nodes updating *different* config files of one
-    /// bucket must both survive. Each rewrites the whole metadata blob, so
-    /// without a lock spanning the read-modify-write the later save carries
-    /// the earlier writer's field back to its pre-update value — losing an
-    /// orthogonal config while both clients were told the write succeeded.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    #[serial]
-    async fn concurrent_config_writes_from_separate_nodes_do_not_lose_writes() {
-        use crate::bucket::metadata::{BUCKET_POLICY_CONFIG, BUCKET_TAGGING_CONFIG};
-
-        let (_dirs, node_a, node_b) = two_nodes_over_one_store().await;
-        let bucket = "cross-node-config-writes";
-        node_a
-            .read()
-            .await
-            .persist_and_set(BucketMetadata::new(bucket))
-            .await
-            .expect("initial metadata should persist");
-
-        // Several rounds: a single pass can serialize by luck, but a lost
-        // update only needs one interleaving to show up.
-        const ROUNDS: usize = 8;
-        for round in 0..ROUNDS {
-            let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
-            let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
-
-            let start = Arc::new(tokio::sync::Barrier::new(2));
-            let tagging_writer = {
-                let (node, start, tagging) = (node_a.clone(), start.clone(), tagging.clone());
-                tokio::spawn(async move {
-                    start.wait().await;
-                    update_with_sys(node, bucket, BUCKET_TAGGING_CONFIG, tagging).await
-                })
-            };
-            let policy_writer = {
-                let (node, start, policy) = (node_b.clone(), start.clone(), policy.clone());
-                tokio::spawn(async move {
-                    start.wait().await;
-                    update_with_sys(node, bucket, BUCKET_POLICY_CONFIG, policy).await
-                })
-            };
-
-            tagging_writer
-                .await
-                .expect("tagging writer should join")
-                .expect("tagging update should succeed");
-            policy_writer
-                .await
-                .expect("policy writer should join")
-                .expect("policy update should succeed");
-
-            // Disk truth, not either node's cache: the losing write is the one
-            // that never reached the metadata file.
-            let persisted = node_a
-                .read()
-                .await
-                .get_config_from_disk(bucket)
-                .await
-                .expect("metadata should load from disk");
-            assert_eq!(
-                persisted.tagging_config_xml, tagging,
-                "round {round}: the policy write clobbered the concurrent tagging write"
-            );
-            assert_eq!(
-                persisted.policy_config_json, policy,
-                "round {round}: the tagging write clobbered the concurrent policy write"
-            );
-        }
-    }
-
-    /// The guard has to cover the load as well as the save. If it were taken
-    /// only around the save, a second node could load between the two and
-    /// still overwrite with pre-update state.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    #[serial]
-    async fn bucket_metadata_transaction_lock_blocks_a_concurrent_config_write() {
-        use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
-
-        let (_dirs, node_a, node_b) = two_nodes_over_one_store().await;
-        let bucket = "cross-node-transaction-lock";
-        node_a
-            .read()
-            .await
-            .persist_and_set(BucketMetadata::new(bucket))
-            .await
-            .expect("initial metadata should persist");
-
-        let held = acquire_transaction_lock_with_sys(&node_a, bucket)
-            .await
-            .expect("transaction lock should be acquirable");
-
-        let blocked = tokio::spawn({
-            let node_b = node_b.clone();
-            async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
-        });
-
-        // Long enough for the write to have finished had it not waited: the
-        // whole read-modify-write against temp disks is far quicker than this.
-        tokio::time::sleep(Duration::from_millis(500)).await;
-        assert!(
-            !blocked.is_finished(),
-            "a config write must not proceed while another node holds the bucket's transaction lock"
-        );
-
-        drop(held);
-
-        let updated = timeout(Duration::from_secs(10), blocked)
-            .await
-            .expect("the blocked write should proceed once the lock is released")
-            .expect("blocked writer should join");
-        updated.expect("the write should succeed after acquiring the lock");
-    }
-
-    /// A holder of the transaction lock must not call the locking entry
-    /// point: the namespace lock is not reentrant, so it would block on
-    /// itself until the acquire timeout.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    #[serial]
-    async fn transaction_lock_is_not_reentrant() {
-        let (_dirs, node_a, node_b) = two_nodes_over_one_store().await;
-        let bucket = "transaction-lock-reentrancy";
-
-        let held = acquire_transaction_lock_with_sys(&node_a, bucket)
-            .await
-            .expect("first acquisition should succeed");
-
-        // Same store, hence the same locker owner: exclusion must not depend
-        // on the two acquisitions coming from different owners. Either
-        // outcome is acceptable — still waiting, or refused — as long as no
-        // second guard is handed out.
-        let reacquired = timeout(Duration::from_millis(500), acquire_transaction_lock_with_sys(&node_b, bucket)).await;
-        assert!(
-            !matches!(reacquired, Ok(Ok(_))),
-            "the transaction lock must exclude a second holder even under the same owner"
-        );
-
-        drop(held);
-        timeout(Duration::from_secs(10), acquire_transaction_lock_with_sys(&node_b, bucket))
-            .await
-            .expect("re-acquisition should not time out once released")
-            .expect("the lock should be acquirable after release");
-    }
-
     fn target(bucket: &str, id: &str) -> BucketTarget {
         BucketTarget {
             source_bucket: bucket.to_string(),

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -222,6 +222,21 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
         .map_err(|_| Error::other("peer returned an invalid heal control response proof"))
 }
 
+fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
+    let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
+    if topology_member != expected_member {
+        return Err(Error::other(
+            "peer returned a remote version state capability for a different topology member",
+        ));
+    }
+    let server_epoch =
+        Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
+    if server_epoch.is_nil() {
+        return Err(Error::other("peer returned a nil remote version state epoch"));
+    }
+    Ok(server_epoch)
+}
+
 #[derive(Clone, Debug)]
 pub struct PeerLiveEventsBatch {
     pub events: Vec<u8>,
@@ -233,6 +248,7 @@ pub struct PeerLiveEventsBatch {
 pub struct PeerRestClient {
     pub host: XHost,
     pub grid_host: String,
+    topology_member: String,
     offline: Arc<AtomicBool>,
     recovery_running: Arc<AtomicBool>,
 }
@@ -325,9 +341,11 @@ impl PeerRestClient {
     }
 
     pub fn new(host: XHost, grid_host: String) -> Self {
+        let topology_member = host.to_string();
         Self {
             host,
             grid_host,
+            topology_member,
             offline: Arc::new(AtomicBool::new(false)),
             recovery_running: Arc::new(AtomicBool::new(false)),
         }
@@ -347,7 +365,11 @@ impl PeerRestClient {
 
             let client = match grid_host {
                 Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
-                    Ok(host) => Some(PeerRestClient::new(host, grid_host)),
+                    Ok(host) => {
+                        let mut client = PeerRestClient::new(host, grid_host);
+                        client.topology_member = peer_host_port.clone();
+                        Some(client)
+                    }
                     Err(err) => {
                         warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
                         None
@@ -1154,6 +1176,15 @@ impl PeerRestClient {
         validate_heal_control_capability_proof(&canonical_ack, &proof)
     }
 
+    pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
+        let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
+        let result = self
+            .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
+            .await?;
+        let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
+        Ok((self.topology_member.clone(), epoch))
+    }
+
     pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
         self.finalize_result(
             async {
@@ -2470,6 +2501,22 @@ mod tests {
         }
     }
 
+    #[test]
+    fn remote_version_state_capability_decoder_fails_closed() {
+        let epoch = Uuid::new_v4();
+        let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
+            .expect("small capability response should encode");
+        assert_eq!(
+            decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
+            epoch
+        );
+        assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
+        assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
+        let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
+            .expect("small capability response should encode");
+        assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
+    }
+
     struct TierMutationResponseFixture<'a> {
         version: u32,
         phase: TierMutationRpcPhase,

crates/ecstore/src/services/notification_sys.rs

@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
 use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use rustfs_utils::XHost;
-use std::collections::{HashMap, hash_map::DefaultHasher};
+use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
 use std::future::Future;
 use std::hash::{Hash, Hasher};
 use std::sync::{Arc, Mutex, OnceLock};
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 use tokio::time::{sleep, timeout};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
@@ -47,6 +47,9 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
 const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
 const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
+const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
+const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
+const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 
 /// Cached result from the last successful admin call to a peer.
 struct PeerAdminCache {
@@ -91,6 +94,180 @@ lazy_static! {
     pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
 }
 
+#[derive(Clone)]
+struct RemoteVersionStateFleetProof {
+    topology_fingerprint: String,
+    peer_epochs: Arc<BTreeMap<String, Uuid>>,
+    expires_at: Instant,
+}
+
+impl RemoteVersionStateFleetProof {
+    fn token(&self) -> RemoteVersionStateFleetProofToken {
+        RemoteVersionStateFleetProofToken {
+            topology_fingerprint: self.topology_fingerprint.clone(),
+            peer_epochs: self.peer_epochs.clone(),
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Eq)]
+pub(crate) struct RemoteVersionStateFleetProofToken {
+    topology_fingerprint: String,
+    peer_epochs: Arc<BTreeMap<String, Uuid>>,
+}
+
+#[derive(Default)]
+struct RemoteVersionStateFleetProofState {
+    proof: Option<RemoteVersionStateFleetProof>,
+    topology_conflict: bool,
+}
+
+static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
+static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
+
+fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
+    REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
+}
+
+fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
+    replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
+}
+
+fn replace_remote_version_state_fleet_proof_in(
+    slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
+    proof: Option<RemoteVersionStateFleetProof>,
+) {
+    slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
+}
+
+fn publish_remote_version_state_probe_result(
+    slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
+    topology_fingerprint: &str,
+    result: Result<BTreeMap<String, Uuid>>,
+    observed_at: Instant,
+) -> Option<Error> {
+    match result {
+        Ok(peer_epochs) => {
+            let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
+            let peer_epochs = state
+                .proof
+                .as_ref()
+                .filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
+                .map(|proof| Arc::clone(&proof.peer_epochs))
+                .unwrap_or_else(|| Arc::new(peer_epochs));
+            state.proof = Some(RemoteVersionStateFleetProof {
+                topology_fingerprint: topology_fingerprint.to_string(),
+                peer_epochs,
+                expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
+            });
+            None
+        }
+        Err(err) => {
+            replace_remote_version_state_fleet_proof_in(slot, None);
+            Some(err)
+        }
+    }
+}
+
+pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = remote_version_state_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
+}
+
+fn acquire_remote_version_state_fleet_proof_from(
+    state: &RemoteVersionStateFleetProofState,
+    expected_topology: &str,
+    now: Instant,
+) -> Option<RemoteVersionStateFleetProofToken> {
+    if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
+        return None;
+    }
+    state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
+}
+
+pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
+    let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
+        return false;
+    };
+    let state = remote_version_state_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    if state.topology_conflict {
+        return false;
+    }
+    state.proof.as_ref().is_some_and(|current| {
+        current.topology_fingerprint == *expected_topology
+            && current.topology_fingerprint == proof.topology_fingerprint
+            && Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
+            && Instant::now() < current.expires_at
+    })
+}
+
+fn remote_version_state_fleet_proof_valid_at(
+    proof: Option<&RemoteVersionStateFleetProof>,
+    expected_topology: &str,
+    now: Instant,
+) -> bool {
+    proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
+}
+
+fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
+    if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
+        return Err(Error::other("remote version state capability peer identity is invalid"));
+    }
+    Ok(())
+}
+
+pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
+    if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
+        if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
+            let mut state = remote_version_state_fleet_proof_slot()
+                .write()
+                .unwrap_or_else(std::sync::PoisonError::into_inner);
+            state.topology_conflict = true;
+            state.proof = None;
+        }
+        return;
+    }
+
+    tokio::spawn(async move {
+        loop {
+            let result = match get_global_notification_sys() {
+                Some(notification_sys) => {
+                    match timeout(
+                        REMOTE_VERSION_STATE_PROBE_TIMEOUT,
+                        notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
+                    )
+                    .await
+                    {
+                        Ok(result) => result,
+                        Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
+                    }
+                }
+                None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
+            };
+            let topology_conflict = remote_version_state_fleet_proof_slot()
+                .read()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .topology_conflict;
+            if topology_conflict {
+                replace_remote_version_state_fleet_proof(None);
+            } else if let Some(err) = publish_remote_version_state_probe_result(
+                remote_version_state_fleet_proof_slot(),
+                &topology_fingerprint,
+                result,
+                Instant::now(),
+            ) {
+                debug!(error = %err, "remote version state fleet capability probe failed closed");
+            }
+            sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
+        }
+    });
+}
+
 pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
     let _ = GLOBAL_NOTIFICATION_SYS
         .set(Arc::new(NotificationSys::new(eps).await))
@@ -115,7 +292,17 @@ pub struct NotificationSys {
 
 impl NotificationSys {
     pub async fn new(eps: EndpointServerPools) -> Self {
+        let expected_remote_hosts = eps
+            .peer_grid_host_slots_sorted()
+            .into_iter()
+            .filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
+            .collect::<Vec<_>>();
         let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
+        let peer_topology_hosts = if peer_topology_hosts.is_empty() {
+            expected_remote_hosts
+        } else {
+            peer_topology_hosts
+        };
         let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
         Self {
             peer_clients,
@@ -125,6 +312,24 @@ impl NotificationSys {
             tier_config_reload_workers: Default::default(),
         }
     }
+
+    async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
+        if self.peer_clients.len() != self.peer_topology_hosts.len() {
+            return Err(Error::other("remote version state capability fleet membership is incomplete"));
+        }
+        let probes = self.peer_clients.iter().map(|client| async {
+            let client = client
+                .as_ref()
+                .ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
+            client.probe_remote_version_state(topology_fingerprint.to_string()).await
+        });
+        let mut peer_epochs = BTreeMap::new();
+        for result in join_all(probes).await {
+            let (peer, epoch) = result?;
+            insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
+        }
+        Ok(peer_epochs)
+    }
 }
 
 pub struct NotificationPeerErr {
@@ -1890,6 +2095,183 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
 mod tests {
     use super::*;
 
+    #[test]
+    fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
+        let now = Instant::now();
+        let mut peer_epochs = BTreeMap::new();
+        peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(peer_epochs),
+            expires_at: now + Duration::from_secs(1),
+        };
+
+        assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
+        assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
+        assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
+        assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
+        let mut peer_epochs = BTreeMap::new();
+
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
+        assert!(peer_epochs.is_empty());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_accepts_single_node_membership() {
+        let now = Instant::now();
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(BTreeMap::new()),
+            expires_at: now + Duration::from_secs(1),
+        };
+
+        assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
+        let now = Instant::now();
+        let proof = RemoteVersionStateFleetProof {
+            topology_fingerprint: "topology-a".to_string(),
+            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            expires_at: now + Duration::from_secs(1),
+        };
+        let captured = proof.token();
+        let restarted = RemoteVersionStateFleetProof {
+            topology_fingerprint: proof.topology_fingerprint.clone(),
+            peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
+            expires_at: proof.expires_at,
+        };
+
+        assert!(captured != restarted.token());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
+        let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
+        let now = Instant::now();
+        let epoch = Uuid::new_v4();
+        let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
+        let original = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("successful probe should publish proof")
+            .token();
+
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
+        );
+        let renewed = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("renewal should retain proof")
+            .token();
+        assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
+
+        let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
+                .is_none()
+        );
+        let replaced = slot
+            .read()
+            .expect("proof slot should not poison")
+            .proof
+            .as_ref()
+            .expect("restarted peer should publish a new proof")
+            .token();
+        assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
+    }
+
+    #[test]
+    fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
+        let now = Instant::now();
+        let mut state = RemoteVersionStateFleetProofState {
+            proof: Some(RemoteVersionStateFleetProof {
+                topology_fingerprint: "topology-a".to_string(),
+                peer_epochs: Arc::new(BTreeMap::new()),
+                expires_at: now + Duration::from_secs(1),
+            }),
+            topology_conflict: false,
+        };
+        assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
+
+        state.topology_conflict = true;
+        assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
+        let epoch = Uuid::new_v4();
+        let mut peer_epochs = BTreeMap::new();
+        insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
+            .expect("first member should be admitted");
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
+        assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
+    }
+
+    #[test]
+    fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
+        let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
+        let now = Instant::now();
+        let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
+        assert!(slot.read().expect("proof slot should not poison").proof.is_some());
+
+        assert!(
+            publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
+        );
+        assert!(slot.read().expect("proof slot should not poison").proof.is_none());
+
+        let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
+        assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
+        assert!(slot.read().expect("proof slot should not poison").proof.is_some());
+    }
+
+    #[tokio::test]
+    async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
+        let notification_sys = NotificationSys {
+            peer_clients: vec![None],
+            all_peer_clients: vec![None, None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
+            tier_config_reload_workers: Default::default(),
+        };
+
+        let err = notification_sys
+            .probe_remote_version_state_fleet("topology-a")
+            .await
+            .expect_err("an unreachable configured member must fail the fleet proof");
+        assert!(err.to_string().contains("unreachable"));
+    }
+
+    #[tokio::test]
+    async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
+        let notification_sys = NotificationSys {
+            peer_clients: Vec::new(),
+            all_peer_clients: vec![None],
+            peer_topology_hosts: vec!["peer-a".to_string()],
+            peer_admin_caches: Vec::new(),
+            tier_config_reload_workers: Default::default(),
+        };
+
+        let err = notification_sys
+            .probe_remote_version_state_fleet("topology-a")
+            .await
+            .expect_err("a missing configured member slot must fail the fleet proof");
+        assert!(err.to_string().contains("incomplete"));
+    }
+
     fn build_props(endpoint: &str) -> ServerProperties {
         ServerProperties {
             endpoint: endpoint.to_string(),

crates/ecstore/src/services/tier/tier.rs

@@ -236,6 +236,7 @@ struct TierPublishTransition {
 
 struct PreparedTierDriver {
     tier_name: String,
+    tier_config: TierConfig,
     config_fingerprint: TierDriverFingerprint,
     backend_identity: TierDestinationId,
     exact_get_delete: bool,
@@ -1342,6 +1343,7 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
 
 struct TierDriverGeneration {
     tier_name: Arc<str>,
+    tier_config: TierConfig,
     generation: DriverRevision,
     // Process-local only: this may reflect credential changes and must never be persisted or logged.
     config_fingerprint: TierDriverFingerprint,
@@ -1349,6 +1351,9 @@ struct TierDriverGeneration {
     backend_identity: TierDestinationId,
     exact_get_delete: bool,
     driver: SharedWarmBackend,
+    reconciler: tokio::sync::OnceCell<
+        Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
+    >,
     accepting: AtomicBool,
     active_leases: AtomicUsize,
     drained: tokio::sync::Notify,
@@ -1473,6 +1478,35 @@ impl TierOperationLease {
         self.inner.backend_identity
     }
 
+    pub(crate) async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        transaction_id: uuid::Uuid,
+    ) -> io::Result<TransitionCandidateProbe> {
+        let Some(reconciler) = self
+            .inner
+            .reconciler
+            .get_or_try_init(|| async {
+                crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
+                    .await
+                    .map(|reconciler| reconciler.map(Arc::from))
+            })
+            .await
+            .map_err(|err| io::Error::other(err.message))?
+        else {
+            return self.inner.driver.probe_transition_candidate(object).await;
+        };
+        reconciler
+            .probe_transition_candidate_for(
+                object,
+                crate::services::tier::warm_backend::TransitionCandidateIdentity {
+                    transaction_id,
+                    destination_id: self.backend_identity(),
+                },
+            )
+            .await
+    }
+
     pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
         self.inner.driver.validate_remote_version_id(remote_version_id)?;
         if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
@@ -2833,6 +2867,7 @@ impl TierConfigMgr {
                     let exact_get_delete = tier_exact_get_delete(config);
                     Some(PreparedTierDriver {
                         tier_name: tier_name.to_string(),
+                        tier_config: config.clone(),
                         config_fingerprint,
                         backend_identity,
                         exact_get_delete,
@@ -2861,11 +2896,13 @@ impl TierConfigMgr {
             })?;
             let entry = Arc::new(TierDriverGeneration {
                 tier_name: Arc::from(prepared.tier_name.as_str()),
+                tier_config: prepared.tier_config.clone(),
                 generation,
                 config_fingerprint: prepared.config_fingerprint,
                 backend_identity: prepared.backend_identity,
                 exact_get_delete: prepared.exact_get_delete,
                 driver: prepared.driver.clone(),
+                reconciler: tokio::sync::OnceCell::new(),
                 accepting: AtomicBool::new(true),
                 active_leases: AtomicUsize::new(0),
                 drained: tokio::sync::Notify::new(),
@@ -3609,11 +3646,13 @@ impl TierConfigMgr {
         let driver: SharedWarmBackend = Arc::from(driver);
         let entry = Arc::new(TierDriverGeneration {
             tier_name: Arc::from(tier_name),
+            tier_config: config.clone(),
             generation,
             config_fingerprint,
             backend_identity,
             exact_get_delete,
             driver: driver.clone(),
+            reconciler: tokio::sync::OnceCell::new(),
             accepting: AtomicBool::new(true),
             active_leases: AtomicUsize::new(0),
             drained: tokio::sync::Notify::new(),

crates/ecstore/src/services/tier/warm_backend.rs

@@ -73,6 +73,21 @@ pub enum TransitionCandidateProbe {
     Unsupported,
 }
 
+#[derive(Clone, Copy)]
+pub(crate) struct TransitionCandidateIdentity {
+    pub transaction_id: uuid::Uuid,
+    pub destination_id: [u8; 32],
+}
+
+#[async_trait::async_trait]
+pub(crate) trait TransitionCandidateReconciler {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error>;
+}
+
 #[async_trait::async_trait]
 pub trait WarmBackend {
     async fn validate(&self) -> Result<(), std::io::Error> {
@@ -189,6 +204,20 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
         metadata.remove(key);
     }
 
+    for suffix in [
+        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+        rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+    ] {
+        for key in [
+            rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
+            format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
+        ] {
+            if let Some(value) = metadata.remove(&key) {
+                metadata.insert(format!("x-amz-meta-{key}"), value);
+            }
+        }
+    }
+
     opts.user_metadata = metadata;
     opts
 }
@@ -446,6 +475,51 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
     Ok(d)
 }
 
+pub(crate) async fn new_transition_candidate_reconciler(
+    tier: &TierConfig,
+) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
+    let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
+        TierType::S3 => Box::new(
+            WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::MinIO => Box::new(
+            WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::RustFS => Box::new(
+            WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        TierType::R2 => Box::new(
+            WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
+                .await
+                .map_err(|err| {
+                    let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
+                    admin_err.message = err.to_string();
+                    admin_err
+                })?,
+        ),
+        _ => return Ok(None),
+    };
+    Ok(Some(reconciler))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -775,6 +849,37 @@ mod tests {
         assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
     }
 
+    #[test]
+    fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            "5a".repeat(32),
+        );
+
+        let opts = build_transition_put_options("COLD".to_string(), metadata);
+
+        for suffix in [
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+        ] {
+            assert!(opts.user_metadata.contains_key(&format!(
+                "x-amz-meta-{}",
+                rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
+            )));
+            assert!(opts.user_metadata.contains_key(&format!(
+                "x-amz-meta-{}{suffix}",
+                rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
+            )));
+        }
+    }
+
     #[test]
     fn build_transition_put_options_requests_no_checksum_and_content_md5() {
         // Regression for rustfs/rustfs#4811: transition uploads must leave the

crates/ecstore/src/services/tier/warm_backend_minio.rs

@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendMinIO {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_r2.rs

@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendR2 {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_rustfs.rs

@@ -132,6 +132,20 @@ impl WarmBackend for WarmBackendRustFS {
     }
 }
 
+#[async_trait::async_trait]
+impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
+            &self.0, object, identity,
+        )
+        .await
+    }
+}
+
 fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
     let mut object_size = object_size;
     if object_size == -1 {

crates/ecstore/src/services/tier/warm_backend_s3.rs

@@ -37,7 +37,10 @@ use crate::error::ErrorResponse;
 use crate::error::error_resp_to_object_err;
 use crate::services::tier::{
     tier_config::TierS3,
-    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
+    warm_backend::{
+        TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
+        build_transition_put_options,
+    },
 };
 use http::HeaderMap;
 use rustfs_utils::egress::validate_outbound_url;
@@ -219,6 +222,92 @@ impl WarmBackendS3 {
             advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
         }
     }
+
+    async fn probe_transition_candidate_identity(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+        bucket_versioning: RemoteBucketVersioning,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
+        let mut opts = ListObjectsOptions::default();
+        opts.set("prefix", &remote_object);
+        opts.set("max-keys", "1000");
+        let mut key_marker = String::new();
+        let mut version_id_marker = String::new();
+        let mut matched_version = None;
+        let mut saw_unproven_candidate = false;
+
+        loop {
+            let versions = self
+                .client
+                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
+                .await?;
+            for version in versions.versions.iter().filter(|version| version.key == remote_object) {
+                let mut stat_opts = GetObjectOptions::default();
+                stat_opts.version_id.clone_from(&version.version_id);
+                let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
+                let mut metadata = info.user_metadata;
+                for (name, value) in &info.metadata {
+                    if (name
+                        .as_str()
+                        .starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
+                        || name
+                            .as_str()
+                            .starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
+                        && let Ok(value) = value.to_str()
+                    {
+                        metadata.insert(name.as_str().to_string(), value.to_string());
+                    }
+                }
+                if transition_candidate_metadata_matches(&metadata, identity)? {
+                    if matched_version.is_some() {
+                        return Ok(TransitionCandidateProbe::Ambiguous);
+                    }
+                    matched_version = Some(version.version_id.clone());
+                } else {
+                    saw_unproven_candidate = true;
+                }
+            }
+            if !versions.is_truncated {
+                if matched_version.is_none() && saw_unproven_candidate {
+                    return Ok(TransitionCandidateProbe::Unsupported);
+                }
+                let candidates = TransitionCandidateVersions {
+                    version_id: matched_version,
+                    ambiguous: false,
+                };
+                return classify_transition_candidates(candidates, bucket_versioning);
+            }
+            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
+        }
+    }
+}
+
+fn transition_candidate_metadata_matches(
+    metadata: &HashMap<String, String>,
+    identity: TransitionCandidateIdentity,
+) -> Result<bool, std::io::Error> {
+    use rustfs_utils::http::metadata_compat::{
+        SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
+    };
+
+    let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
+    let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
+    if transaction_id.is_none() || destination_id.is_none() {
+        if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
+            || contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
+        {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidData,
+                "transition candidate identity metadata is empty or conflicting",
+            ));
+        }
+        return Ok(false);
+    }
+    let expected_transaction_id = identity.transaction_id.to_string();
+    let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
+    Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
 }
 
 fn classify_transition_candidates(
@@ -342,6 +431,60 @@ mod tests {
         candidates.classify(bucket_versioning)
     }
 
+    fn candidate_identity() -> TransitionCandidateIdentity {
+        TransitionCandidateIdentity {
+            transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
+            destination_id: [0x5a; 32],
+        }
+    }
+
+    fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            identity.transaction_id.to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut metadata,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(identity.destination_id),
+        );
+        metadata
+    }
+
+    #[test]
+    fn transition_candidate_identity_requires_exact_compatible_metadata() {
+        let identity = candidate_identity();
+        let metadata = candidate_metadata(identity);
+        assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
+
+        let mut adjacent = metadata;
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut adjacent,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
+                .unwrap()
+                .to_string(),
+        );
+        assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
+
+        assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
+    }
+
+    #[test]
+    fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
+        let identity = candidate_identity();
+        let mut metadata = candidate_metadata(identity);
+        metadata.insert(
+            rustfs_utils::http::metadata_compat::internal_key_rustfs(
+                rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            ),
+            uuid::Uuid::new_v4().to_string(),
+        );
+        assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
+    }
+
     #[test]
     fn transition_candidate_probe_classifier_is_fail_closed() {
         assert_eq!(
@@ -503,3 +646,16 @@ impl WarmBackend for WarmBackendS3 {
         Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
     }
 }
+
+#[async_trait::async_trait]
+impl TransitionCandidateReconciler for WarmBackendS3 {
+    async fn probe_transition_candidate_for(
+        &self,
+        object: &str,
+        identity: TransitionCandidateIdentity,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let bucket_versioning = self.remote_bucket_versioning().await?;
+        self.probe_transition_candidate_identity(object, identity, bucket_versioning)
+            .await
+    }
+}

crates/ecstore/src/set_disk/ops/object.rs

@@ -2320,13 +2320,25 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
     }
 }
 
+#[cfg(test)]
 fn persisted_transition_version(
     remote_version: &str,
 ) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
     persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
 }
 
+#[cfg(test)]
 fn remote_version_state_writer_enabled() -> bool {
+    remote_version_state_writer_fleet_proof().is_some()
+}
+
+fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
+    remote_version_state_writer_requested()
+        .then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
+        .flatten()
+}
+
+fn remote_version_state_writer_requested() -> bool {
     remote_version_state_writer_enabled_for(
         rustfs_utils::get_env_bool(
             rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
@@ -2336,11 +2348,25 @@ fn remote_version_state_writer_enabled() -> bool {
             rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
             rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
         ),
+        true,
     )
 }
 
-fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
-    requested && fleet_confirmed
+fn remote_version_state_writer_fleet_proof_matches(
+    proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
+) -> bool {
+    remote_version_state_writer_fleet_proof_matches_for(
+        remote_version_state_writer_requested(),
+        crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
+    )
+}
+
+fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
+    requested && fleet_proof_matches
+}
+
+fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
+    requested && fleet_confirmed && fleet_proof_valid
 }
 
 fn persisted_transition_version_with_gate(
@@ -2359,7 +2385,7 @@ fn persisted_transition_version_with_gate(
         Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
         Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
             std::io::ErrorKind::Unsupported,
-            "opaque remote tier versions require the operator-attested fleet gate",
+            "opaque remote tier versions require the operator-attested live fleet capability gate",
         )),
         Err(_) if remote_version == "null" => {
             Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
@@ -2605,7 +2631,7 @@ mod transition_upload_completion_tests {
 mod transition_version_id_tests {
     use super::{
         TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
-        remote_version_state_writer_enabled_for,
+        remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
     };
     use rustfs_filemeta::TransitionVersionState;
     use uuid::Uuid;
@@ -2648,15 +2674,35 @@ mod transition_version_id_tests {
 
     #[test]
     fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
-        for (case, requested, fleet_confirmed, expected) in [
-            ("old defaults", false, false, false),
-            ("missing fleet confirmation", true, false, false),
-            ("missing local opt-in", false, true, false),
-            ("explicitly unconfirmed fleet", true, false, false),
-            ("rolled-back writer", false, true, false),
-            ("fully upgraded fleet", true, true, true),
+        for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
+            ("old defaults", false, false, false, false),
+            ("missing fleet confirmation", true, false, true, false),
+            ("missing local opt-in", false, true, true, false),
+            ("missing fleet proof", true, true, false, false),
+            ("explicitly unconfirmed fleet", true, false, true, false),
+            ("rolled-back writer", false, true, true, false),
+            ("fully upgraded fleet", true, true, true, true),
         ] {
-            assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
+            assert_eq!(
+                remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
+                expected,
+                "{case}"
+            );
+        }
+    }
+
+    #[test]
+    fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
+        for (case, requested, fleet_proof_matches, expected) in [
+            ("operator gate closed", false, true, false),
+            ("fleet proof changed", true, false, false),
+            ("current authorization", true, true, true),
+        ] {
+            assert_eq!(
+                remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
+                expected,
+                "{case}"
+            );
         }
     }
 
@@ -3841,6 +3887,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let dest_obj = transaction.remote_object.clone();
         let mut transition_meta = (*oi.user_defined).clone();
         transition_meta.insert("name".to_string(), object.to_string());
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut transition_meta,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
+            transaction.transaction_id.to_string(),
+        );
+        rustfs_utils::http::metadata_compat::insert_str(
+            &mut transition_meta,
+            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
+            rustfs_utils::crypto::hex(transaction.backend_fingerprint),
+        );
 
         if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
             transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
@@ -3951,20 +4007,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
             return Err(err.into());
         }
-        let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
-            Ok(version) => version,
-            Err(err) => {
-                let cleanup_api = transition_cleanup_store(&self.ctx).await;
-                if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
-                    return Err(StorageError::Io(std::io::Error::other(format!(
-                        "{err}; rejected remote upload cleanup failed: {cleanup_err}"
-                    ))));
+        let fleet_proof = remote_version_state_writer_fleet_proof();
+        let remote_version_requires_fleet_proof =
+            !candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
+        let (transition_version_id, transition_version_state) =
+            match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
+                Ok(version) => version,
+                Err(err) => {
+                    let cleanup_api = transition_cleanup_store(&self.ctx).await;
+                    if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
+                        return Err(StorageError::Io(std::io::Error::other(format!(
+                            "{err}; rejected remote upload cleanup failed: {cleanup_err}"
+                        ))));
+                    }
+                    delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
+                        .await;
+                    return Err(err.into());
                 }
-                delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
-                    .await;
-                return Err(err.into());
-            }
-        };
+            };
         if let Err(err) = advance_and_save_transition_transaction(
             transaction_api.as_ref(),
             &mut transaction,
@@ -4080,6 +4140,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
         #[cfg(test)]
         pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
+        // This check is the fleet-proof lease linearization point. Revocation
+        // blocks later commits; an already-authorized local quorum commit is
+        // allowed to finish without holding a synchronous lock across I/O.
+        if remote_version_requires_fleet_proof
+            && !fleet_proof
+                .as_ref()
+                .is_some_and(remote_version_state_writer_fleet_proof_matches)
+        {
+            drop(transition_lock_guard);
+            if upload_cleanup.cleanup().await.is_ok() {
+                delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
+                    .await;
+            }
+            return Err(Error::other("remote version state fleet capability changed during transition"));
+        }
         if let Err(err) = advance_and_save_transition_transaction(
             transaction_api.as_ref(),
             &mut transaction,

crates/ecstore/src/store/init.rs

@@ -562,10 +562,12 @@ mod tests {
             },
             tier_sweeper::Jentry,
             transition_transaction::{
-                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
-                TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
-                TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
-                save_transition_transaction_record,
+                TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
+                TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
+                TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator, load_transition_transaction_record,
+                recover_transition_transaction_records, save_transition_transaction_record,
             },
         },
         client::transition_api::ReaderImpl,
@@ -575,6 +577,7 @@ mod tests {
         services::tier::{
             test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
             tier::{TIER_CONFIG_FILE, TierConfigMgr},
+            tier_config::{TierConfig, TierType, TierWasabi},
             tier_mutation_intent::{
                 TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
                 TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -1163,6 +1166,37 @@ mod tests {
             .len()
     }
 
+    #[cfg(feature = "test-util")]
+    async fn register_operator_reconcile_test_tier(
+        handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
+        tier_name: &str,
+    ) -> MockWarmBackend {
+        let backend = MockWarmBackend::new();
+        let mut manager = handle.write().await;
+        manager.tiers.insert(
+            tier_name.to_string(),
+            TierConfig {
+                version: "v1".to_string(),
+                tier_type: TierType::Wasabi,
+                name: tier_name.to_string(),
+                wasabi: Some(TierWasabi {
+                    name: tier_name.to_string(),
+                    endpoint: "https://s3.wasabisys.com".to_string(),
+                    access_key: "test-access-key".to_string(),
+                    secret_key: "test-secret-key".to_string(),
+                    bucket: "mock-tier".to_string(),
+                    prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
+                    region: "us-east-1".to_string(),
+                }),
+                ..Default::default()
+            },
+        );
+        manager
+            .install_test_driver(tier_name, Box::new(backend.clone()))
+            .expect("mock fallback tier driver should install");
+        backend
+    }
+
     #[cfg(feature = "test-util")]
     async fn wait_for_tier_delete_journal_recovery(
         store: Arc<crate::store::ECStore>,
@@ -2809,6 +2843,157 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXOPERATOR";
+        let backend = register_operator_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: None,
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Unversioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("transaction should enter unknown upload outcome state");
+
+        let remote_version = uuid::Uuid::new_v4().to_string();
+        backend.set_put_remote_version(Some(remote_version.clone())).await;
+        let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
+        backend
+            .put(
+                &transaction.remote_object,
+                ReaderImpl::Body(candidate.clone()),
+                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
+            )
+            .await
+            .expect("mock backend should accept candidate");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+
+        let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
+            .await
+            .expect("operator inspection should probe the candidate");
+        assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
+
+        let wrong_version = uuid::Uuid::new_v4().to_string();
+        let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
+            .await
+            .expect_err("a mismatched exact version must fail before deleting a candidate");
+        assert!(matches!(
+            err,
+            TransitionOperatorError::CandidateVersionMismatch {
+                expected,
+                actual: TransitionOperatorProbe::VersionedPresent(ref observed),
+            } if expected == wrong_version && observed == &remote_version
+        ));
+        assert!(backend.contains(&transaction.remote_object).await);
+        assert_eq!(backend.exact_remove_count(), 0);
+        load_transition_transaction_record(store.clone(), transaction.transaction_id)
+            .await
+            .expect("an incorrect exact version must retain the transaction journal");
+
+        let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
+            .await
+            .expect("operator-confirmed exact candidate should be deleted");
+        assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
+        assert!(result.journal_observed_after_delete);
+        assert_eq!(backend.exact_remove_count(), 1);
+        assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
+        load_transition_transaction_record(store.clone(), transaction.transaction_id)
+            .await
+            .expect("candidate deletion must retain the transaction journal");
+
+        finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
+            .await
+            .expect("a separately confirmed missing candidate should permit finalization");
+        assert!(matches!(
+            load_transition_transaction_record(store, transaction.transaction_id).await,
+            Err(Error::ConfigNotFound)
+        ));
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn operator_finalize_retains_record_without_missing_proof() {
+        let temp_dir = tempfile::tempdir().expect("create temp store dir");
+        let (ctx, store, _shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+
+        let tier_name = "TXOPERATORFAIL";
+        let backend = register_operator_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
+        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
+            .await
+            .expect("tier lease should resolve")
+            .backend_identity();
+        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
+            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
+            transaction_id: uuid::Uuid::new_v4(),
+            owner_epoch: uuid::Uuid::new_v4(),
+            write_id: uuid::Uuid::new_v4(),
+            source: TransitionSourceIdentity {
+                bucket: "source-bucket".to_string(),
+                object: "source-object".to_string(),
+                version_id: None,
+                data_dir: uuid::Uuid::new_v4(),
+                mod_time_unix_nanos: 1_770_000_000_000_000_000,
+                size: 42,
+                etag: "source-etag".to_string(),
+                version_mode: TransitionSourceVersionMode::Unversioned,
+            },
+            tier_name: tier_name.to_string(),
+            backend_fingerprint: backend_identity,
+            not_after_unix_nanos: 1,
+        })
+        .expect("transaction should build");
+        transaction
+            .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
+            .expect("transaction should enter unknown upload outcome state");
+        save_transition_transaction_record(store.clone(), &transaction)
+            .await
+            .expect("transaction record should persist");
+        backend
+            .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
+            .await;
+
+        assert!(matches!(
+            finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
+            Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
+        ));
+        load_transition_transaction_record(store, transaction.transaction_id)
+            .await
+            .expect("an unsupported probe must retain the transaction journal");
+        assert_eq!(backend.remove_count().await, 0);
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]

crates/protocols/src/swift/router.rs

@@ -84,19 +84,20 @@ impl SwiftRouter {
         Self { enabled, url_prefix }
     }
 
-    /// Return whether a URI matches the Swift route shape without allocating
-    /// decoded route components.
-    pub fn matches(&self, uri: &Uri) -> bool {
-        let Some(path) = self.route_path(uri) else {
-            return false;
-        };
-        let mut segments = path.trim_start_matches('/').split('/');
-        segments.next() == Some("v1") && segments.next().is_some_and(Self::is_valid_account)
-    }
-
     /// Parse a URI and return a SwiftRoute if it matches Swift URL pattern
     pub fn route(&self, uri: &Uri, method: Method) -> Option<SwiftRoute> {
-        let path = self.route_path(uri)?;
+        if !self.enabled {
+            return None;
+        }
+
+        let path = uri.path();
+
+        // Strip optional prefix
+        let path = if let Some(prefix) = &self.url_prefix {
+            path.strip_prefix(&format!("/{}/", prefix))?
+        } else {
+            path
+        };
 
         // Split path into segments - preserve empty segments to maintain object key fidelity
         // Swift allows trailing slashes and consecutive slashes in object names (e.g., "dir/" or "a//b")
@@ -174,17 +175,6 @@ impl SwiftRouter {
     fn is_valid_account(account: &str) -> bool {
         ACCOUNT_PATTERN.is_match(account)
     }
-
-    fn route_path<'a>(&self, uri: &'a Uri) -> Option<&'a str> {
-        if !self.enabled {
-            return None;
-        }
-        let path = uri.path();
-        let Some(prefix) = &self.url_prefix else {
-            return Some(path);
-        };
-        path.strip_prefix('/')?.strip_prefix(prefix)?.strip_prefix('/')
-    }
 }
 
 #[cfg(test)]
@@ -291,42 +281,6 @@ mod tests {
         assert_eq!(route, None);
     }
 
-    #[test]
-    fn test_matches_agrees_with_route_without_decoding_components() {
-        let router = SwiftRouter::new(true, None);
-        for path in [
-            "/v1/AUTH_project",
-            "/v1/AUTH_project/",
-            "/v1/AUTH_project/container",
-            "/v1/AUTH_project/container/a%20long/object",
-            "//v1/AUTH_project/container/object",
-            "/v1/not-a-swift-account/object",
-            "/v1/AUTH_/object",
-            "/bucket/object",
-        ] {
-            let uri = path.parse().expect("Swift route test URI");
-            assert_eq!(
-                router.matches(&uri),
-                router.route(&uri, Method::GET).is_some(),
-                "classification must match full routing for {path}"
-            );
-        }
-
-        let prefixed_router = SwiftRouter::new(true, Some("swift".to_string()));
-        for path in [
-            "/swift/v1/AUTH_project/container",
-            "/swiftish/v1/AUTH_project/container",
-            "/v1/AUTH_project/container",
-        ] {
-            let uri = path.parse().expect("prefixed Swift route test URI");
-            assert_eq!(
-                prefixed_router.matches(&uri),
-                prefixed_router.route(&uri, Method::GET).is_some(),
-                "prefixed classification must match full routing for {path}"
-            );
-        }
-    }
-
     #[test]
     fn test_project_id_extraction() {
         let route = SwiftRoute::Account {

crates/protos/src/lib.rs

@@ -171,6 +171,7 @@ pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SI
 pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
 pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
 pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
+pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
 pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
 pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
 pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
@@ -197,6 +198,49 @@ pub fn is_heal_control_capability_probe(command: &[u8]) -> bool {
     command.len() == HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + 16 && command.starts_with(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX)
 }
 
+pub fn remote_version_state_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
+    let mut probe = Vec::with_capacity(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
+    probe.extend_from_slice(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX);
+    probe.extend_from_slice(nonce);
+    probe
+}
+
+pub fn is_remote_version_state_capability_probe(command: &[u8]) -> bool {
+    command.len() == REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + 16
+        && command.starts_with(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)
+}
+
+pub fn encode_remote_version_state_capability(
+    topology_member: &str,
+    process_epoch: &[u8; 16],
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let topology_member = topology_member.as_bytes();
+    let mut result = Vec::with_capacity(8 + topology_member.len() + process_epoch.len());
+    result.extend_from_slice(&u64::try_from(topology_member.len())?.to_be_bytes());
+    result.extend_from_slice(topology_member);
+    result.extend_from_slice(process_epoch);
+    Ok(result)
+}
+
+pub fn decode_remote_version_state_capability(result: &[u8]) -> Result<(&str, &[u8; 16]), &'static str> {
+    let member_len = result
+        .get(..8)
+        .and_then(|value| value.try_into().ok())
+        .map(u64::from_be_bytes)
+        .ok_or("remote version state capability is truncated")?;
+    let member_len = usize::try_from(member_len).map_err(|_| "remote version state member length cannot be represented")?;
+    let member_end = 8_usize
+        .checked_add(member_len)
+        .ok_or("remote version state member length overflow")?;
+    let topology_member = std::str::from_utf8(result.get(8..member_end).ok_or("remote version state member is truncated")?)
+        .map_err(|_| "remote version state member is not UTF-8")?;
+    let process_epoch = result
+        .get(member_end..)
+        .and_then(|value| value.try_into().ok())
+        .ok_or("remote version state process epoch has an invalid length")?;
+    Ok((topology_member, process_epoch))
+}
+
 /// Builds the stable byte representation authenticated for a heal-control request.
 ///
 /// This deliberately does not reuse protobuf encoding: mixed-version peers may
@@ -1656,10 +1700,12 @@ mod scanner_activity_tests {
 #[cfg(test)]
 mod heal_control_tests {
     use super::{
-        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
-        canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
+        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
+        canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body,
+        decode_remote_version_state_capability, encode_remote_version_state_capability, heal_control_capability_probe,
         heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
-        internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
+        internode_rpc_timeout, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
+        normalize_internode_rpc_timeout, remote_version_state_capability_probe,
     };
     use crate::heal_control;
     use std::time::Duration;
@@ -1716,6 +1762,29 @@ mod heal_control_tests {
         assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
     }
 
+    #[test]
+    fn remote_version_state_capability_probe_requires_exact_nonce() {
+        let probe = remote_version_state_capability_probe(&[7; 16]);
+        assert!(is_remote_version_state_capability_probe(&probe));
+        assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
+    }
+
+    #[test]
+    fn remote_version_state_capability_binds_member_and_process_epoch() {
+        let encoded =
+            encode_remote_version_state_capability("node-a:9000", &[7; 16]).expect("small capability response should encode");
+        assert_eq!(
+            decode_remote_version_state_capability(&encoded).expect("capability response should decode"),
+            ("node-a:9000", &[7; 16])
+        );
+        assert!(decode_remote_version_state_capability(&encoded[..encoded.len() - 1]).is_err());
+
+        let mut invalid_utf8 =
+            encode_remote_version_state_capability("node-a", &[7; 16]).expect("small capability response should encode");
+        invalid_utf8[8] = 0xff;
+        assert!(decode_remote_version_state_capability(&invalid_utf8).is_err());
+    }
+
     #[test]
     fn canonical_response_binds_request_and_result() {
         let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();

crates/s3select-query/Cargo.toml

@@ -25,9 +25,6 @@ keywords = ["s3-select", "query-engine", "rustfs", "Minio", "data-retrieval"]
 categories = ["web-programming", "development-tools", "data-structures"]
 documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_query/"
 
-[lints]
-workspace = true
-
 [dependencies]
 rustfs-s3select-api = { workspace = true }
 async-recursion = { workspace = true }

crates/s3select-query/src/metadata/mod.rs

@@ -76,7 +76,7 @@ impl ContextProviderExtension for MetadataProvider {
 
         let table_handle = self.build_table_handle()?;
 
-        Ok(Arc::new(TableSourceAdapter::try_new(table_ref, table_name, table_handle)?))
+        Ok(Arc::new(TableSourceAdapter::try_new(table_ref.clone(), table_name, table_handle)?))
     }
 }
 

crates/utils/src/http/metadata_compat.rs

@@ -40,6 +40,7 @@ pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
 pub const SUFFIX_TRANSITIONED_VERSION_STATE: &str = "transitioned-version-state";
 pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
 pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id";
+pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id";
 pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id";
 pub const SUFFIX_FREE_VERSION: &str = "free-version";
 pub const SUFFIX_PURGESTATUS: &str = "purgestatus";

docs/operations/tier-ilm-debugging.md

@@ -98,6 +98,39 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-obje
 
 Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned by the admin response. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
 
+## Reconcile an unknown transition upload
+
+Historical transition transactions in `upload_outcome_unknown` state can use an explicit two-stage operator workflow when the tier probe is ambiguous and the provider supports exact version deletion. The endpoint refuses transactions that are still inside their ownership window or are in any other state.
+
+First inspect the transaction without changing it:
+
+```text
+GET /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
+```
+
+If independent provider evidence identifies the exact remote version to remove, submit that opaque version identifier with explicit confirmation:
+
+```json
+POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
+{
+  "action": "delete_candidate",
+  "confirm": true,
+  "remote_version_id": "<exact-provider-version>"
+}
+```
+
+This operation performs only an exact version delete. Its response reports whether the transaction journal was still observed after the delete; background recovery may have finalized the same transaction concurrently. If the journal remains, inspect the transaction again and finalize it only after the live provider probe proves that the candidate is missing:
+
+```json
+POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
+{
+  "action": "finalize_missing",
+  "confirm": true
+}
+```
+
+`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; this endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
+
 ## Historical fixes (for context, already merged)
 
 - Expire/GET race (`NoSuchVersion` during expiry of a tiered object):

rustfs/src/admin/handlers/ilm_transition.rs

@@ -20,8 +20,10 @@ use crate::admin::storage_api::error::StorageError;
 use crate::admin::storage_api::lifecycle::{
     ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
     ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
-    ManualTransitionScopeAdmissionClaim, claim_manual_transition_scope_admission,
-    delete_manual_transition_scope_admission_if_current, enqueue_transition_for_existing_objects_scoped,
+    ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
+    claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
+    delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
+    finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
     load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
     manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
     persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
@@ -33,6 +35,7 @@ use crate::server::{ADMIN_PREFIX, RemoteAddr};
 use http::{HeaderMap, HeaderValue};
 use hyper::{Method, StatusCode};
 use matchit::Params;
+use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use rustfs_utils::{
     MaskedAccessKey,
@@ -56,6 +59,7 @@ const MAX_MANUAL_TRANSITION_DURATION_SECONDS: u64 = 3600;
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition";
 const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state";
+const EVENT_ADMIN_ILM_TRANSITION_RECONCILE: &str = "admin_ilm_transition_reconcile";
 
 static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock<Mutex<Vec<ManualTransitionRunScope>>> = OnceLock::new();
 #[cfg(feature = "e2e-test-hooks")]
@@ -218,6 +222,16 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
         format!("{ADMIN_PREFIX}/v3/ilm/transition/jobs/{{job_id}}").as_str(),
         AdminOperation(&ManualTransitionJobCancelHandler {}),
     )?;
+    r.insert(
+        Method::GET,
+        format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
+        AdminOperation(&TransitionReconcileInspectHandler {}),
+    )?;
+    r.insert(
+        Method::POST,
+        format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
+        AdminOperation(&TransitionReconcileApplyHandler {}),
+    )?;
     Ok(())
 }
 
@@ -389,6 +403,10 @@ fn log_manual_transition_completed(
 }
 
 async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<String> {
+    authorize_transition_admin_request(req, AdminAction::SetTierAction).await
+}
+
+async fn authorize_transition_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<String> {
     let Some(input_cred) = req.credentials.as_ref() else {
         return Err(s3_error!(InvalidRequest, "authentication required"));
     };
@@ -401,19 +419,122 @@ async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<
         .get::<Option<RemoteAddr>>()
         .and_then(|opt| opt.map(|addr| addr.0));
 
-    validate_admin_request(
-        &req.headers,
-        &cred,
-        owner,
-        false,
-        vec![Action::AdminAction(AdminAction::SetTierAction)],
-        remote_addr,
-    )
-    .await?;
+    validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await?;
 
     Ok(actor)
 }
 
+fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uuid> {
+    Uuid::parse_str(params.get("transaction_id").unwrap_or(""))
+        .map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
+}
+
+fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
+    match err {
+        TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
+        TransitionOperatorError::NotExpired => {
+            s3_error!(OperationAborted, "transition transaction is still inside its active ownership window")
+        }
+        TransitionOperatorError::InvalidState(_) => {
+            s3_error!(OperationAborted, "transition transaction is not eligible for operator reconciliation")
+        }
+        TransitionOperatorError::RemoteVersionRequired => {
+            s3_error!(InvalidArgument, "an exact non-empty remote version is required")
+        }
+        TransitionOperatorError::CandidateNotMissing(_) => {
+            s3_error!(OperationAborted, "remote candidate is not proven missing")
+        }
+        TransitionOperatorError::CandidateVersionMismatch { .. } => {
+            s3_error!(OperationAborted, "remote candidate version does not match requested exact version")
+        }
+        TransitionOperatorError::Store(_) | TransitionOperatorError::Remote(_) => {
+            s3_error!(InternalError, "transition reconciliation failed")
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum TransitionReconcileAction {
+    DeleteCandidate,
+    FinalizeMissing,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct TransitionReconcileRequest {
+    action: TransitionReconcileAction,
+    confirm: bool,
+    #[serde(default)]
+    remote_version_id: Option<String>,
+}
+
+enum ValidatedTransitionReconcileAction<'a> {
+    DeleteCandidate(&'a str),
+    FinalizeMissing,
+}
+
+fn validate_transition_reconcile_request(
+    request: &TransitionReconcileRequest,
+) -> S3Result<ValidatedTransitionReconcileAction<'_>> {
+    if !request.confirm {
+        return Err(s3_error!(
+            InvalidRequest,
+            "transition reconciliation requires confirm=true; use GET to inspect without changes"
+        ));
+    }
+    match request.action {
+        TransitionReconcileAction::DeleteCandidate => request
+            .remote_version_id
+            .as_deref()
+            .filter(|version_id| !version_id.is_empty())
+            .map(ValidatedTransitionReconcileAction::DeleteCandidate)
+            .ok_or_else(|| s3_error!(InvalidArgument, "delete_candidate requires remote_version_id")),
+        TransitionReconcileAction::FinalizeMissing if request.remote_version_id.is_none() => {
+            Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
+        }
+        TransitionReconcileAction::FinalizeMissing => {
+            Err(s3_error!(InvalidArgument, "finalize_missing must not include remote_version_id"))
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct TransitionCandidateDeleteResponse {
+    outcome: &'static str,
+    result: TransitionOperatorDeleteResult,
+}
+
+#[derive(Debug, Serialize)]
+struct TransitionFinalizeMissingResponse {
+    outcome: &'static str,
+    journal_retained: bool,
+    transaction_id: Uuid,
+}
+
+fn log_transition_reconcile_applied(
+    transaction_id: Uuid,
+    action: &str,
+    outcome: &str,
+    request_id: &str,
+    actor: &str,
+    remote_addr: &str,
+) {
+    info!(
+        event = EVENT_ADMIN_ILM_TRANSITION_RECONCILE,
+        component = LOG_COMPONENT_ADMIN,
+        subsystem = LOG_SUBSYSTEM_ILM_TRANSITION,
+        operation = "transition_operator_reconcile",
+        transaction_id = %transaction_id,
+        action,
+        outcome,
+        request_id = %request_id,
+        actor = %actor,
+        remote_addr = %remote_addr,
+        "admin transition reconciliation applied"
+    );
+}
+
 fn response_state(report: &ManualTransitionRunReport) -> &'static str {
     if report.was_truncated() || report.has_partial_enqueue() || report.tier_failure > 0 || report.transition_failed > 0 {
         "partial"
@@ -902,6 +1023,81 @@ impl Operation for ManualTransitionJobCancelHandler {
     }
 }
 
+pub struct TransitionReconcileInspectHandler {}
+
+#[async_trait::async_trait]
+impl Operation for TransitionReconcileInspectHandler {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
+        let transaction_id = transition_transaction_id_from_params(&params)?;
+        let Some(store) = object_store_from_extensions(&req.extensions) else {
+            return Err(s3_error!(InternalError, "object store is not initialized"));
+        };
+        let status = inspect_transition_transaction_for_operator(store, transaction_id)
+            .await
+            .map_err(map_transition_operator_error)?;
+        json_response(&status, StatusCode::OK)
+    }
+}
+
+pub struct TransitionReconcileApplyHandler {}
+
+#[async_trait::async_trait]
+impl Operation for TransitionReconcileApplyHandler {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
+        let remote_addr = admin_remote_addr(&req).unwrap_or_default();
+        let actor = authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?;
+        let transaction_id = transition_transaction_id_from_params(&params)?;
+        let Some(store) = object_store_from_extensions(&req.extensions) else {
+            return Err(s3_error!(InternalError, "object store is not initialized"));
+        };
+        let mut input = req.input;
+        let body = input
+            .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
+            .await
+            .map_err(|_| s3_error!(InvalidRequest, "transition reconciliation body is too large or unreadable"))?;
+        let request: TransitionReconcileRequest = serde_json::from_slice(&body)
+            .map_err(|_| s3_error!(InvalidRequest, "transition reconciliation request must be valid JSON"))?;
+
+        match validate_transition_reconcile_request(&request)? {
+            ValidatedTransitionReconcileAction::DeleteCandidate(remote_version_id) => {
+                let result = delete_transition_candidate_for_operator(store, transaction_id, remote_version_id)
+                    .await
+                    .map_err(map_transition_operator_error)?;
+                let outcome = if result.journal_observed_after_delete {
+                    "exact_delete_completed_journal_observed"
+                } else {
+                    "exact_delete_completed_journal_already_finalized"
+                };
+                log_transition_reconcile_applied(transaction_id, "delete_candidate", outcome, &request_id, &actor, &remote_addr);
+                json_response(&TransitionCandidateDeleteResponse { outcome, result }, StatusCode::OK)
+            }
+            ValidatedTransitionReconcileAction::FinalizeMissing => {
+                finalize_missing_transition_transaction_for_operator(store, transaction_id)
+                    .await
+                    .map_err(map_transition_operator_error)?;
+                log_transition_reconcile_applied(
+                    transaction_id,
+                    "finalize_missing",
+                    "journal_deleted_after_missing_probe",
+                    &request_id,
+                    &actor,
+                    &remote_addr,
+                );
+                json_response(
+                    &TransitionFinalizeMissingResponse {
+                        outcome: "journal_finalized",
+                        journal_retained: false,
+                        transaction_id,
+                    },
+                    StatusCode::OK,
+                )
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -931,6 +1127,65 @@ mod tests {
         }
     }
 
+    #[test]
+    fn transition_reconcile_request_is_explicit_and_fail_closed() {
+        let unconfirmed: TransitionReconcileRequest =
+            serde_json::from_slice(br#"{"action":"delete_candidate","confirm":false,"remote_version_id":"v1"}"#)
+                .expect("request should decode");
+        assert!(validate_transition_reconcile_request(&unconfirmed).is_err());
+
+        let missing_version: TransitionReconcileRequest =
+            serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true}"#).expect("request should decode");
+        assert!(validate_transition_reconcile_request(&missing_version).is_err());
+
+        let unsafe_finalize: TransitionReconcileRequest =
+            serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true,"remote_version_id":"v1"}"#)
+                .expect("request should decode");
+        assert!(validate_transition_reconcile_request(&unsafe_finalize).is_err());
+
+        let delete: TransitionReconcileRequest =
+            serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true,"remote_version_id":"opaque-v1"}"#)
+                .expect("request should decode");
+        assert!(matches!(
+            validate_transition_reconcile_request(&delete),
+            Ok(ValidatedTransitionReconcileAction::DeleteCandidate("opaque-v1"))
+        ));
+
+        let finalize: TransitionReconcileRequest =
+            serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true}"#).expect("request should decode");
+        assert!(matches!(
+            validate_transition_reconcile_request(&finalize),
+            Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
+        ));
+
+        assert!(
+            serde_json::from_slice::<TransitionReconcileRequest>(
+                br#"{"action":"finalize_missing","confirm":true,"unexpected":true}"#
+            )
+            .is_err()
+        );
+    }
+
+    #[test]
+    fn transition_reconcile_routes_use_read_and_write_tier_actions() {
+        let src = include_str!("ilm_transition.rs");
+        let inspect = src
+            .split("impl Operation for TransitionReconcileInspectHandler")
+            .nth(1)
+            .and_then(|block| block.split("impl Operation for TransitionReconcileApplyHandler").next())
+            .expect("inspect handler block");
+        assert!(inspect.contains("AdminAction::ListTierAction"));
+        assert!(!inspect.contains("AdminAction::SetTierAction"));
+
+        let apply = src
+            .split("impl Operation for TransitionReconcileApplyHandler")
+            .nth(1)
+            .and_then(|block| block.split("#[cfg(test)]").next())
+            .expect("apply handler block");
+        assert!(apply.contains("AdminAction::SetTierAction"));
+        assert!(!apply.contains("AdminAction::ListTierAction"));
+    }
+
     #[test]
     fn manual_transition_query_defaults_to_bounded_run() {
         let (bucket, options, run_mode) =

rustfs/src/admin/handlers/site_replication.rs

@@ -7205,7 +7205,7 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
         return bucket_status;
     };
     let _targets_guard = lock_bucket_targets_metadata(bucket).await;
-    let _transaction_guard = match metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await {
+    let _transaction_guard = match metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await {
         Ok(guard) => guard,
         Err(_) => {
             bucket_status.status = "failed".to_string();

rustfs/src/admin/route_policy.rs

@@ -432,6 +432,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         SET_TIER,
         RouteRiskLevel::High,
     ),
+    admin(
+        HttpMethod::Get,
+        "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
+        LIST_TIER,
+        RouteRiskLevel::High,
+    ),
+    admin(
+        HttpMethod::Post,
+        "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
+        SET_TIER,
+        RouteRiskLevel::High,
+    ),
     admin(
         HttpMethod::Get,
         "/rustfs/admin/v3/audit/target/list",
@@ -1825,13 +1837,17 @@ mod tests {
     }
 
     #[test]
-    fn route_policy_requires_set_tier_for_manual_transition_routes() {
+    fn route_policy_uses_tier_actions_for_transition_routes() {
         assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER);
         assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
         assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
+        assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
+        assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
         assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SERVER_INFO);
         assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
         assert_not_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
+        assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
+        assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
     }
 
     #[test]

rustfs/src/admin/route_registration_test.rs

@@ -202,6 +202,16 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
             "/v3/ilm/transition/jobs/{job_id}",
             "/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111",
         ),
+        admin_route_sample(
+            Method::GET,
+            "/v3/ilm/transition/reconcile/{transaction_id}",
+            "/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
+        ),
+        admin_route_sample(
+            Method::POST,
+            "/v3/ilm/transition/reconcile/{transaction_id}",
+            "/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
+        ),
         admin_route_sample(
             Method::DELETE,
             "/v3/ilm/transition/jobs/{job_id}",
@@ -856,6 +866,16 @@ fn test_register_routes_cover_representative_admin_paths() {
         Method::DELETE,
         &admin_path("/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111"),
     );
+    assert_route(
+        &router,
+        Method::GET,
+        &admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
+    );
+    assert_route(
+        &router,
+        Method::POST,
+        &admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
+    );
 
     assert_route(&router, Method::GET, &table_catalog_path("/config"));
     assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics"));

rustfs/src/admin/router.rs

@@ -2478,7 +2478,7 @@ async fn start_replication_resync(bucket: &str, reset: &ReplicationResetStartReq
     };
 
     let _targets_guard = lock_bucket_targets_metadata(bucket).await;
-    let _transaction_guard = metadata_sys::acquire_bucket_metadata_transaction_lock(bucket)
+    let _transaction_guard = metadata_sys::acquire_bucket_targets_transaction_lock(bucket)
         .await
         .map_err(ApiError::from)?;
     let (config, _) = metadata_sys::get_replication_config(bucket).await.map_err(ApiError::from)?;

rustfs/src/admin/storage_api.rs

@@ -210,6 +210,10 @@ pub(crate) mod lifecycle {
     pub(crate) type ManualTransitionRunOptions =
         super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions;
     pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport;
+    pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{
+        TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator,
+        finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
+    };
 
     pub(crate) async fn enqueue_transition_for_existing_objects_scoped(
         api: std::sync::Arc<super::ECStore>,
@@ -282,8 +286,8 @@ pub(crate) mod metadata_sys {
         crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await
     }
 
-    pub(crate) async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
-        crate::storage::storage_api::acquire_bucket_metadata_transaction_lock(bucket).await
+    pub(crate) async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
+        crate::storage::storage_api::acquire_bucket_targets_transaction_lock(bucket).await
     }
 
     pub(crate) async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec<u8>) -> Result<OffsetDateTime> {

rustfs/src/app/object_usecase.rs

@@ -72,7 +72,7 @@ use super::storage_api::object_usecase::error::{
     is_err_version_not_found,
 };
 use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
-use super::storage_api::object_usecase::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context};
+use super::storage_api::object_usecase::helper::{OperationHelper, spawn_background_with_context};
 use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader};
 #[cfg(test)]
 use super::storage_api::object_usecase::object_cache::GetObjectBodySource;
@@ -130,7 +130,9 @@ use rustfs_object_capacity::capacity_manager::get_capacity_manager;
 use rustfs_policy::policy::action::{Action, S3Action};
 use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object};
 use rustfs_s3select_api::object_store::bytes_stream;
-use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent};
+use rustfs_targets::{
+    EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
+};
 use rustfs_utils::CompressionAlgorithm;
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
@@ -6538,7 +6540,6 @@ impl DefaultObjectUsecase {
         }
 
         let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event();
-        let request_context = helper.request_context_or_from_request(&req);
         let (bucket, delete) = {
             let bucket = req.input.bucket.clone();
             let delete = req.input.delete.clone();
@@ -6947,12 +6948,10 @@ impl DefaultObjectUsecase {
 
         let req_headers = req.headers.clone();
         let notify = current_notify_interface_for_context(self.context.as_deref());
-        let req_params = rustfs_targets::extract_params_header(&req_headers);
-        let resp_elements =
-            build_event_resp_elements(&S3Response::new(DeleteObjectsOutput::default()), &request_context.request_id);
+        let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
         let deleted_any = delete_results.iter().any(|result| result.delete_object.is_some());
         let notify_bucket = bucket.clone();
-        spawn_background_with_context(Some(request_context), async move {
+        spawn_background_with_context(request_context, async move {
             let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify);
             for res in delete_results {
                 if let Some(dobj) = res.delete_object {
@@ -6967,8 +6966,8 @@ impl DefaultObjectUsecase {
                         }),
                     )
                     .version_id(dobj.version_id.map(|v| v.to_string()).unwrap_or_default())
-                    .req_params(req_params.clone())
-                    .resp_elements(resp_elements.clone())
+                    .req_params(extract_params_header(&req_headers))
+                    .resp_elements(extract_resp_elements(&S3Response::new(DeleteObjectsOutput::default())))
                     .host(get_request_host(&req_headers))
                     .user_agent(get_request_user_agent(&req_headers))
                     .build();
@@ -7850,7 +7849,6 @@ impl DefaultObjectUsecase {
     #[instrument(level = "debug", skip(self, req))]
     pub async fn execute_put_object_extract(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
         let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event();
-        let request_context = helper.request_context_or_from_request(&req);
         let auth_method = req.method.clone();
         let auth_uri = req.uri.clone();
         let auth_headers = req.headers.clone();
@@ -8027,7 +8025,7 @@ impl DefaultObjectUsecase {
         };
 
         let notify = current_notify_interface_for_context(self.context.as_deref());
-        let req_params = rustfs_targets::extract_params_header(&req.headers);
+        let req_params = extract_params_header(&req.headers);
         let host = get_request_host(&req.headers);
         let port = get_request_port(&req.headers);
         let user_agent = get_request_user_agent(&req.headers);
@@ -8256,7 +8254,7 @@ impl DefaultObjectUsecase {
                 bucket_name: bucket.clone(),
                 object: convert_ecstore_object_info(obj_info.clone()),
                 req_params: req_params.clone(),
-                resp_elements: build_event_resp_elements(&S3Response::new(output.clone()), &request_context.request_id),
+                resp_elements: extract_resp_elements(&S3Response::new(output.clone())),
                 version_id: version_id.clone(),
                 host: host.clone(),
                 port,
@@ -8264,7 +8262,8 @@ impl DefaultObjectUsecase {
             };
 
             let notify = notify.clone();
-            spawn_background_with_context(Some(request_context.clone()), async move {
+            let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
+            spawn_background_with_context(request_context, async move {
                 notify.notify(event_args).await;
             });
         }

rustfs/src/app/storage_api.rs

@@ -853,9 +853,7 @@ pub(crate) mod head_prefix {
 }
 
 pub(crate) mod helper {
-    pub(crate) use crate::storage::storage_api::helper_consumer::{
-        OperationHelper, build_event_resp_elements, spawn_background_with_context,
-    };
+    pub(crate) use crate::storage::storage_api::helper_consumer::{OperationHelper, spawn_background_with_context};
 }
 
 pub(crate) mod object_utils {

rustfs/src/server/http.rs

@@ -23,9 +23,9 @@ use crate::server::{
     hybrid::hybrid,
     layer::{
         BodylessStatusFixLayer, ConditionalCorsLayer, DoubleSlashListBucketsCompatLayer, EmptyBodyContentLengthCompatLayer,
-        ExternalRequestContextLayer, HeadRequestBodyFixLayer, IcebergRestErrorCompatLayer, ObjectAttributesEtagFixLayer,
-        PublicHealthEndpointLayer, RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer,
-        StsQueryApiCompatLayer, VirtualHostStyleHintLayer, redact_sensitive_uri_query,
+        HeadRequestBodyFixLayer, IcebergRestErrorCompatLayer, ObjectAttributesEtagFixLayer, PublicHealthEndpointLayer,
+        RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer, StsQueryApiCompatLayer,
+        VirtualHostStyleHintLayer, redact_sensitive_uri_query,
     },
     rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env},
     tls_material::{
@@ -34,6 +34,7 @@ use crate::server::{
     },
 };
 use crate::storage_api::server::http as storage;
+use crate::storage_api::server::http::request_context::{RequestContext, extract_request_id_from_headers};
 use crate::storage_api::server::http::rpc::InternodeRpcService;
 use crate::storage_api::server::http::tonic_service::make_server;
 use crate::storage_api::server::http::{
@@ -1134,6 +1135,47 @@ struct PathDispatchService<A, B> {
     internode: B,
 }
 
+#[derive(Clone, Default)]
+struct InternodeRequestContextLiteLayer;
+
+impl<S> tower::Layer<S> for InternodeRequestContextLiteLayer {
+    type Service = InternodeRequestContextLiteService<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        InternodeRequestContextLiteService { inner }
+    }
+}
+
+#[derive(Clone)]
+struct InternodeRequestContextLiteService<S> {
+    inner: S,
+}
+
+impl<S, B> Service<HttpRequest<B>> for InternodeRequestContextLiteService<S>
+where
+    S: Service<HttpRequest<B>> + Clone,
+{
+    type Response = S::Response;
+    type Error = S::Error;
+    type Future = S::Future;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
+        let request_id = extract_request_id_from_headers(req.headers());
+        req.extensions_mut().insert(RequestContext {
+            x_amz_request_id: request_id.clone(),
+            request_id,
+            trace_id: None,
+            span_id: None,
+            start_time: std::time::Instant::now(),
+        });
+        self.inner.call(req)
+    }
+}
+
 impl<A, B> PathDispatchService<A, B> {
     fn new(external: A, internode: B) -> Self {
         Self { external, internode }
@@ -1346,28 +1388,34 @@ fn process_connection(
         //  1. AddExtensionLayer<RemoteAddr>           — per-connection peer address
         //  2. AddExtensionLayer<SocketAddr>           — per-connection raw socket addr (TrustedProxy)
         //  3. TrustedProxyLayer                       — conditional, parses X-Forwarded-For
-        //  4. ExternalRequestContextLayer            — S3 canonical ID / control-plane propagated ID
-        //  5. StsQueryApiCompatLayer                 — route-scoped STS envelopes, including outer short-circuit errors
-        //  6. EmptyBodyContentLengthCompatLayer       — adds Content-Length: 0 for known empty-body API routes
-        //  7. CatchPanicLayer                        — panic → 500
-        //  8. RateLimitLayer                         — conditional (external stack only), per-client 429 throttling
-        //  9. ReadinessGateLayer                     — blocks until ready
-        // 10. KeystoneAuthLayer                      — X-Auth-Token validation
-        // 11. TraceLayer                             — request span creation + metrics
-        // 12. RequestLoggingLayer                    — single completion event per request
-        // 13. CompressionLayer                       — response compression (whitelist, path-aware)
-        // 14. PathCategoryInjectionLayer             — injects path category for compression predicate
-        // 15. S3ErrorMessageCompatLayer              — missing S3 error message compatibility
-        // 16. IcebergRestErrorCompatLayer            — Iceberg REST JSON error compatibility
-        // 17. ObjectAttributesEtagFixLayer           — ETag fix for GetObjectAttributes
-        // 18. ConditionalCorsLayer                   — S3 API CORS
-        // 19. RedirectLayer                          — console redirect (conditional)
-        // 20. BodylessStatusFixLayer                 — clears body for 1xx/204/205/304 responses
-        // 21. HeadRequestBodyFixLayer                — strips actual body bytes from HEAD responses
-        // 22. PublicHealthEndpointLayer              — handles public health before s3s host parsing
-        // 23. VirtualHostStyleHintLayer              — actionable error for unroutable virtual-hosted-style (conditional)
-        // 24. DoubleSlashListBucketsCompatLayer      — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
+        //  4. SetRequestIdLayer                       — generates X-Request-ID
+        //  5. RequestContextLayer                    — creates RequestContext in extensions
+        //  6. StsQueryApiCompatLayer                 — route-scoped STS envelopes, including outer short-circuit errors
+        //  7. EmptyBodyContentLengthCompatLayer       — adds Content-Length: 0 for known empty-body API routes
+        //  8. CatchPanicLayer                        — panic → 500
+        //  9. RateLimitLayer                         — conditional (external stack only), per-client 429 throttling
+        // 10. ReadinessGateLayer                     — blocks until ready
+        // 11. KeystoneAuthLayer                      — X-Auth-Token validation
+        // 12. TraceLayer                             — request span creation + metrics
+        // 13. RequestLoggingLayer                    — single completion event per request
+        // 14. PropagateRequestIdLayer                — X-Request-ID → response
+        // 15. CompressionLayer                       — response compression (whitelist, path-aware)
+        // 16. PathCategoryInjectionLayer             — injects path category for compression predicate
+        // 17. S3ErrorMessageCompatLayer              — missing S3 error message compatibility
+        // 18. IcebergRestErrorCompatLayer            — Iceberg REST JSON error compatibility
+        // 19. ObjectAttributesEtagFixLayer           — ETag fix for GetObjectAttributes
+        // 20. ConditionalCorsLayer                   — S3 API CORS
+        // 21. RedirectLayer                          — console redirect (conditional)
+        // 22. BodylessStatusFixLayer                 — clears body for 1xx/204/205/304 responses
+        // 23. HeadRequestBodyFixLayer                — strips actual body bytes from HEAD responses
+        // 24. PublicHealthEndpointLayer              — handles public health before s3s host parsing
+        // 25. VirtualHostStyleHintLayer              — actionable error for unroutable virtual-hosted-style (conditional)
+        // 26. DoubleSlashListBucketsCompatLayer      — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
         // ─────────────────────────────────────────────────────────────
+        // Batch 1 intentionally keeps the external and internode stacks behaviorally
+        // identical while giving each path family a named construction boundary.
+        // Later batches will trim internode-only middleware without risking drift in
+        // the public HTTP stack.
         let build_external_stack = |service| {
             ServiceBuilder::new()
                 // NOTE: Both extension types are intentionally inserted to maintain compatibility:
@@ -1382,7 +1430,8 @@ fn process_connection(
                 // This should be placed before TraceLayer so that logs reflect the real client IP
                 // Pre-computed in ConnectionContext to avoid per-connection is_enabled() check.
                 .option_layer(trusted_proxy_layer.clone())
-                .layer(ExternalRequestContextLayer::new(is_console))
+                .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
+                .layer(InternodeRequestContextLiteLayer)
                 .layer(StsQueryApiCompatLayer)
                 .layer(EmptyBodyContentLengthCompatLayer)
                 .layer(CatchPanicLayer::new())
@@ -1535,6 +1584,7 @@ fn process_connection(
                         }),
                 )
                 .layer(RequestLoggingLayer)
+                .layer(PropagateRequestIdLayer::x_request_id())
                 .layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
                 .layer(PathCategoryInjectionLayer)
                 .layer(S3ErrorMessageCompatLayer)

rustfs/src/server/hybrid.rs

@@ -28,13 +28,6 @@ pub(crate) fn hybrid<MakeRest, Grpc>(make_rest: MakeRest, grpc: Grpc) -> HybridS
     HybridService { rest: make_rest, grpc }
 }
 
-pub(crate) fn is_grpc_request<B>(req: &Request<B>) -> bool {
-    matches!(
-        (req.version(), req.headers().get(hyper::header::CONTENT_TYPE)),
-        (hyper::Version::HTTP_2, Some(value)) if value.as_bytes().starts_with(b"application/grpc")
-    )
-}
-
 /// The service that can serve both gRPC and REST HTTP Requests
 #[derive(Clone)]
 pub struct HybridService<Rest, Grpc> {
@@ -70,14 +63,14 @@ where
     /// and if the Content-Type is "application/grpc"; otherwise, the request is served
     /// as a REST request
     fn call(&mut self, req: Request<Incoming>) -> Self::Future {
-        if is_grpc_request(&req) {
-            HybridFuture::Grpc {
+        match (req.version(), req.headers().get(hyper::header::CONTENT_TYPE)) {
+            (hyper::Version::HTTP_2, Some(hv)) if hv.as_bytes().starts_with(b"application/grpc") => HybridFuture::Grpc {
                 grpc_future: self.grpc.call(req),
-            }
-        } else {
-            HybridFuture::Rest {
+            },
+
+            _ => HybridFuture::Rest {
                 rest_future: self.rest.call(req),
-            }
+            },
         }
     }
 }

rustfs/src/server/layer.rs

@@ -17,28 +17,27 @@ use crate::admin::console::is_console_path;
 use crate::error::ApiError;
 use crate::server::RemoteAddr;
 use crate::server::cors;
-use crate::server::hybrid::{HybridBody, is_grpc_request};
+use crate::server::hybrid::HybridBody;
 use crate::server::{
     ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HealthProbe, MINIO_ADMIN_PREFIX,
     MINIO_ADMIN_V3_PREFIX, MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH,
-    MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests,
-    build_health_response_parts, collect_probe_readiness, has_path_prefix, is_admin_path, is_table_catalog_path,
+    MINIO_HEALTH_READY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests, build_health_response_parts,
+    collect_probe_readiness, has_path_prefix, is_admin_path, is_table_catalog_path,
 };
 use crate::storage_api::server::layer::apply_cors_headers;
-use crate::storage_api::server::layer::request_context::{RequestContext, extract_request_id_from_headers, spawn_traced};
+use crate::storage_api::server::layer::request_context::{
+    RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
+};
 use bytes::{Bytes, BytesMut};
 use futures::future::Either;
 use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode, Uri};
 use http_body::Body;
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
-use pin_project_lite::pin_project;
 use quick_xml::events::Event;
-#[cfg(feature = "swift")]
-use rustfs_protocols::swift::SwiftRouter;
 use rustfs_trusted_proxies::ClientInfo;
 use rustfs_utils::get_env_opt_str;
-use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
+use rustfs_utils::http::headers::AMZ_REQUEST_ID;
 use s3s::S3ErrorCode;
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
@@ -63,8 +62,6 @@ const HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD: Duration = Duration::from_secs(5);
 const STS_RESPONSE_METADATA_TAG: &str = "ResponseMetadata";
 const STS_REQUEST_ID_TAG: &str = "RequestId";
 const STS_SUCCESS_RESPONSE_TAGS: [&str; 2] = ["AssumeRoleResponse", "AssumeRoleWithWebIdentityResponse"];
-#[cfg(feature = "swift")]
-const SWIFT_API_PATH_PREFIX: &str = "/v1/";
 
 pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String {
     let path = uri.path();
@@ -120,6 +117,9 @@ fn is_object_zip_download_path(path: &str) -> bool {
 ///
 /// This layer must be placed after `SetRequestIdLayer` in the middleware stack,
 /// as it reads the `x-request-id` header that `SetRequestIdLayer` generates.
+///
+/// Additionally, it preserves any upstream `x-amz-request-id` in the separate
+/// `RequestContext.x_amz_request_id` field without mutating signed request headers.
 #[derive(Clone, Default)]
 pub struct RequestContextLayer;
 
@@ -150,163 +150,35 @@ where
     }
 
     fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
-        let request_context = RequestContext::from_headers(req.headers());
-        req.extensions_mut().insert(request_context);
+        let request_id = extract_request_id_from_headers(req.headers());
+
+        let (trace_id, span_id) = extract_trace_context_ids_from_headers(req.headers())
+            .map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))
+            .unwrap_or((None, None));
+
+        // Preserve the upstream x-amz-request-id if present as the S3 compatibility alias;
+        // otherwise mirror the canonical internal request_id.
+        let x_amz_request_id = req
+            .headers()
+            .get(AMZ_REQUEST_ID)
+            .and_then(|v| v.to_str().ok())
+            .map(String::from)
+            .unwrap_or_else(|| request_id.clone());
+
+        let ctx = RequestContext {
+            request_id,
+            x_amz_request_id,
+            trace_id,
+            span_id,
+            start_time: Instant::now(),
+        };
+
+        req.extensions_mut().insert(ctx);
 
         self.inner.call(req)
     }
 }
 
-fn uses_server_owned_s3_request_id<B>(req: &HttpRequest<B>, console_redirect_enabled: bool) -> bool {
-    if is_grpc_request(req)
-        || req.uri().path().starts_with(RPC_PREFIX)
-        || is_sts_query_request(req.method(), req.uri(), req.headers())
-        || (console_redirect_enabled && is_console_redirect_request(req))
-    {
-        return false;
-    }
-
-    #[cfg(feature = "swift")]
-    if req.uri().path().starts_with(SWIFT_API_PATH_PREFIX) && SwiftRouter::new(true, None).matches(req.uri()) {
-        return false;
-    }
-
-    let path = req.uri().path();
-    let method = req.method();
-    let is_admin_health_request =
-        (method == Method::GET || method == Method::HEAD) && matches!(path, HEALTH_PREFIX | HEALTH_READY_PATH);
-    let is_profile_request = method == Method::GET && matches!(path, PROFILE_CPU_PATH | PROFILE_MEMORY_PATH);
-    let is_public_health_alias_request = (method == Method::GET || method == Method::HEAD)
-        && matches!(
-            path,
-            HEALTH_COMPAT_LIVE_PATH
-                | MINIO_HEALTH_LIVE_PATH
-                | MINIO_HEALTH_READY_PATH
-                | MINIO_HEALTH_CLUSTER_PATH
-                | MINIO_HEALTH_CLUSTER_READ_PATH
-        )
-        && is_public_health_endpoint_request(method, path);
-
-    !(is_admin_path(path)
-        || is_console_path(path)
-        || is_admin_health_request
-        || is_profile_request
-        || is_public_health_alias_request)
-}
-
-/// Creates a server-owned context and response ID for S3 requests while
-/// preserving the existing request-ID propagation contract for non-S3 routes.
-#[derive(Clone, Default)]
-pub struct ExternalRequestContextLayer {
-    console_redirect_enabled: bool,
-}
-
-impl ExternalRequestContextLayer {
-    pub(crate) fn new(console_redirect_enabled: bool) -> Self {
-        Self {
-            console_redirect_enabled,
-        }
-    }
-}
-
-impl<S> Layer<S> for ExternalRequestContextLayer {
-    type Service = ExternalRequestContextService<S>;
-
-    fn layer(&self, inner: S) -> Self::Service {
-        ExternalRequestContextService {
-            inner,
-            console_redirect_enabled: self.console_redirect_enabled,
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct ExternalRequestContextService<S> {
-    inner: S,
-    console_redirect_enabled: bool,
-}
-
-impl<S, B, ResBody> Service<HttpRequest<B>> for ExternalRequestContextService<S>
-where
-    S: Service<HttpRequest<B>, Response = Response<ResBody>>,
-{
-    type Response = Response<ResBody>;
-    type Error = S::Error;
-    type Future = ExternalRequestContextFuture<S::Future>;
-
-    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
-        self.inner.poll_ready(cx)
-    }
-
-    fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
-        let is_s3 = uses_server_owned_s3_request_id(&req, self.console_redirect_enabled);
-        let has_request_id = req.headers().contains_key(REQUEST_ID_HEADER);
-        let request_context = if is_s3 {
-            let request_context = RequestContext::from_external_headers(req.headers());
-            if !has_request_id && let Ok(request_id) = HeaderValue::from_str(&request_context.request_id) {
-                req.headers_mut().insert(REQUEST_ID_HEADER, request_id);
-            }
-            request_context
-        } else {
-            if !has_request_id {
-                let request_id = uuid::Uuid::new_v4().to_string();
-                if let Ok(request_id) = HeaderValue::from_str(&request_id) {
-                    req.headers_mut().insert(REQUEST_ID_HEADER, request_id);
-                }
-            }
-            RequestContext::from_headers_without_trace_context(req.headers())
-        };
-        let request_id = if is_s3 {
-            HeaderValue::from_str(&request_context.request_id).ok()
-        } else {
-            req.headers().get(REQUEST_ID_HEADER).cloned()
-        };
-        req.extensions_mut().insert(request_context);
-
-        ExternalRequestContextFuture {
-            inner: self.inner.call(req),
-            request_id,
-            is_s3,
-        }
-    }
-}
-
-pin_project! {
-    pub struct ExternalRequestContextFuture<F> {
-        #[pin]
-        inner: F,
-        request_id: Option<HeaderValue>,
-        is_s3: bool,
-    }
-}
-
-impl<F, ResBody, E> Future for ExternalRequestContextFuture<F>
-where
-    F: Future<Output = Result<Response<ResBody>, E>>,
-{
-    type Output = Result<Response<ResBody>, E>;
-
-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        let this = self.project();
-        let mut response = match this.inner.poll(cx) {
-            Poll::Ready(Ok(response)) => response,
-            Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
-            Poll::Pending => return Poll::Pending,
-        };
-
-        if let Some(request_id) = this.request_id.take() {
-            if *this.is_s3 {
-                response.headers_mut().insert(REQUEST_ID_HEADER, request_id.clone());
-                response.headers_mut().insert(AMZ_REQUEST_ID, request_id);
-            } else if !response.headers().contains_key(REQUEST_ID_HEADER) {
-                response.headers_mut().insert(REQUEST_ID_HEADER, request_id);
-            }
-        }
-
-        Poll::Ready(Ok(response))
-    }
-}
-
 #[derive(Clone, Default)]
 pub struct RequestLoggingLayer;
 
@@ -523,18 +395,6 @@ pub struct RedirectService<S> {
     inner: S,
 }
 
-fn is_console_redirect_request<B>(req: &HttpRequest<B>) -> bool {
-    let path = req.uri().path().trim_end_matches('/');
-    req.method() == http::Method::GET
-        && !req.headers().contains_key(http::header::AUTHORIZATION)
-        && req
-            .headers()
-            .get(http::header::USER_AGENT)
-            .and_then(|value| value.to_str().ok())
-            .is_some_and(|user_agent| user_agent.contains("Mozilla"))
-        && (path.is_empty() || path == "/rustfs" || path == "/index.html")
-}
-
 impl<S, RestBody, GrpcBody> Service<HttpRequest<Incoming>> for RedirectService<S>
 where
     S: Service<HttpRequest<Incoming>, Response = Response<HybridBody<RestBody, GrpcBody>>> + Clone + Send + 'static,
@@ -552,8 +412,20 @@ where
     }
 
     fn call(&mut self, req: HttpRequest<Incoming>) -> Self::Future {
+        // Check if this is a GET request without Authorization header and User-Agent contains Mozilla
+        // and the path is either "/" or "/index.html"
         let path = req.uri().path().trim_end_matches('/');
-        if is_console_redirect_request(&req) {
+        let should_redirect = req.method() == http::Method::GET
+            && !req.headers().contains_key(http::header::AUTHORIZATION)
+            && req
+                .headers()
+                .get(http::header::USER_AGENT)
+                .and_then(|v| v.to_str().ok())
+                .map(|ua| ua.contains("Mozilla"))
+                .unwrap_or(false)
+            && (path.is_empty() || path == "/rustfs" || path == "/index.html");
+
+        if should_redirect {
             debug!("Redirecting browser request from {} to console", path);
 
             // Create redirect response
@@ -1861,7 +1733,7 @@ impl ConditionalCorsLayer {
         // Expose common headers
         response_headers.insert(
             cors::response::ACCESS_CONTROL_EXPOSE_HEADERS,
-            HeaderValue::from_static("x-request-id, x-amz-request-id, content-type, content-length, etag"),
+            HeaderValue::from_static("x-request-id, content-type, content-length, etag"),
         );
 
         // Credentials are only safe for origins matched from an explicit allow-list.
@@ -2174,25 +2046,12 @@ mod tests {
     #[derive(Clone, Default)]
     struct HeaderCaptureService {
         headers: Arc<Mutex<Option<HeaderMap>>>,
-        request_context: Arc<Mutex<Option<RequestContext>>>,
-        response_request_id: Option<HeaderValue>,
     }
 
     impl HeaderCaptureService {
-        fn with_response_request_id(request_id: &'static str) -> Self {
-            Self {
-                response_request_id: Some(HeaderValue::from_static(request_id)),
-                ..Self::default()
-            }
-        }
-
         fn headers(&self) -> Arc<Mutex<Option<HeaderMap>>> {
             Arc::clone(&self.headers)
         }
-
-        fn request_context(&self) -> Arc<Mutex<Option<RequestContext>>> {
-            Arc::clone(&self.request_context)
-        }
     }
 
     impl<B: Send + 'static> Service<Request<B>> for HeaderCaptureService {
@@ -2206,350 +2065,10 @@ mod tests {
 
         fn call(&mut self, req: Request<B>) -> Self::Future {
             *self.headers.lock().expect("capture headers") = Some(req.headers().clone());
-            *self.request_context.lock().expect("capture request context") = req.extensions().get::<RequestContext>().cloned();
-            let mut response = Response::new(Full::from(Bytes::new()));
-            if let Some(request_id) = self.response_request_id.clone() {
-                response.headers_mut().insert(REQUEST_ID_HEADER, request_id);
-            }
-            ready(Ok(response))
+            ready(Ok(Response::new(Full::from(Bytes::new()))))
         }
     }
 
-    async fn assert_non_s3_request_id_contract(mut request: Request<()>, route: &str) {
-        let capture = HeaderCaptureService::default();
-        let captured_context = capture.request_context();
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        request
-            .headers_mut()
-            .insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
-        request
-            .headers_mut()
-            .insert(AMZ_REQUEST_ID, HeaderValue::from_static("client-amz-request-id"));
-
-        let response = service.call(request).await.expect("non-S3 response");
-
-        assert_eq!(
-            response
-                .headers()
-                .get(REQUEST_ID_HEADER)
-                .and_then(|value| value.to_str().ok()),
-            Some("client-request-id"),
-            "non-S3 x-request-id contract changed for {route}"
-        );
-        assert!(
-            !response.headers().contains_key(AMZ_REQUEST_ID),
-            "non-S3 response unexpectedly gained x-amz-request-id for {route}"
-        );
-        let context = captured_context
-            .lock()
-            .expect("captured request context")
-            .clone()
-            .expect("non-S3 request context");
-        assert_eq!(context.request_id, "client-request-id", "non-S3 context changed for {route}");
-        assert_eq!(context.x_amz_request_id, "client-request-id");
-        assert!(context.trace_id.is_none());
-        assert!(context.span_id.is_none());
-    }
-
-    #[tokio::test]
-    async fn external_request_context_rejects_client_request_id_as_canonical() {
-        global::set_text_map_propagator(TraceContextPropagator::new());
-        let capture = HeaderCaptureService::default();
-        let captured_headers = capture.headers();
-        let captured_context = capture.request_context();
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        let mut request = Request::builder().uri("/bucket/object").body(()).expect("build S3 request");
-        request
-            .headers_mut()
-            .insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-supplied-request-id"));
-        request
-            .headers_mut()
-            .insert(AMZ_REQUEST_ID, HeaderValue::from_static("client-supplied-amz-request-id"));
-        request.headers_mut().insert(
-            "traceparent",
-            HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
-        );
-
-        let response = service.call(request).await.expect("response");
-        let request_id = response
-            .headers()
-            .get(REQUEST_ID_HEADER)
-            .and_then(|value| value.to_str().ok())
-            .expect("canonical request ID");
-
-        assert!(uuid::Uuid::parse_str(request_id).is_ok());
-        assert_ne!(request_id, "client-supplied-request-id");
-        assert_eq!(
-            response.headers().get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
-            Some(request_id)
-        );
-        let headers = captured_headers.lock().expect("captured headers");
-        let headers = headers.as_ref().expect("inner request headers");
-        assert_eq!(headers.get(REQUEST_ID_HEADER).expect("client x-request-id"), "client-supplied-request-id");
-        assert_eq!(
-            headers.get(AMZ_REQUEST_ID).expect("client x-amz-request-id"),
-            "client-supplied-amz-request-id"
-        );
-        assert_eq!(
-            headers.get("traceparent").expect("client traceparent"),
-            "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
-        );
-        let context = captured_context
-            .lock()
-            .expect("captured request context")
-            .clone()
-            .expect("S3 request context");
-        assert_eq!(context.request_id, request_id);
-        assert_eq!(context.trace_id.as_deref(), Some("4bf92f3577b34da6a3ce929d0e0e4736"));
-        assert_eq!(context.span_id.as_deref(), Some("00f067aa0ba902b7"));
-    }
-
-    #[tokio::test]
-    async fn external_request_context_replaces_empty_response_id() {
-        let capture = HeaderCaptureService::default();
-        let captured_headers = capture.headers();
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        let mut request = Request::builder().uri("/bucket/object").body(()).expect("build S3 request");
-        request.headers_mut().insert(REQUEST_ID_HEADER, HeaderValue::from_static(""));
-        request.headers_mut().insert(AMZ_REQUEST_ID, HeaderValue::from_static("   "));
-
-        let response = service.call(request).await.expect("response");
-        let request_id = response
-            .headers()
-            .get(REQUEST_ID_HEADER)
-            .and_then(|value| value.to_str().ok())
-            .expect("canonical request ID");
-
-        assert!(uuid::Uuid::parse_str(request_id).is_ok());
-        assert_eq!(
-            response.headers().get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
-            Some(request_id)
-        );
-        let headers = captured_headers.lock().expect("captured headers");
-        let headers = headers.as_ref().expect("inner request headers");
-        assert_eq!(headers.get(REQUEST_ID_HEADER).expect("empty client x-request-id"), "");
-        assert_eq!(headers.get(AMZ_REQUEST_ID).expect("blank client x-amz-request-id"), "   ");
-    }
-
-    #[tokio::test]
-    async fn external_request_context_inserts_generated_id_when_header_is_absent() {
-        let capture = HeaderCaptureService::default();
-        let captured_headers = capture.headers();
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        let request = Request::builder().uri("/bucket/object").body(()).expect("build S3 request");
-
-        let response = service.call(request).await.expect("response");
-        let response_request_id = response
-            .headers()
-            .get(REQUEST_ID_HEADER)
-            .and_then(|value| value.to_str().ok())
-            .expect("canonical request ID");
-        let headers = captured_headers.lock().expect("captured headers");
-        let headers = headers.as_ref().expect("inner request headers");
-
-        assert_eq!(
-            headers.get(REQUEST_ID_HEADER).and_then(|value| value.to_str().ok()),
-            Some(response_request_id)
-        );
-    }
-
-    #[tokio::test]
-    async fn non_s3_request_id_contract_preserves_client_correlation() {
-        for path in [
-            "/rustfs/admin/v3/info",
-            "/minio/admin/v3/info",
-            "/rustfs/console/",
-            HEALTH_PREFIX,
-            "/iceberg/v1/config",
-            "/rustfs/rpc/v1/read-file",
-            "/rustfs/rpcx",
-        ] {
-            let request = Request::builder().uri(path).body(()).expect("build non-S3 request");
-            assert_non_s3_request_id_contract(request, path).await;
-        }
-    }
-
-    #[test]
-    fn console_redirect_request_id_contract_follows_redirect_enablement() {
-        for path in ["/", "/rustfs", "/index.html"] {
-            let request = Request::builder()
-                .method(Method::GET)
-                .uri(path)
-                .header(http::header::USER_AGENT, "Mozilla/5.0")
-                .body(())
-                .expect("build console redirect request");
-
-            assert!(!uses_server_owned_s3_request_id(&request, true));
-            assert!(uses_server_owned_s3_request_id(&request, false));
-        }
-    }
-
-    #[test]
-    fn method_scoped_control_routes_preserve_request_id_contract() {
-        for path in [HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH] {
-            let control_request = Request::builder()
-                .method(Method::GET)
-                .uri(path)
-                .body(())
-                .expect("build control-plane request");
-            assert!(!uses_server_owned_s3_request_id(&control_request, false));
-
-            let s3_request = Request::builder()
-                .method(Method::POST)
-                .uri(path)
-                .body(())
-                .expect("build S3 request");
-            assert!(uses_server_owned_s3_request_id(&s3_request, false));
-        }
-    }
-
-    #[test]
-    #[serial]
-    fn public_health_alias_request_id_contract_follows_enablement() {
-        let request = Request::builder()
-            .method(Method::GET)
-            .uri(MINIO_HEALTH_LIVE_PATH)
-            .body(())
-            .expect("build health request");
-
-        with_var(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"), || {
-            assert!(!uses_server_owned_s3_request_id(&request, false));
-        });
-        with_var(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("false"), || {
-            assert!(uses_server_owned_s3_request_id(&request, false));
-        });
-    }
-
-    #[tokio::test]
-    async fn grpc_request_id_contract_preserves_client_correlation() {
-        for path in [
-            "/node_service.NodeService/GetMetrics",
-            "/node_service.HealControlService/HealControl",
-            "/node_service.TierMutationControlService/PrepareTierMutation",
-        ] {
-            let request = Request::builder()
-                .version(http::Version::HTTP_2)
-                .uri(path)
-                .header(http::header::CONTENT_TYPE, "application/grpc")
-                .body(())
-                .expect("build gRPC request");
-            assert_non_s3_request_id_contract(request, path).await;
-        }
-    }
-
-    #[tokio::test]
-    async fn sts_query_request_id_contract_preserves_client_correlation() {
-        let request = Request::builder()
-            .method(Method::POST)
-            .uri("/")
-            .header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
-            .body(())
-            .expect("build STS Query request");
-        assert_non_s3_request_id_contract(request, "STS Query").await;
-    }
-
-    #[cfg(feature = "swift")]
-    #[tokio::test]
-    async fn swift_request_id_contract_preserves_client_correlation() {
-        let path = "/v1/AUTH_project/container/object";
-        let request = Request::builder().uri(path).body(()).expect("build Swift request");
-        assert_non_s3_request_id_contract(request, path).await;
-    }
-
-    #[cfg(feature = "swift")]
-    #[test]
-    fn swift_request_id_classification_preserves_v1_prefix_boundary() {
-        let swift_request = Request::builder()
-            .uri("/v1/AUTH_project/container/object")
-            .body(())
-            .expect("build Swift request");
-        assert!(!uses_server_owned_s3_request_id(&swift_request, false));
-
-        let double_slash_request = Request::builder()
-            .uri("//v1/AUTH_project/container/object")
-            .body(())
-            .expect("build double-slash request");
-        assert!(uses_server_owned_s3_request_id(&double_slash_request, false));
-    }
-
-    #[cfg(feature = "swift")]
-    #[tokio::test]
-    async fn non_swift_v1_path_keeps_s3_request_id_contract() {
-        let capture = HeaderCaptureService::default();
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        let mut request = Request::builder()
-            .uri("/v1/not-a-swift-account/object")
-            .body(())
-            .expect("build S3 request");
-        request
-            .headers_mut()
-            .insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
-
-        let response = service.call(request).await.expect("S3 response");
-        let response_request_id = response
-            .headers()
-            .get(REQUEST_ID_HEADER)
-            .and_then(|value| value.to_str().ok())
-            .expect("S3 response request ID");
-
-        assert!(uuid::Uuid::parse_str(response_request_id).is_ok());
-        assert_eq!(
-            response.headers().get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
-            Some(response_request_id)
-        );
-    }
-
-    #[tokio::test]
-    async fn non_s3_response_preserves_handler_request_id() {
-        let capture = HeaderCaptureService::with_response_request_id("handler-request-id");
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        let mut request = Request::builder()
-            .uri("/rustfs/admin/v3/info")
-            .body(())
-            .expect("build admin request");
-        request
-            .headers_mut()
-            .insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
-
-        let response = service.call(request).await.expect("admin response");
-
-        assert_eq!(
-            response
-                .headers()
-                .get(REQUEST_ID_HEADER)
-                .and_then(|value| value.to_str().ok()),
-            Some("handler-request-id")
-        );
-        assert!(!response.headers().contains_key(AMZ_REQUEST_ID));
-    }
-
-    #[tokio::test]
-    async fn non_s3_request_without_id_generates_only_x_request_id() {
-        let capture = HeaderCaptureService::default();
-        let captured_context = capture.request_context();
-        let mut service = ExternalRequestContextLayer::default().layer(capture);
-        let request = Request::builder()
-            .uri("/rustfs/admin/v3/info")
-            .body(())
-            .expect("build admin request");
-
-        let response = service.call(request).await.expect("admin response");
-        let response_request_id = response
-            .headers()
-            .get(REQUEST_ID_HEADER)
-            .and_then(|value| value.to_str().ok())
-            .expect("generated admin request ID");
-
-        assert!(uuid::Uuid::parse_str(response_request_id).is_ok());
-        assert!(!response.headers().contains_key(AMZ_REQUEST_ID));
-        let context = captured_context
-            .lock()
-            .expect("captured request context")
-            .clone()
-            .expect("admin request context");
-        assert_eq!(context.request_id, response_request_id);
-    }
-
     #[derive(Clone, Default)]
     struct CountingHybridService {
         calls: Arc<AtomicUsize>,
@@ -4052,12 +3571,6 @@ mod tests {
             "https://allowed.com"
         );
         assert_eq!(resp_headers.get(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS).unwrap(), "true");
-        let exposed = resp_headers
-            .get(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS)
-            .and_then(|value| value.to_str().ok())
-            .expect("exposed response headers");
-        assert!(exposed.split(',').any(|header| header.trim() == "x-request-id"));
-        assert!(exposed.split(',').any(|header| header.trim() == "x-amz-request-id"));
     }
 
     #[test]
@@ -4106,7 +3619,7 @@ mod tests {
     }
 
     #[test]
-    fn request_context_layer_does_not_mutate_upstream_s3_request_id() {
+    fn request_context_layer_preserves_upstream_s3_request_id() {
         let mut service = RequestContextLayer.layer(CaptureService);
         let request = Request::builder()
             .uri("/bucket/object")

rustfs/src/server/rate_limit.rs

@@ -57,7 +57,6 @@ use crate::server::{
     MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
     RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix, is_admin_path, is_table_catalog_path,
 };
-use crate::storage_api::server::layer::request_context::RequestContext;
 use bytes::Bytes;
 use futures::future::{Either, Ready, ready};
 use http::{HeaderMap, HeaderValue, Request, Response, StatusCode};
@@ -409,10 +408,15 @@ fn u64_header(value: u64) -> HeaderValue {
 type BoxError = Box<dyn std::error::Error + Send + Sync>;
 type BoxBody = http_body_util::combinators::UnsyncBoxBody<Bytes, BoxError>;
 
-/// Build the S3-style `429` rejection. The server-owned request ID is echoed
-/// manually because the rejection short-circuits the inner response stack.
-fn s3_too_many_requests_response(request_id: Option<&str>, limit_rpm: u32, throttle: &ThrottleInfo) -> Response<BoxBody> {
+/// Build the S3-style `429` rejection. The request id (already generated by
+/// `SetRequestIdLayer`, which sits outside this layer) is echoed manually
+/// because the rejection short-circuits below `PropagateRequestIdLayer`.
+fn s3_too_many_requests_response(request_id: Option<&HeaderValue>, limit_rpm: u32, throttle: &ThrottleInfo) -> Response<BoxBody> {
+    // The header may be client-supplied (SetRequestIdLayer only fills it when
+    // absent), so gate the XML interpolation on a UUID-safe charset instead of
+    // reflecting arbitrary bytes into the body.
     let request_id_xml = request_id
+        .and_then(|value| value.to_str().ok())
         .filter(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
         .map(|id| format!("<RequestId>{id}</RequestId>"))
         .unwrap_or_default();
@@ -432,8 +436,8 @@ fn s3_too_many_requests_response(request_id: Option<&str>, limit_rpm: u32, throt
         .headers_mut()
         .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/xml"));
     apply_throttle_headers(response.headers_mut(), limit_rpm, throttle);
-    if let Some(id) = request_id.and_then(|id| HeaderValue::from_str(id).ok()) {
-        response.headers_mut().insert(X_REQUEST_ID, id);
+    if let Some(id) = request_id {
+        response.headers_mut().insert(X_REQUEST_ID, id.clone());
     }
     response
 }
@@ -555,9 +559,7 @@ fn rejected_response<F, E, ReqBody>(
         "Request rejected by API rate limit"
     );
     Either::Right(ready(Ok(s3_too_many_requests_response(
-        req.extensions()
-            .get::<RequestContext>()
-            .map(|context| context.request_id.as_str()),
+        req.headers().get(X_REQUEST_ID),
         limit_rpm,
         throttle,
     ))))
@@ -869,15 +871,7 @@ mod tests {
         }
 
         let mut req = request_from(ip(1), "/bucket/object");
-        req.headers_mut()
-            .insert(X_REQUEST_ID, HeaderValue::from_static("client-request-id"));
-        req.extensions_mut().insert(RequestContext {
-            request_id: "req-123".to_string(),
-            x_amz_request_id: "req-123".to_string(),
-            trace_id: None,
-            span_id: None,
-            start_time: Instant::now(),
-        });
+        req.headers_mut().insert(X_REQUEST_ID, HeaderValue::from_static("req-123"));
         let resp = service.call(req).await.expect("ok");
         assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
         assert_eq!(resp.headers().get(http::header::RETRY_AFTER).and_then(|v| v.to_str().ok()), Some("1"));
@@ -897,31 +891,20 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn hostile_request_id_does_not_override_the_429_request_id() {
+    async fn hostile_request_id_is_not_reflected_into_the_429_body() {
         let mut service = service_with_quota(60, 1);
         let _ = service.call(request_from(ip(3), "/bucket/object")).await.expect("ok");
 
         let mut req = request_from(ip(3), "/bucket/object");
         req.headers_mut()
             .insert(X_REQUEST_ID, HeaderValue::from_static("<Code>evil</Code>"));
-        req.extensions_mut().insert(RequestContext {
-            request_id: "server-request-id".to_string(),
-            x_amz_request_id: "server-request-id".to_string(),
-            trace_id: None,
-            span_id: None,
-            start_time: Instant::now(),
-        });
         let resp = service.call(req).await.expect("ok");
         assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
-        assert_eq!(
-            resp.headers().get(X_REQUEST_ID).and_then(|value| value.to_str().ok()),
-            Some("server-request-id")
-        );
 
         let body = resp.into_body().collect().await.expect("body").to_bytes();
         let body = String::from_utf8_lossy(&body);
         assert!(!body.contains("evil"), "client-controlled request id must not be reflected: {body}");
-        assert!(body.contains("<RequestId>server-request-id</RequestId>"), "body: {body}");
+        assert!(!body.contains("<RequestId>"), "malformed id must be omitted entirely: {body}");
     }
 
     #[tokio::test]

rustfs/src/storage/helper.rs

@@ -14,7 +14,7 @@
 
 use crate::server::{convert_ecstore_object_info, is_audit_module_enabled, is_notify_module_enabled};
 use crate::storage::access::{ReqInfo, request_context_from_req};
-use crate::storage::request_context::RequestContext;
+use crate::storage::request_context::{RequestContext, extract_request_id_from_headers};
 use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
 use hashbrown::HashMap;
 use http::StatusCode;
@@ -73,13 +73,6 @@ where
     }
 }
 
-/// Builds S3-compatible notification response elements with the canonical request ID.
-pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id: &str) -> HashMap<String, String> {
-    let mut resp_elements = extract_resp_elements(response);
-    resp_elements.insert(AMZ_REQUEST_ID.to_string(), request_id.to_string());
-    resp_elements
-}
-
 /// A unified helper structure for building and distributing audit logs and event notifications via RAII mode at the end of an S3 operation scope.
 pub enum OperationHelper {
     Disabled,
@@ -93,7 +86,7 @@ pub struct EnabledOperationHelper {
     api_builder: ApiDetailsBuilder,
     event_builder: Option<EventArgsBuilder>,
     start_time: std::time::Instant,
-    request_context: RequestContext,
+    request_context: Option<RequestContext>,
 }
 
 impl OperationHelper {
@@ -164,13 +157,16 @@ impl OperationHelper {
             api_builder = api_builder.object(&object_key);
         }
         // Audit builder
-        // Resolve the canonical request context once for both output chains.
+        // Resolve canonical request context and request_id in a single pass:
+        //   RequestContext.request_id > extract_request_id_from_headers() > generated fallback id
         let request_context = request_context_from_req(req);
         if request_context.is_none() {
             counter!("rustfs_log_chain_orphan_total", "component" => "operation_helper").increment(1);
         }
-        let request_context = request_context.unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
-        let request_id = request_context.request_id.clone();
+        let request_id = request_context
+            .as_ref()
+            .map(|ctx| ctx.request_id.clone())
+            .unwrap_or_else(|| extract_request_id_from_headers(&req.headers));
 
         let audit_builder = if audit_enabled {
             Some(
@@ -193,6 +189,12 @@ impl OperationHelper {
         };
 
         let mut req_params = extract_params_header(&req.headers);
+        // Inject x-amz-request-id from RequestContext into req_params for event correlation
+        if let Some(ref ctx) = request_context {
+            req_params
+                .entry(AMZ_REQUEST_ID.to_string())
+                .or_insert_with(|| ctx.x_amz_request_id.clone());
+        }
         if let Some(principal_id) = req_info
             .and_then(|info| info.cred.as_ref())
             .map(|cred| cred.access_key.clone())
@@ -226,7 +228,10 @@ impl OperationHelper {
             audit_builder,
             api_builder,
             event_builder,
-            start_time: request_context.start_time,
+            start_time: request_context
+                .as_ref()
+                .map(|ctx| ctx.start_time)
+                .unwrap_or_else(std::time::Instant::now),
             request_context,
         }))
     }
@@ -326,12 +331,14 @@ impl OperationHelper {
             }
 
             // Inject OpenTelemetry trace context into audit tags for distributed tracing correlation
-            if state.request_context.trace_id.is_some() || state.request_context.span_id.is_some() {
+            if let Some(ref ctx) = state.request_context
+                && (ctx.trace_id.is_some() || ctx.span_id.is_some())
+            {
                 let mut tags = HashMap::new();
-                if let Some(ref tid) = state.request_context.trace_id {
+                if let Some(ref tid) = ctx.trace_id {
                     tags.insert("traceId".to_string(), Value::String(tid.clone()));
                 }
-                if let Some(ref sid) = state.request_context.span_id {
+                if let Some(ref sid) = ctx.span_id {
                     tags.insert("spanId".to_string(), Value::String(sid.clone()));
                 }
                 final_builder = final_builder.tags(tags);
@@ -345,7 +352,7 @@ impl OperationHelper {
         if state.notify_enabled
             && let (Some(builder), Ok(res)) = (state.event_builder.take(), result)
         {
-            state.event_builder = Some(builder.resp_elements(build_event_resp_elements(res, &state.request_context.request_id)));
+            state.event_builder = Some(builder.resp_elements(extract_resp_elements(res)));
         }
 
         self
@@ -358,17 +365,6 @@ impl OperationHelper {
         }
         self
     }
-
-    /// Returns the operation's canonical context, reading the ingress extension
-    /// when the disabled fast path holds no request state.
-    pub(crate) fn request_context_or_from_request<T>(&self, req: &S3Request<T>) -> RequestContext {
-        match self {
-            Self::Enabled(state) => state.request_context.clone(),
-            Self::Disabled => {
-                request_context_from_req(req).unwrap_or_else(|| RequestContext::from_external_headers(&req.headers))
-            }
-        }
-    }
 }
 
 fn should_build_notification_event(notify_module_enabled: bool) -> bool {
@@ -385,7 +381,7 @@ impl Drop for OperationHelper {
         if state.audit_enabled
             && let Some(builder) = state.audit_builder.take()
         {
-            let ctx = Some(state.request_context.clone());
+            let ctx = state.request_context.clone();
             spawn_background_with_context(ctx, async move {
                 AuditLogger::log(builder.build()).await;
             });
@@ -399,7 +395,7 @@ impl Drop for OperationHelper {
             let event_args = builder.build();
             // Avoid generating notifications for copy requests
             if !event_args.is_replication_request() {
-                let ctx = Some(state.request_context.clone());
+                let ctx = state.request_context.clone();
                 spawn_background_with_context(ctx, async move {
                     runtime_sources::current_notify_interface().notify(event_args).await;
                 });
@@ -420,9 +416,8 @@ mod tests {
     use rustfs_credentials::Credentials;
     use rustfs_s3_ops::S3Operation;
     use rustfs_s3_types::EventName;
-    use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
-    use s3s::dto::{DeleteObjectTaggingInput, DeleteObjectTaggingOutput};
-    use s3s::{S3Request, S3Response};
+    use s3s::S3Request;
+    use s3s::dto::DeleteObjectTaggingInput;
     use std::sync::{Arc, Mutex};
     use temp_env::with_vars;
 
@@ -559,14 +554,11 @@ mod tests {
                 let mut req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
                 req.headers.insert("host", HeaderValue::from_static("example.com"));
                 req.headers.insert("user-agent", HeaderValue::from_static("rustfs-test"));
-                req.headers
-                    .insert(REQUEST_ID_HEADER, HeaderValue::from_static("ingress-canonical-uuid"));
-                req.headers
-                    .insert("x-amz-request-id", HeaderValue::from_static("client-supplied-request-id"));
 
+                // Insert RequestContext (set by ingress layer) with a specific request_id
                 req.extensions.insert(RequestContext {
                     request_id: "ingress-canonical-uuid".to_string(),
-                    x_amz_request_id: "client-supplied-request-id".to_string(),
+                    x_amz_request_id: "ingress-canonical-uuid".to_string(),
                     trace_id: None,
                     span_id: None,
                     start_time: std::time::Instant::now(),
@@ -578,32 +570,20 @@ mod tests {
                     ..Default::default()
                 });
 
-                let result = Ok(S3Response::new(DeleteObjectTaggingOutput::default()));
-                let mut helper =
-                    OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).complete(&result);
+                let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
 
                 // Verify the helper stored the RequestContext
-                let OperationHelper::Enabled(state) = &mut helper else {
+                let OperationHelper::Enabled(state) = &helper else {
                     panic!("helper should be enabled when notify/audit switches are on");
                 };
-                assert_eq!(state.request_context.request_id, "ingress-canonical-uuid");
-                let audit_entry = state.audit_builder.take().expect("audit builder should exist").build();
-                assert_eq!(audit_entry.request_id.as_deref(), Some("ingress-canonical-uuid"));
-                let event_args = state.event_builder.clone().expect("event builder should exist").build();
-                assert_eq!(
-                    event_args.req_params.get(AMZ_REQUEST_ID).map(String::as_str),
-                    Some("client-supplied-request-id")
-                );
-                assert_eq!(
-                    event_args.resp_elements.get(AMZ_REQUEST_ID).map(String::as_str),
-                    Some("ingress-canonical-uuid")
-                );
+                assert!(state.request_context.is_some());
+                assert_eq!(state.request_context.as_ref().unwrap().request_id, "ingress-canonical-uuid");
             },
         );
     }
 
     #[test]
-    fn operation_helper_reuses_generated_context_when_headers_absent() {
+    fn operation_helper_no_request_context_when_absent() {
         with_vars(
             [
                 (rustfs_config::ENV_NOTIFY_ENABLE, Some("true")),
@@ -621,6 +601,8 @@ mod tests {
                 let mut req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
                 req.headers.insert("host", HeaderValue::from_static("example.com"));
                 req.headers.insert("user-agent", HeaderValue::from_static("rustfs-test"));
+                req.headers
+                    .insert("x-amz-request-id", HeaderValue::from_static("amz-header-uuid"));
 
                 // No RequestContext inserted
                 req.extensions.insert(ReqInfo {
@@ -630,21 +612,12 @@ mod tests {
                 });
 
                 let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
-                let request_context = helper.request_context_or_from_request(&req);
-                let request_id = request_context.request_id;
-                assert!(uuid::Uuid::parse_str(&request_id).is_ok());
-                let result = Ok(S3Response::new(DeleteObjectTaggingOutput::default()));
-                let helper = helper.complete(&result);
+
+                // Verify the helper has no RequestContext
                 let OperationHelper::Enabled(state) = &helper else {
                     panic!("helper should be enabled when notify/audit switches are on");
                 };
-                assert_eq!(state.request_context.request_id, request_id);
-                let event_args = state.event_builder.clone().expect("event builder should exist").build();
-                assert!(!event_args.req_params.contains_key(AMZ_REQUEST_ID));
-                assert_eq!(
-                    event_args.resp_elements.get(AMZ_REQUEST_ID).map(String::as_str),
-                    Some(request_id.as_str())
-                );
+                assert!(state.request_context.is_none());
             },
         );
     }
@@ -665,20 +638,10 @@ mod tests {
                     .key("test-key".to_string())
                     .build()
                     .unwrap();
-                let mut req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
-                req.headers
-                    .insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
-                req.extensions.insert(RequestContext {
-                    request_id: "server-request-id".to_string(),
-                    x_amz_request_id: "server-request-id".to_string(),
-                    trace_id: None,
-                    span_id: None,
-                    start_time: std::time::Instant::now(),
-                });
+                let req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
                 let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
 
-                assert!(matches!(&helper, OperationHelper::Disabled));
-                assert_eq!(helper.request_context_or_from_request(&req).request_id, "server-request-id");
+                assert!(matches!(helper, OperationHelper::Disabled));
             },
         );
     }

rustfs/src/storage/request_context.rs

@@ -17,10 +17,11 @@
 //! # Architecture
 //!
 //! ```text
-//! External S3 HTTP ingress
-//!   → generates a server-owned request ID without mutating signed headers
-//!   → ExternalRequestContextLayer creates RequestContext
+//! HTTP Ingress (SetRequestIdLayer)
+//!   → generates x-request-id UUID
+//!   → RequestContextLayer creates RequestContext
 //!     → stores in request.extensions()
+//!     → stores the S3-compatible request-id alias without changing signed headers
 //! Auth (FS::check)
 //!   → copies RequestContext into ReqInfo.request_context
 //! Storage (FS methods)
@@ -39,11 +40,10 @@
 //! # Frozen Rules (T00 Guardrails)
 //!
 //! ## request-id contract
-//! - External S3 response headers: `x-request-id` and `x-amz-request-id`
-//! - Non-S3 response header: propagated `x-request-id`
+//! - Canonical wire header: `x-request-id` (set by `SetRequestIdLayer`)
 //! - Compatibility wire header: `x-amz-request-id`
 //! - Canonical internal field: `RequestContext.request_id`
-//! - Client-provided request ID headers are never canonical on external S3 requests
+//! - S3 compatibility internal alias field: `RequestContext.x_amz_request_id`
 //! - Internal modules MUST NOT generate a second request id under the field name `request_id`
 //!   except for orphan/non-ingress fallback paths where no canonical request-id exists.
 //! - Internal identifiers for sub-operations should use `operation_id` or `subtask_id`
@@ -72,14 +72,10 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;
 /// Created exactly once at HTTP ingress. Cloned by value; never mutated after creation.
 #[derive(Clone, Debug)]
 pub struct RequestContext {
-    /// Canonical request ID: server-owned for external S3 requests and
-    /// propagated for non-S3 and trusted internal requests.
+    /// Canonical request ID (from `x-request-id` header, set by `SetRequestIdLayer`).
     pub request_id: String,
-    /// Compatibility-only alias that preserves an incoming
-    /// `x-amz-request-id`, or mirrors [`Self::request_id`] when absent.
-    ///
-    /// Internal correlation must use [`Self::request_id`]. This field remains
-    /// for compatibility with existing in-crate consumers.
+    /// S3-compatible request ID alias (preserves upstream `x-amz-request-id` if present,
+    /// otherwise equals `request_id`).
     pub x_amz_request_id: String,
     /// OpenTelemetry trace ID (if present from upstream propagation).
     pub trace_id: Option<String>,
@@ -90,54 +86,6 @@ pub struct RequestContext {
 }
 
 impl RequestContext {
-    /// Create a context for a trusted internal request that may propagate its
-    /// canonical ID through headers.
-    pub(crate) fn from_headers(headers: &HeaderMap) -> Self {
-        Self::new(
-            extract_request_id_from_headers(headers),
-            headers,
-            extract_trace_context_ids_from_headers(headers),
-        )
-    }
-
-    /// Create a context from propagated request headers without copying trace
-    /// state into the request context.
-    pub(crate) fn from_headers_without_trace_context(headers: &HeaderMap) -> Self {
-        let request_id = extract_request_id_from_headers(headers);
-        Self {
-            x_amz_request_id: request_id.clone(),
-            request_id,
-            trace_id: None,
-            span_id: None,
-            start_time: Instant::now(),
-        }
-    }
-
-    /// Create an external request context with a server-owned ID while keeping
-    /// client headers unchanged for signature verification.
-    pub(crate) fn from_external_headers(headers: &HeaderMap) -> Self {
-        Self::new(uuid::Uuid::new_v4().to_string(), headers, extract_trace_context_ids_from_headers(headers))
-    }
-
-    fn new(request_id: String, headers: &HeaderMap, trace_context: Option<(String, String)>) -> Self {
-        let x_amz_request_id = headers
-            .get(AMZ_REQUEST_ID)
-            .and_then(|value| value.to_str().ok())
-            .map(String::from)
-            .unwrap_or_else(|| request_id.clone());
-        let (trace_id, span_id) = trace_context
-            .map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))
-            .unwrap_or((None, None));
-
-        Self {
-            request_id,
-            x_amz_request_id,
-            trace_id,
-            span_id,
-            start_time: Instant::now(),
-        }
-    }
-
     /// Create a fallback `RequestContext` for paths that bypass HTTP ingress.
     /// Generates a canonical internal `request_id` in `trace-{trace_id}` or `req-{uuid}` format.
     pub fn fallback() -> Self {
@@ -258,7 +206,7 @@ where
 #[allow(unused_imports)]
 mod tests {
     use super::{RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers};
-    use http::{HeaderMap, HeaderValue};
+    use http::HeaderMap;
     use opentelemetry::global;
     use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
     use opentelemetry_sdk::propagation::TraceContextPropagator;
@@ -304,64 +252,6 @@ mod tests {
         assert!(ctx.span_id.is_none());
     }
 
-    #[test]
-    fn test_request_context_from_headers_prioritizes_canonical_request_id() {
-        let mut headers = HeaderMap::new();
-        headers.insert("x-request-id", HeaderValue::from_static("canonical-request-id"));
-        headers.insert("x-amz-request-id", HeaderValue::from_static("client-request-id"));
-
-        let ctx = RequestContext::from_headers(&headers);
-
-        assert_eq!(ctx.request_id, "canonical-request-id");
-        assert_eq!(ctx.x_amz_request_id, "client-request-id");
-    }
-
-    #[test]
-    fn test_request_context_from_headers_preserves_empty_amz_alias() {
-        let mut headers = HeaderMap::new();
-        headers.insert("x-request-id", HeaderValue::from_static("canonical-request-id"));
-        headers.insert("x-amz-request-id", HeaderValue::from_static(""));
-
-        let ctx = RequestContext::from_headers(&headers);
-
-        assert_eq!(ctx.request_id, "canonical-request-id");
-        assert_eq!(ctx.x_amz_request_id, "");
-    }
-
-    #[test]
-    fn test_propagated_request_context_mirrors_canonical_request_id() {
-        let mut headers = HeaderMap::new();
-        headers.insert("x-request-id", HeaderValue::from_static("canonical-request-id"));
-        headers.insert("x-amz-request-id", HeaderValue::from_static("untrusted-amz-request-id"));
-
-        let ctx = RequestContext::from_headers_without_trace_context(&headers);
-
-        assert_eq!(ctx.request_id, "canonical-request-id");
-        assert_eq!(ctx.x_amz_request_id, "canonical-request-id");
-        assert!(ctx.trace_id.is_none());
-        assert!(ctx.span_id.is_none());
-    }
-
-    #[test]
-    fn test_external_request_context_owns_id_and_preserves_trace_context() {
-        global::set_text_map_propagator(TraceContextPropagator::new());
-        let mut headers = HeaderMap::new();
-        headers.insert("x-request-id", HeaderValue::from_static("client-request-id"));
-        headers.insert("x-amz-request-id", HeaderValue::from_static("client-amz-request-id"));
-        headers.insert(
-            "traceparent",
-            HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
-        );
-
-        let ctx = RequestContext::from_external_headers(&headers);
-
-        assert_ne!(ctx.request_id, "client-request-id");
-        assert!(uuid::Uuid::parse_str(&ctx.request_id).is_ok());
-        assert_eq!(ctx.x_amz_request_id, "client-amz-request-id");
-        assert_eq!(ctx.trace_id.as_deref(), Some("4bf92f3577b34da6a3ce929d0e0e4736"));
-        assert_eq!(ctx.span_id.as_deref(), Some("00f067aa0ba902b7"));
-    }
-
     #[test]
     fn test_request_context_fallback_uses_trace_prefix_when_span_context_valid() {
         let trace_id = "70f5f77e2f0a4f24be343b59f8b66f8f";

rustfs/src/storage/rpc/node_service.rs

@@ -52,7 +52,7 @@ use std::{
     collections::HashMap,
     io::Cursor,
     pin::Pin,
-    sync::{Arc, OnceLock},
+    sync::{Arc, LazyLock, OnceLock},
 };
 use time::OffsetDateTime;
 use tokio::spawn;
@@ -128,6 +128,7 @@ fn remove_heal_control_replay(
 }
 
 static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
+static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
 
 fn admit_heal_control_replay(
     replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -465,6 +466,19 @@ impl HealControlRpcService {
 pub(crate) async fn initialize_heal_topology_fingerprint(
     cache: Arc<tokio::sync::OnceCell<String>>,
     endpoint_pools: EndpointServerPools,
+) -> Result<(), String> {
+    initialize_heal_topology_fingerprint_with_probe(
+        cache,
+        endpoint_pools,
+        crate::storage::storage_api::start_remote_version_state_fleet_probe,
+    )
+    .await
+}
+
+async fn initialize_heal_topology_fingerprint_with_probe(
+    cache: Arc<tokio::sync::OnceCell<String>>,
+    endpoint_pools: EndpointServerPools,
+    start_probe: impl FnOnce(String),
 ) -> Result<(), String> {
     if cache.get().is_some() {
         return Ok(());
@@ -472,7 +486,8 @@ pub(crate) async fn initialize_heal_topology_fingerprint(
     let fingerprint = tokio::task::spawn_blocking(move || heal::heal_topology_fingerprint(&endpoint_pools))
         .await
         .map_err(|_| "heal control topology calculation task failed".to_string())??;
-    let _ = cache.set(fingerprint);
+    let _ = cache.set(fingerprint.clone());
+    start_probe(fingerprint);
     Ok(())
 }
 
@@ -806,6 +821,35 @@ impl heal_control_service_server::HealControlService for HealControlRpcService {
                 response_proof: Bytes::new(),
             }));
         }
+        if rustfs_protos::is_remote_version_state_capability_probe(&request.get_ref().command) {
+            let topology_member = self
+                .endpoint_pools()
+                .await
+                .ok_or_else(|| Status::failed_precondition("heal control topology is not initialized"))?
+                .peers()
+                .1;
+            if topology_member.is_empty() {
+                return Err(Status::failed_precondition("local topology member identity is unavailable"));
+            }
+            let result =
+                rustfs_protos::encode_remote_version_state_capability(&topology_member, NODE_CAPABILITY_SERVER_EPOCH.as_bytes())
+                    .map_err(|_| Status::internal("remote version state capability length cannot be represented"))?;
+            let canonical_response = rustfs_protos::canonical_heal_control_response_body(
+                request.get_ref().version,
+                &request.get_ref().topology_fingerprint,
+                &request.get_ref().command,
+                &result,
+            )
+            .map_err(|_| Status::internal("heal control response length cannot be represented"))?;
+            let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
+                .map_err(|_| Status::internal("heal control response proof is unavailable"))?;
+            return Ok(Response::new(HealControlResponse {
+                success: true,
+                result: result.into(),
+                error_info: None,
+                response_proof: response_proof.into(),
+            }));
+        }
         let endpoints = self
             .endpoint_pools()
             .await
@@ -2090,10 +2134,10 @@ mod tests {
         PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
         SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
         STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
-        execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
-        make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
-        make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
-        scanner_activity_response, stop_rebalance_response,
+        execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint,
+        initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server,
+        make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
+        previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
     };
     use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
     use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -2147,6 +2191,7 @@ mod tests {
     use tokio::time::Duration;
     use tokio_stream::wrappers::TcpListenerStream;
     use tonic::{Request, Response, Status};
+    use uuid::Uuid;
 
     const DISK_MUTATION_RPC_METHODS: [&str; 18] = [
         "renamedata",
@@ -3132,6 +3177,60 @@ mod tests {
         assert_eq!(non_coordinator.code(), tonic::Code::FailedPrecondition);
     }
 
+    #[tokio::test]
+    async fn remote_version_state_probe_authenticates_topology_challenge_and_process_epoch() {
+        let _ = rustfs_credentials::set_global_rpc_secret("remote-version-state-node-service-test-secret".to_string());
+        let endpoints = heal_control_test_endpoints_with_coordinator("node-d", true);
+        let fingerprint = heal_topology_fingerprint(&endpoints).expect("test topology should hash");
+        let (service, source) = super::make_heal_control_server_for_source();
+        *source.write().await = Some(endpoints);
+        let probe_command = rustfs_protos::remote_version_state_capability_probe(&[7; 16]);
+        let mut request = Request::new(HealControlRequest {
+            version: rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
+            topology_fingerprint: fingerprint.clone(),
+            command: Bytes::from(probe_command.clone()),
+        });
+        let body = rustfs_protos::canonical_heal_control_request_body(
+            request.get_ref().version,
+            &request.get_ref().topology_fingerprint,
+            &request.get_ref().command,
+        )
+        .expect("probe should encode");
+        set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
+        mark_v2_authenticated(&mut request);
+        let response = service
+            .heal_control(request)
+            .await
+            .expect("matching topology should be acknowledged")
+            .into_inner();
+
+        let (topology_member, process_epoch) =
+            rustfs_protos::decode_remote_version_state_capability(&response.result).expect("capability response should decode");
+        assert_eq!(topology_member, "node-a:9000");
+        let server_epoch = Uuid::from_slice(process_epoch).expect("server epoch should be a UUID");
+        assert!(!server_epoch.is_nil());
+        let canonical_response = rustfs_protos::canonical_heal_control_response_body(
+            rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
+            &fingerprint,
+            &probe_command,
+            &response.result,
+        )
+        .expect("response should encode");
+        crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
+            .expect("outer proof should bind the response to the request");
+
+        let different_probe = rustfs_protos::remote_version_state_capability_probe(&[8; 16]);
+        let different_response = rustfs_protos::canonical_heal_control_response_body(
+            rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
+            &fingerprint,
+            &different_probe,
+            &response.result,
+        )
+        .expect("different response should encode");
+        crate::storage::storage_api::verify_tonic_rpc_response_proof(&different_response, &response.response_proof)
+            .expect_err("proof from one challenge must not be reusable");
+    }
+
     #[tokio::test]
     async fn heal_control_coordinator_rejects_expired_and_non_admin_starts() {
         let _ = rustfs_credentials::set_global_rpc_secret("heal-control-node-service-test-secret".to_string());
@@ -3194,10 +3293,15 @@ mod tests {
         let topology = heal_control_test_endpoints("node-d");
         let expected = heal_topology_fingerprint(&topology).expect("test topology should hash");
         let cache = Arc::new(tokio::sync::OnceCell::new());
-        initialize_heal_topology_fingerprint(Arc::clone(&cache), topology)
-            .await
-            .expect("valid topology should initialize");
+        let started_probe = Arc::new(std::sync::Mutex::new(None));
+        let started_probe_capture = Arc::clone(&started_probe);
+        initialize_heal_topology_fingerprint_with_probe(Arc::clone(&cache), topology, move |fingerprint| {
+            *started_probe_capture.lock().expect("probe capture should not poison") = Some(fingerprint);
+        })
+        .await
+        .expect("valid topology should initialize");
         assert_eq!(cache.get(), Some(&expected));
+        assert_eq!(started_probe.lock().expect("probe capture should not poison").as_ref(), Some(&expected));
 
         let mut invalid = heal_control_test_endpoints("node-d");
         invalid.as_mut()[0].endpoints.as_mut()[0].pool_idx = -1;

rustfs/src/storage/storage_api.rs

@@ -174,7 +174,7 @@ pub(crate) mod head_prefix_consumer {
 }
 
 pub(crate) mod helper_consumer {
-    pub(crate) use super::super::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context};
+    pub(crate) use super::super::helper::{OperationHelper, spawn_background_with_context};
 
     pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
 }
@@ -203,7 +203,9 @@ pub(crate) mod options_consumer {
 }
 
 pub(crate) mod request_context_consumer {
-    pub(crate) use super::super::request_context::{RequestContext, extract_request_id_from_headers, spawn_traced};
+    pub(crate) use super::super::request_context::{
+        RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
+    };
 }
 
 pub(crate) mod rpc_consumer {
@@ -470,7 +472,7 @@ pub(crate) mod ecstore_metrics {
 #[allow(unused_imports)]
 pub(crate) mod ecstore_notification {
     pub(crate) use rustfs_ecstore::api::notification::{
-        NotificationSys, get_global_notification_sys, new_global_notification_sys,
+        NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe,
     };
 }
 
@@ -834,7 +836,7 @@ pub(crate) async fn reconcile_bucket_resync_target_intents(buckets: &[String]) -
     };
 
     for bucket in buckets {
-        let _transaction_guard = ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await?;
+        let _transaction_guard = ecstore_bucket::metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await?;
         let status = pool.get_bucket_resync_status(bucket).await?;
         if status.targets_map.is_empty() {
             continue;
@@ -961,6 +963,10 @@ pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPo
     ecstore_notification::new_global_notification_sys(endpoint_pools).await
 }
 
+pub(crate) fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
+    ecstore_notification::start_remote_version_state_fleet_probe(topology_fingerprint);
+}
+
 pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
     ecstore_config::com::read_config(api, file).await
 }
@@ -1458,8 +1464,8 @@ pub(crate) async fn update_bucket_metadata_config(
     Ok(updated_at)
 }
 
-pub(crate) async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
-    ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await
+pub(crate) async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
+    ecstore_bucket::metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await
 }
 
 pub(crate) async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec<u8>) -> Result<time::OffsetDateTime> {

rustfs/src/storage_api.rs

@@ -124,7 +124,9 @@ pub(crate) mod server {
         }
 
         pub(crate) mod request_context {
-            pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
+            pub(crate) use crate::storage::storage_api::request_context_consumer::{
+                RequestContext, extract_request_id_from_headers,
+            };
         }
 
         pub(crate) mod rpc {
@@ -147,7 +149,7 @@ pub(crate) mod server {
 
         pub(crate) mod request_context {
             pub(crate) use crate::storage::storage_api::request_context_consumer::{
-                RequestContext, extract_request_id_from_headers, spawn_traced,
+                RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
             };
         }
     }