fix(cache): harden object data cache coordination (#5004)

* fix(cache): enforce projected entry capacity Refs: rustfs/backlog#1335 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): fence identity budget eviction by generation Refs rustfs/backlog#1334. Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): fence clear against concurrent fills Refs rustfs/backlog#1333 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): linearize memory reservation claims Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): retain allocation memory claims Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): publish memory snapshots by epoch Co-Authored-By: heihutu <heihutu@gmail.com> * fix(cache): coordinate cold object fills Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): fence metadata cache transition races Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-19 19:16:17 +00:00 · 2026-07-18 22:37:10 +08:00
parent 4faea7fcbc
commit 15f4e75870
39 changed files with 9930 additions and 538 deletions
@@ -160,7 +160,7 @@ use rustfs_utils::{
 };
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
 use sha2::{Digest, Sha256};
-use std::hash::Hash;
+use std::hash::{BuildHasher, Hash, Hasher};
 use std::mem::{self};
 use std::pin::Pin;
 use std::sync::OnceLock;
@@ -547,6 +547,7 @@ const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
 const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
 const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
 const ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: &str = "RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES";
+const GET_OBJECT_METADATA_CACHE_FENCE_SHARDS: u16 = 4096;

 // --- Codec Streaming Configuration ---

@@ -623,12 +624,284 @@ mod core;
 mod ctx;
 mod metadata;
 mod ops;
+pub(crate) use ops::object::body_cache_plaintext_len;
 mod read;
 mod replication;
 pub(crate) mod shard_source;

 pub use ops::heal_walk::HealWalkVersion;

+pub(crate) struct PreparedGetObjectMetadata {
+    fi: FileInfo,
+    files: Vec<FileInfo>,
+    disks: Vec<Option<DiskStore>>,
+    object_info: Option<ObjectInfo>,
+}
+
+impl PreparedGetObjectMetadata {
+    pub(crate) fn object_info(&self) -> &ObjectInfo {
+        self.object_info
+            .as_ref()
+            .expect("prepared GET metadata must retain its ObjectInfo until consumed")
+    }
+
+    pub(crate) fn take_object_info(&mut self) -> ObjectInfo {
+        self.object_info
+            .take()
+            .expect("prepared GET metadata ObjectInfo must be consumed exactly once")
+    }
+}
+
+tokio::task_local! {
+    static PREPARED_GET_OBJECT_METADATA: std::cell::RefCell<Option<PreparedGetObjectMetadata>>;
+}
+
+#[cfg(test)]
+tokio::task_local! {
+    static GET_OBJECT_INFO_CONVERSIONS: Arc<AtomicU64>;
+}
+
+fn build_get_object_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
+    #[cfg(test)]
+    let _ = GET_OBJECT_INFO_CONVERSIONS.try_with(|conversions| {
+        conversions.fetch_add(1, Ordering::Relaxed);
+    });
+    ObjectInfo::from_file_info(fi, bucket, object, versioned)
+}
+
+fn take_prepared_get_object_metadata() -> Option<PreparedGetObjectMetadata> {
+    PREPARED_GET_OBJECT_METADATA
+        .try_with(|prepared| prepared.borrow_mut().take())
+        .ok()
+        .flatten()
+}
+
+async fn with_prepared_get_object_metadata<F>(metadata: PreparedGetObjectMetadata, future: F) -> F::Output
+where
+    F: std::future::Future,
+{
+    PREPARED_GET_OBJECT_METADATA
+        .scope(std::cell::RefCell::new(Some(metadata)), future)
+        .await
+}
+
+#[cfg(test)]
+mod prepared_get_object_metadata_tests {
+    use super::*;
+    use crate::ecstore_validation_blackbox::make_local_set_disks;
+    use crate::object_api::{BLOCK_SIZE_V2, PutObjReader};
+    use crate::set_disk::core::io_primitives::disk_call_counters;
+    use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
+    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
+    use http::HeaderMap;
+    use tokio::io::AsyncReadExt;
+
+    #[tokio::test]
+    async fn prepared_metadata_is_consumed_exactly_once() {
+        let metadata = PreparedGetObjectMetadata {
+            fi: FileInfo::default(),
+            files: Vec::new(),
+            disks: Vec::new(),
+            object_info: None,
+        };
+
+        with_prepared_get_object_metadata(metadata, async {
+            assert!(take_prepared_get_object_metadata().is_some());
+            assert!(take_prepared_get_object_metadata().is_none());
+        })
+        .await;
+        assert!(take_prepared_get_object_metadata().is_none());
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn prepared_reader_reuses_metadata_fanout_exactly_once() {
+        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
+        let bucket = "prepared-metadata-fanout";
+        let object = "prepared-metadata-fanout-object.bin";
+        let payload = b"prepared-metadata-fanout-payload-".repeat(40_000);
+        let opts = ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        };
+
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+        let mut put_reader = PutObjReader::from_vec(payload.clone());
+        set_disks
+            .put_object(bucket, object, &mut put_reader, &opts)
+            .await
+            .expect("object should be written");
+
+        let calls = disk_call_counters::observe(object);
+        let conversions = Arc::new(AtomicU64::new(0));
+        let restored = GET_OBJECT_INFO_CONVERSIONS
+            .scope(Arc::clone(&conversions), async {
+                let metadata = set_disks
+                    .prepare_get_object_metadata(bucket, object, &opts)
+                    .await
+                    .expect("prepared metadata should resolve");
+                assert_eq!(
+                    calls.total(disk_call_counters::KIND_READ_VERSION),
+                    4,
+                    "preparation should fan out to each online disk exactly once"
+                );
+
+                let mut reader = set_disks
+                    .get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
+                    .await
+                    .expect("prepared body reader should open");
+                let mut restored = Vec::new();
+                reader
+                    .stream
+                    .read_to_end(&mut restored)
+                    .await
+                    .expect("prepared body should stream");
+                restored
+            })
+            .await;
+
+        assert_eq!(restored, payload);
+        assert_eq!(
+            conversions.load(Ordering::Relaxed),
+            1,
+            "prepared reader must consume the ObjectInfo built during metadata preparation"
+        );
+        assert_eq!(
+            calls.total(disk_call_counters::KIND_READ_VERSION),
+            4,
+            "reader construction must consume prepared metadata instead of repeating the fanout"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn prepared_reader_rebuilds_object_info_when_precomputed_value_is_absent() {
+        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
+        let bucket = "prepared-object-info-fallback";
+        let object = "prepared-object-info-fallback.bin";
+        let payload = b"prepared-object-info-fallback-payload".repeat(4_000);
+        let opts = ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        };
+
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+        let mut put_reader = PutObjReader::from_vec(payload.clone());
+        set_disks
+            .put_object(bucket, object, &mut put_reader, &opts)
+            .await
+            .expect("object should be written");
+
+        let conversions = Arc::new(AtomicU64::new(0));
+        let restored = GET_OBJECT_INFO_CONVERSIONS
+            .scope(Arc::clone(&conversions), async {
+                let mut metadata = set_disks
+                    .prepare_get_object_metadata(bucket, object, &opts)
+                    .await
+                    .expect("prepared metadata should resolve");
+                metadata.object_info = None;
+                let mut reader = set_disks
+                    .get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
+                    .await
+                    .expect("reader should rebuild missing prepared ObjectInfo");
+                let mut restored = Vec::new();
+                reader
+                    .stream
+                    .read_to_end(&mut restored)
+                    .await
+                    .expect("fallback reader should stream");
+                restored
+            })
+            .await;
+
+        assert_eq!(restored, payload);
+        assert_eq!(
+            conversions.load(Ordering::Relaxed),
+            2,
+            "missing precomputed ObjectInfo must trigger exactly one structural fallback rebuild"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial(body_cache_hook)]
+    async fn prepared_reader_restores_full_body_with_one_offline_shard() {
+        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
+        let bucket = "prepared-reader-offline-shard";
+        let object = "prepared-reader-offline-shard-object.bin";
+        let payload = (0..(BLOCK_SIZE_V2 + 321))
+            .map(|idx| ((idx * 19) % 251) as u8)
+            .collect::<Vec<_>>();
+        let opts = ObjectOptions {
+            no_lock: true,
+            ..Default::default()
+        };
+
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+        let mut put_reader = PutObjReader::from_vec(payload.clone());
+        set_disks
+            .put_object(bucket, object, &mut put_reader, &opts)
+            .await
+            .expect("object should be written");
+        set_disks.disks.write().await[0] = None;
+
+        let metadata = set_disks
+            .prepare_get_object_metadata(bucket, object, &opts)
+            .await
+            .expect("prepared metadata should tolerate one offline shard");
+        let mut reader = set_disks
+            .get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
+            .await
+            .expect("prepared body reader should open with one offline shard");
+        let mut restored = Vec::new();
+        reader
+            .stream
+            .read_to_end(&mut restored)
+            .await
+            .expect("degraded prepared body should stream");
+
+        assert_eq!(restored, payload);
+    }
+}
+
+impl SetDisks {
+    pub(crate) async fn prepare_get_object_metadata(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+    ) -> Result<PreparedGetObjectMetadata> {
+        let (fi, files, disks) = self.get_object_fileinfo(bucket, object, opts, true, true).await?;
+        let object_info = build_get_object_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
+        Ok(PreparedGetObjectMetadata {
+            fi,
+            files,
+            disks,
+            object_info: Some(object_info),
+        })
+    }
+
+    pub(crate) async fn get_object_reader_with_prepared_metadata(
+        &self,
+        bucket: &str,
+        object: &str,
+        range: Option<HTTPRangeSpec>,
+        headers: HeaderMap,
+        opts: &ObjectOptions,
+        metadata: PreparedGetObjectMetadata,
+    ) -> Result<GetObjectReader> {
+        with_prepared_get_object_metadata(metadata, self.get_object_reader(bucket, object, range, headers, opts)).await
+    }
+}
+
 /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
 /// Defaults to 30 seconds if not set or invalid
 /// Lock acquisition timeout. Cached: this is consulted on every object
@@ -1742,6 +2015,8 @@ pub struct SetDisks {
    pub format: FormatV3,
    disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
    get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
+    get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
+    get_object_metadata_cache_generations: Arc<[AtomicU64]>,
    pub lockers: Vec<Arc<dyn LockClient>>,
    local_lock_manager: Arc<rustfs_lock::GlobalLockManager>,
    /// Per-instance runtime context (Phase 5, backlog#939).
@@ -1761,21 +2036,109 @@ pub struct SetDisks {
    capacity_dirty_generation: Arc<AtomicU64>,
 }

-#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 struct GetObjectMetadataCacheKey {
+    bucket: Arc<str>,
+    object: Arc<str>,
+    generation: u64,
+    hash: u64,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct GetObjectMetadataCacheGeneration {
+    index: usize,
+    value: u64,
+    hash: u64,
+}
+
+#[cfg(test)]
+struct MetadataCacheInvalidationProbeState {
    bucket: String,
    object: String,
+    count: AtomicU64,
+}
+
+#[cfg(test)]
+struct MetadataCacheInvalidationProbe {
+    state: Arc<MetadataCacheInvalidationProbeState>,
+}
+
+#[cfg(test)]
+static METADATA_CACHE_INVALIDATION_PROBE: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<MetadataCacheInvalidationProbeState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl MetadataCacheInvalidationProbe {
+    fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(MetadataCacheInvalidationProbeState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            count: AtomicU64::new(0),
+        });
+        let mut slot = METADATA_CACHE_INVALIDATION_PROBE
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("metadata cache invalidation probe mutex should not poison");
+        assert!(
+            slot.is_none(),
+            "metadata cache invalidation probe must be installed by one test at a time"
+        );
+        *slot = Some(Arc::clone(&state));
+        drop(slot);
+        Self { state }
+    }
+
+    fn count(&self) -> u64 {
+        self.state.count.load(Ordering::Acquire)
+    }
+}
+
+#[cfg(test)]
+impl Drop for MetadataCacheInvalidationProbe {
+    fn drop(&mut self) {
+        let mut slot = METADATA_CACHE_INVALIDATION_PROBE
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("metadata cache invalidation probe mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+fn record_metadata_cache_invalidation(bucket: &str, object: &str) {
+    let probe = METADATA_CACHE_INVALIDATION_PROBE
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("metadata cache invalidation probe mutex should not poison")
+        .as_ref()
+        .filter(|probe| probe.bucket == bucket && probe.object == object)
+        .cloned();
+    if let Some(probe) = probe {
+        probe.count.fetch_add(1, Ordering::AcqRel);
+    }
 }

 impl GetObjectMetadataCacheKey {
-    fn new(bucket: &str, object: &str) -> Self {
+    fn new(bucket: &str, object: &str, generation: GetObjectMetadataCacheGeneration) -> Self {
        Self {
-            bucket: bucket.to_string(),
-            object: object.to_string(),
+            bucket: Arc::from(bucket),
+            object: Arc::from(object),
+            generation: generation.value,
+            hash: generation.hash,
        }
    }
 }

+impl Hash for GetObjectMetadataCacheKey {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.hash.hash(state);
+        self.generation.hash(state);
+    }
+}
+
 #[derive(Clone, Debug)]
 struct GetObjectMetadataCacheEntry {
    #[allow(dead_code)] // Kept for debugging; moka handles TTL internally
@@ -1803,10 +2166,50 @@ impl DiskHealthEntry {
 }

 impl SetDisks {
+    fn get_object_metadata_cache_hash(&self, bucket: &str, object: &str) -> u64 {
+        let mut hasher = self.get_object_metadata_cache_hash_builder.build_hasher();
+        bucket.hash(&mut hasher);
+        object.hash(&mut hasher);
+        hasher.finish()
+    }
+
+    fn get_object_metadata_cache_generation(&self, bucket: &str, object: &str) -> Option<GetObjectMetadataCacheGeneration> {
+        let hash = self.get_object_metadata_cache_hash(bucket, object);
+        let hash_bytes = hash.to_le_bytes();
+        let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
+        let value = self.get_object_metadata_cache_generations[index].load(Ordering::Acquire);
+        (value != u64::MAX).then_some(GetObjectMetadataCacheGeneration { index, value, hash })
+    }
+
+    fn is_get_object_metadata_cache_generation_current(&self, generation: GetObjectMetadataCacheGeneration) -> bool {
+        self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value
+    }
+
    async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
+        let hash = self.get_object_metadata_cache_hash(bucket, object);
+        let hash_bytes = hash.to_le_bytes();
+        let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
+        let generation = &self.get_object_metadata_cache_generations[index];
+        let previous = match generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1)) {
+            Ok(previous) | Err(previous) => previous,
+        };
+        let previous = GetObjectMetadataCacheGeneration {
+            index,
+            value: previous,
+            hash,
+        };
        self.get_object_metadata_cache
-            .invalidate(&GetObjectMetadataCacheKey::new(bucket, object))
+            .invalidate(&GetObjectMetadataCacheKey::new(bucket, object, previous))
            .await;
+        #[cfg(test)]
+        record_metadata_cache_invalidation(bucket, object);
+    }
+
+    fn invalidate_all_get_object_metadata_cache(&self) {
+        for generation in self.get_object_metadata_cache_generations.iter() {
+            let _ = generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1));
+        }
+        self.get_object_metadata_cache.invalidate_all();
    }

    async fn acquire_read_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
@@ -1953,6 +2356,12 @@ impl SetDisks {
                .max_capacity(get_object_metadata_cache_max_entries() as u64)
                .time_to_live(GET_OBJECT_METADATA_CACHE_TTL)
                .build(),
+            get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState::new(),
+            get_object_metadata_cache_generations: Arc::from(
+                (0..usize::from(GET_OBJECT_METADATA_CACHE_FENCE_SHARDS))
+                    .map(|_| AtomicU64::new(0))
+                    .collect::<Vec<_>>(),
+            ),
            lockers,
            // Sourced from the instance context so each instance owns its lock
            // namespace (Phase 5 Slice 3). Single-instance: ctx aliases the
@@ -21,8 +21,9 @@

 use super::super::*;

+use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent;
 use crate::disk::OldCurrentSize;
-use crate::object_api::GetObjectBodySource;
+use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};

 /// Length of the full plaintext body when — and only when — this read's output
 /// is exactly the object's complete plaintext, so the app-layer body cache may
@@ -61,6 +62,11 @@ fn full_object_plaintext_len(range: &Option<HTTPRangeSpec>, opts: &ObjectOptions
        || crate::object_api::restore_request_active(opts)
        || object_info.is_encrypted()
        || object_info.is_remote()
+        || object_info.delete_marker
+        || object_info.size == 0
+        || object_info.version_only
+        || object_info.metadata_only
+        || object_info.is_inline_fast_path_eligible()
    {
        return None;
    }
@@ -72,6 +78,14 @@ fn full_object_plaintext_len(range: &Option<HTTPRangeSpec>, opts: &ObjectOptions
    Some(object_info.size)
 }

+pub(crate) fn body_cache_plaintext_len(
+    range: &Option<HTTPRangeSpec>,
+    opts: &ObjectOptions,
+    object_info: &ObjectInfo,
+) -> Option<i64> {
+    full_object_plaintext_len(range, opts, object_info)
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
    type Error = Error;
@@ -131,16 +145,21 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
        };

        let metadata_stage_start = Instant::now();
-        let (fi, files, disks) = match self.get_object_fileinfo(bucket, object, opts, true, true).await {
-            Ok(result) => result,
-            Err(err) => {
-                rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64());
-                record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err));
-                return Err(to_object_err(err, vec![bucket, object]));
+        let (fi, files, disks, prepared_object_info) = if let Some(prepared) = take_prepared_get_object_metadata() {
+            (prepared.fi, prepared.files, prepared.disks, prepared.object_info)
+        } else {
+            match self.get_object_fileinfo(bucket, object, opts, true, true).await {
+                Ok((fi, files, disks)) => (fi, files, disks, None),
+                Err(err) => {
+                    rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64());
+                    record_get_object_pipeline_failure(GET_STAGE_METADATA, classify_storage_error(&err));
+                    return Err(to_object_err(err, vec![bucket, object]));
+                }
            }
        };
        let object_info_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
-        let object_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
+        let object_info = prepared_object_info
+            .unwrap_or_else(|| build_get_object_info(&fi, bucket, object, opts.versioned || opts.version_suspended));
        let object_class = classify_get_codec_streaming_object_class(&range, &object_info, &fi);
        let size_bucket = rustfs_io_metrics::get_object_size_bucket(object_info.size);
        record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start);
@@ -410,6 +429,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
        // it forward.
        let mut body_source = GetObjectBodySource::Unprobed;
        if let Some(plaintext_len) = full_object_plaintext_len(&range, opts, &object_info)
+            && !get_object_body_cache_hook_suppressed()
            && let Some(hook) = get_object_body_cache_hook()
        {
            match hook.lookup(bucket, object, &object_info).await {
@@ -1222,6 +1242,178 @@ impl SetDisks {
    }
 }

+async fn cleanup_uncommitted_transition_upload(tier: &str, object: &str, remote_version: &str) {
+    if let Err(err) = delete_object_from_remote_tier_idempotent(object, remote_version, tier).await {
+        warn!(
+            tier,
+            object,
+            remote_version,
+            error = ?err,
+            "failed to clean uncommitted transition upload"
+        );
+    }
+}
+
+struct TransitionUploadCleanup {
+    tier: String,
+    object: String,
+    remote_version: String,
+    armed: bool,
+}
+
+impl TransitionUploadCleanup {
+    fn new(tier: &str, object: &str, remote_version: &str) -> Self {
+        Self {
+            tier: tier.to_string(),
+            object: object.to_string(),
+            remote_version: remote_version.to_string(),
+            armed: true,
+        }
+    }
+
+    async fn cleanup(&mut self) {
+        cleanup_uncommitted_transition_upload(&self.tier, &self.object, &self.remote_version).await;
+        self.armed = false;
+    }
+
+    fn disarm(&mut self) {
+        self.armed = false;
+    }
+}
+
+impl Drop for TransitionUploadCleanup {
+    fn drop(&mut self) {
+        if !self.armed {
+            return;
+        }
+        let tier = self.tier.clone();
+        let object = self.object.clone();
+        let remote_version = self.remote_version.clone();
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            handle.spawn(async move {
+                cleanup_uncommitted_transition_upload(&tier, &object, &remote_version).await;
+            });
+        }
+    }
+}
+
+#[cfg(test)]
+struct TransitionCommitBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+struct TransitionCommitBarrier {
+    state: Arc<TransitionCommitBarrierState>,
+}
+
+#[cfg(test)]
+static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionCommitBarrierState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl TransitionCommitBarrier {
+    fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(TransitionCommitBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = TRANSITION_COMMIT_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("transition commit barrier mutex should not poison");
+        assert!(slot.is_none(), "transition commit barrier must be installed by one test at a time");
+        *slot = Some(Arc::clone(&state));
+        drop(slot);
+        Self { state }
+    }
+
+    async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("transition should reach the deterministic commit barrier");
+    }
+
+    fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for TransitionCommitBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = TRANSITION_COMMIT_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("transition commit barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_transition_before_local_commit(bucket: &str, object: &str) {
+    let barrier = TRANSITION_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("transition commit barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
+fn parse_transition_version_id(remote_version: &str) -> std::result::Result<Option<Uuid>, uuid::Error> {
+    if remote_version.is_empty() {
+        return Ok(None);
+    }
+    Uuid::parse_str(remote_version).map(|version_id| (!version_id.is_nil()).then_some(version_id))
+}
+
+fn transition_cleanup_remote_version(remote_version: &str, version_id: Option<Uuid>) -> &str {
+    version_id.map(|_| remote_version).unwrap_or("")
+}
+
+#[cfg(test)]
+mod transition_version_id_tests {
+    use super::{parse_transition_version_id, transition_cleanup_remote_version};
+    use uuid::Uuid;
+
+    #[test]
+    fn normalizes_unversioned_remote_ids() {
+        assert_eq!(parse_transition_version_id("").expect("empty remote version should be valid"), None);
+        assert_eq!(
+            parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"),
+            None
+        );
+        assert_eq!(transition_cleanup_remote_version(&Uuid::nil().to_string(), None), "");
+    }
+
+    #[test]
+    fn preserves_valid_remote_id_and_rejects_invalid_text() {
+        let version_id = Uuid::new_v4();
+        assert_eq!(
+            parse_transition_version_id(&version_id.to_string()).expect("UUID remote version should be valid"),
+            Some(version_id)
+        );
+        assert_eq!(
+            transition_cleanup_remote_version(&version_id.to_string(), Some(version_id)),
+            version_id.to_string()
+        );
+        assert!(parse_transition_version_id("not-a-uuid").is_err());
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
    type Error = Error;
@@ -1908,7 +2100,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                .await
                .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;

-            self.get_object_metadata_cache.invalidate_all();
+            self.invalidate_all_get_object_metadata_cache();
            return Ok(ObjectInfo::default());
        }

@@ -2236,7 +2428,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        if let Some(mod_time1) = opts.mod_time {
            if let Some(mod_time2) = fi.mod_time.as_ref() {
                if mod_time1.unix_timestamp() != mod_time2.unix_timestamp()
-                /*|| transition_etag != stored_etag*/
+                    || (!transition_etag.is_empty() && transition_etag != stored_etag)
                {
                    return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
                }
@@ -2333,25 +2525,88 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            return Err(StorageError::Io(err));
        }
        let rv = rv?;
-        fi.transition_status = TRANSITION_COMPLETE.to_string();
-        fi.transitioned_objname = dest_obj;
-        fi.transition_tier = opts.transition.tier.clone();
-        fi.transition_version_id = if rv.is_empty() { None } else { Some(Uuid::parse_str(&rv)?) };
+        drop(tier_config_mgr);
+
+        let transition_version_id = match parse_transition_version_id(&rv) {
+            Ok(version_id) => version_id,
+            Err(err) => {
+                cleanup_uncommitted_transition_upload(&opts.transition.tier, &dest_obj, &rv).await;
+                return Err(err.into());
+            }
+        };
+        let cleanup_remote_version = transition_cleanup_remote_version(&rv, transition_version_id);
+        let mut upload_cleanup = TransitionUploadCleanup::new(&opts.transition.tier, &dest_obj, cleanup_remote_version);
+
+        let mut commit_opts = opts.clone();
+        commit_opts.no_lock = true;
+        commit_opts.metadata_cache_safe = false;
+        let transition_lock_guard = if opts.no_lock {
+            None
+        } else {
+            match self.acquire_write_lock_diag("transition_object_commit", bucket, object).await {
+                Ok(guard) => Some(guard),
+                Err(err) => {
+                    upload_cleanup.cleanup().await;
+                    return Err(err);
+                }
+            }
+        };
+        self.invalidate_get_object_metadata_cache(bucket, object).await;
+        let current = self.get_object_fileinfo(bucket, object, &commit_opts, true, false).await;
+        let (mut current_fi, _, _) = match current {
+            Ok(current) => current,
+            Err(err) => {
+                drop(transition_lock_guard);
+                upload_cleanup.cleanup().await;
+                return Err(err);
+            }
+        };
+        let source_matches = current_fi.version_id == fi.version_id
+            && current_fi.data_dir == fi.data_dir
+            && current_fi.mod_time == fi.mod_time
+            && current_fi.size == fi.size
+            && rustfs_utils::path::trim_etag(&get_raw_etag(&current_fi.metadata)) == stored_etag;
+        if current_fi.transition_status == TRANSITION_COMPLETE || !source_matches {
+            let already_transitioned = current_fi.transition_status == TRANSITION_COMPLETE;
+            drop(transition_lock_guard);
+            upload_cleanup.cleanup().await;
+            if already_transitioned {
+                return Ok(());
+            }
+            return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
+        }
+
+        current_fi.transition_status = TRANSITION_COMPLETE.to_string();
+        current_fi.transitioned_objname = dest_obj;
+        current_fi.transition_tier = opts.transition.tier.clone();
+        current_fi.transition_version_id = transition_version_id;
+        fi = current_fi;
        let event_name = EventName::LifecycleTransition.as_str();
-        let mut should_notify_transition = true;
-
-        let disks = self.disk_inventory().await;

+        if transition_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            drop(transition_lock_guard);
+            upload_cleanup.cleanup().await;
+            return Err(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "transition_object_commit",
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                required: 1,
+                achieved: 0,
+            });
+        }
+        #[cfg(test)]
+        pause_transition_before_local_commit(bucket, object).await;
+        upload_cleanup.disarm();
        if let Err(err) = self.delete_object_version(bucket, object, &fi, false).await {
-            should_notify_transition = false;
            warn!(
                bucket = bucket,
                object = object,
                error = ?err,
-                "transition completed on remote tier but source cleanup failed; skipping external lifecycle transition notification"
+                "transition remote upload completed but local commit failed"
            );
-        } else {
-            self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
+            self.invalidate_get_object_metadata_cache(bucket, object).await;
+            drop(transition_lock_guard);
+            return Err(err);
        }

        // delete_object_version persisted transition_status=complete and freed the
@@ -2361,6 +2616,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        // early-return above and streams the already-deleted local data to the
        // remote tier again (rustfs/rustfs#4827).
        self.invalidate_get_object_metadata_cache(bucket, object).await;
+        drop(transition_lock_guard);
+        let disks = self.disk_inventory().await;
+        self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);

        for disk in disks.iter() {
            if let Some(disk) = disk {
@@ -2372,17 +2630,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            break;
        }

-        if should_notify_transition {
-            let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
-            send_event(EventArgs {
-                event_name: event_name.to_string(),
-                bucket_name: bucket.to_string(),
-                object: obj_info,
-                user_agent: "Internal: [ILM-Transition]".to_string(),
-                host: runtime_sources::default_local_node_name(),
-                ..Default::default()
-            });
-        }
+        let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
+        send_event(EventArgs {
+            event_name: event_name.to_string(),
+            bucket_name: bucket.to_string(),
+            object: obj_info,
+            user_agent: "Internal: [ILM-Transition]".to_string(),
+            host: runtime_sources::default_local_node_name(),
+            ..Default::default()
+        });
        //let tags = opts.lifecycle_audit_event.tags();
        //auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn)
        Ok(())
@@ -2738,6 +2994,127 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
    }
 }

+#[cfg(all(test, feature = "test-util"))]
+mod transition_commit_failure_tests {
+    use super::hermetic_set_disks_support::hermetic_set_disks;
+    use super::*;
+    use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
+    use crate::disk::DiskAPI as _;
+    use crate::services::tier::test_util::register_mock_tier;
+    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
+    use http::HeaderMap;
+    use tokio::io::AsyncReadExt;
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_commit_failure_returns_error_and_preserves_remote_candidate() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "transition-commit-failure-bucket";
+        let object = "object.bin";
+        let payload = b"transition commit failure must preserve the uploaded candidate".repeat(1024);
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let mut reader = PutObjReader::from_vec(payload.clone());
+        let original = set_disks
+            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("source object should be written");
+        let (fi, parts_metadata, online_disks) = set_disks
+            .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
+            .await
+            .expect("source metadata should resolve");
+        let generation = set_disks
+            .get_object_metadata_cache_generation(bucket, object)
+            .expect("metadata cache generation should be active");
+        let cache_key = GetObjectMetadataCacheKey::new(bucket, object, generation);
+        set_disks
+            .get_object_metadata_cache
+            .insert(
+                cache_key.clone(),
+                Arc::new(GetObjectMetadataCacheEntry {
+                    created_at: Instant::now(),
+                    fi: fi.clone(),
+                    parts_metadata,
+                    online_disks,
+                    read_quorum: 2,
+                }),
+            )
+            .await;
+        assert!(set_disks.get_object_metadata_cache.get(&cache_key).await.is_some());
+
+        let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
+        let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
+        let opts = ObjectOptions {
+            no_lock: true,
+            transition: TransitionOptions {
+                status: TRANSITION_PENDING.to_string(),
+                tier: tier_name,
+                etag: original.etag.clone().unwrap_or_default(),
+                ..Default::default()
+            },
+            version_id: original.version_id.map(|version| version.to_string()),
+            mod_time: original.mod_time,
+            ..Default::default()
+        };
+
+        let barrier = TransitionCommitBarrier::install(bucket, object);
+        let invalidations = MetadataCacheInvalidationProbe::install(bucket, object);
+        let transition_set = Arc::clone(&set_disks);
+        let transition = tokio::spawn(async move { transition_set.transition_object(bucket, object, &opts).await });
+        barrier.wait_until_paused().await;
+        assert_eq!(invalidations.count(), 1, "precommit revalidation must fence the old metadata once");
+        let saved_disks = {
+            let mut disks = set_disks.disks.write().await;
+            let saved = std::mem::take(&mut *disks);
+            *disks = vec![None; saved.len()];
+            saved
+        };
+        barrier.release();
+        let result = transition.await.expect("transition task should not panic");
+        *set_disks.disks.write().await = saved_disks;
+
+        result.expect_err("local write quorum failure must be returned to the transition worker");
+        assert_eq!(
+            invalidations.count(),
+            2,
+            "local commit failure must fence any partially committed metadata again"
+        );
+        assert_eq!(backend.put_count().await, 1);
+        assert_eq!(
+            backend.remove_count().await,
+            0,
+            "ambiguous local commit failure must retain the remote candidate"
+        );
+        assert_eq!(backend.object_count().await, 1);
+        assert!(
+            set_disks.get_object_metadata_cache.get(&cache_key).await.is_none(),
+            "local commit failure must invalidate pre-transition metadata"
+        );
+
+        let mut restored = Vec::new();
+        set_disks
+            .get_object_reader(
+                bucket,
+                object,
+                None,
+                HeaderMap::new(),
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("the local source must remain readable after a fail-before-commit result")
+            .stream
+            .read_to_end(&mut restored)
+            .await
+            .expect("the local source body should drain");
+        assert_eq!(restored, payload);
+    }
+}
+
 #[cfg(test)]
 mod put_object_tmp_cleanup_tests {
    //! Regression coverage for backlog#924 (HP-3): the speculative tmp-dir
@@ -20,21 +20,22 @@ use crate::diagnostics::get::{
    GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM,
    GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED,
    GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
-    GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
-    GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
-    GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
-    GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
-    GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
-    GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
-    GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
-    GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
-    GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
-    GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
-    GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
-    GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
-    GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
-    GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
-    record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
+    GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
+    GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
+    GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA,
+    GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
+    GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
+    GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
+    GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
+    GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
+    GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
+    GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
+    GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP,
+    GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING,
+    GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
+    GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error,
+    get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
+    record_get_stage_duration_if_enabled,
 };
 use crate::erasure::coding::BitrotReader;
 use crate::io_support::bitrot::{
@@ -60,6 +61,7 @@ use super::core::io_primitives::*;

 impl SetDisks {
    async fn get_object_metadata_cache_bypass_reason(
+        &self,
        bucket: &str,
        opts: &ObjectOptions,
        read_data: bool,
@@ -67,7 +69,8 @@ impl SetDisks {
        if let Some(reason) = get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data) {
            return Some(reason);
        }
-        runtime_sources::setup_is_dist_erasure()
+        self.ctx
+            .is_dist_erasure()
            .await
            .then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
    }
@@ -80,11 +83,28 @@ impl SetDisks {
    }

    async fn lookup_cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> MetadataCacheLookup {
-        let key = GetObjectMetadataCacheKey::new(bucket, object);
+        self.lookup_cached_get_object_fileinfo_after_get(bucket, object, || {}).await
+    }
+
+    async fn lookup_cached_get_object_fileinfo_after_get(
+        &self,
+        bucket: &str,
+        object: &str,
+        after_get: impl FnOnce(),
+    ) -> MetadataCacheLookup {
+        let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else {
+            return MetadataCacheLookup::Miss;
+        };
+        let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
        // moka handles TTL expiry automatically; no is_fresh() check needed
        let Some(entry) = self.get_object_metadata_cache.get(&key).await else {
            return MetadataCacheLookup::Miss;
        };
+        after_get();
+        if !self.is_get_object_metadata_cache_generation_current(generation) {
+            self.get_object_metadata_cache.invalidate(&key).await;
+            return MetadataCacheLookup::Miss;
+        }
        if entry.online_disks.iter().filter(|disk| disk.is_some()).count() >= entry.read_quorum {
            MetadataCacheLookup::Hit(entry)
        } else {
@@ -94,8 +114,8 @@ impl SetDisks {

    async fn cache_get_object_fileinfo(
        &self,
-        bucket: &str,
-        object: &str,
+        identity: (&str, &str),
+        generation: Option<GetObjectMetadataCacheGeneration>,
        fi: &FileInfo,
        parts_metadata: &[FileInfo],
        online_disks: &[Option<DiskStore>],
@@ -104,23 +124,51 @@ impl SetDisks {
        if fi.deleted || !fi.is_valid() {
            return;
        }
+        let (bucket, object) = identity;

-        let key = GetObjectMetadataCacheKey::new(bucket, object);
-        // moka handles capacity eviction (LRU) automatically
-        self.get_object_metadata_cache
-            .insert(
-                key,
-                Arc::new(GetObjectMetadataCacheEntry {
-                    created_at: Instant::now(),
-                    fi: fi.clone(),
-                    parts_metadata: parts_metadata.to_vec(),
-                    online_disks: online_disks.to_vec(),
-                    read_quorum,
-                }),
-            )
+        let Some(generation) = generation.filter(|generation| self.is_get_object_metadata_cache_generation_current(*generation))
+        else {
+            rustfs_io_metrics::record_get_object_metadata_cache_decision(
+                GET_OBJECT_PATH_SET_DISK,
+                GET_METADATA_CACHE_DECISION_REJECT,
+                GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
+            );
+            return;
+        };
+
+        let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
+        let entry = Arc::new(GetObjectMetadataCacheEntry {
+            created_at: Instant::now(),
+            fi: fi.clone(),
+            parts_metadata: parts_metadata.to_vec(),
+            online_disks: online_disks.to_vec(),
+            read_quorum,
+        });
+        self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
            .await;
    }

+    async fn insert_get_object_metadata_cache_entry_after_insert(
+        &self,
+        key: GetObjectMetadataCacheKey,
+        generation: GetObjectMetadataCacheGeneration,
+        entry: Arc<GetObjectMetadataCacheEntry>,
+        after_insert: impl FnOnce(),
+    ) {
+        self.get_object_metadata_cache.insert(key.clone(), entry).await;
+        after_insert();
+        if self.is_get_object_metadata_cache_generation_current(generation) {
+            return;
+        }
+
+        self.get_object_metadata_cache.invalidate(&key).await;
+        rustfs_io_metrics::record_get_object_metadata_cache_decision(
+            GET_OBJECT_PATH_SET_DISK,
+            GET_METADATA_CACHE_DECISION_REJECT,
+            GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
+        );
+    }
+
    #[cfg_attr(feature = "hotpath", hotpath::measure)]
    pub async fn read_version_optimized(
        &self,
@@ -193,7 +241,7 @@ impl SetDisks {
        let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();

        let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled);
-        let cache_bypass_reason = Self::get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await;
+        let cache_bypass_reason = self.get_object_metadata_cache_bypass_reason(bucket, opts, read_data).await;
        let use_metadata_cache = cache_bypass_reason.is_none();
        if let Some(reason) = cache_bypass_reason {
            rustfs_io_metrics::record_get_object_metadata_cache_decision(
@@ -238,6 +286,10 @@ impl SetDisks {
            metadata_cache_lookup_start,
        );

+        let metadata_cache_generation = use_metadata_cache
+            .then(|| self.get_object_metadata_cache_generation(bucket, object))
+            .flatten();
+
        let disks = self.disks.read().await;

        let disks = disks.clone();
@@ -311,8 +363,17 @@ impl SetDisks {
            )
            .await;
        } else if use_metadata_cache && metadata_fanout_complete {
-            self.cache_get_object_fileinfo(bucket, object, &fi, &parts_metadata, &op_online_disks, read_quorum)
-                .await;
+            #[cfg(test)]
+            metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
+            self.cache_get_object_fileinfo(
+                (bucket, object),
+                metadata_cache_generation,
+                &fi,
+                &parts_metadata,
+                &op_online_disks,
+                read_quorum,
+            )
+            .await;
        }
        record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
        // debug!("get_object_fileinfo pick fi {:?}", &fi);
@@ -1738,13 +1799,81 @@ mod metadata_cache_tests {
    use super::*;
    use rustfs_common::heal_channel::HealAdmissionDropReason;
    use serial_test::serial;
-    use std::sync::Mutex;
    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Mutex, OnceLock};
+    use tokio::sync::Notify;

    static SLOW_READ_REPAIR_SUBMITTER_CALLS: AtomicUsize = AtomicUsize::new(0);
    static DROPPED_READ_REPAIR_SUBMITTER_CALLS: AtomicUsize = AtomicUsize::new(0);
    static CAPTURED_READ_REPAIR_PRIORITY: Mutex<Option<HealChannelPriority>> = Mutex::new(None);
    static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0);
+    static METADATA_CACHE_PUBLISH_BARRIER: OnceLock<Mutex<Option<Arc<MetadataCachePublishBarrierState>>>> = OnceLock::new();
+
+    struct MetadataCachePublishBarrierState {
+        bucket: String,
+        object: String,
+        arrived: Notify,
+        release: Notify,
+    }
+
+    struct MetadataCachePublishBarrier {
+        state: Arc<MetadataCachePublishBarrierState>,
+    }
+
+    impl MetadataCachePublishBarrier {
+        fn install(bucket: &str, object: &str) -> Self {
+            let state = Arc::new(MetadataCachePublishBarrierState {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                arrived: Notify::new(),
+                release: Notify::new(),
+            });
+            let mut slot = METADATA_CACHE_PUBLISH_BARRIER
+                .get_or_init(|| Mutex::new(None))
+                .lock()
+                .expect("metadata publish barrier mutex should not poison");
+            assert!(slot.is_none(), "metadata publish barrier must be installed by one test at a time");
+            *slot = Some(Arc::clone(&state));
+            Self { state }
+        }
+
+        async fn wait_until_paused(&self) {
+            tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+                .await
+                .expect("metadata publication should reach the deterministic barrier");
+        }
+
+        fn release(&self) {
+            self.state.release.notify_one();
+        }
+    }
+
+    impl Drop for MetadataCachePublishBarrier {
+        fn drop(&mut self) {
+            self.state.release.notify_one();
+            let mut slot = METADATA_CACHE_PUBLISH_BARRIER
+                .get_or_init(|| Mutex::new(None))
+                .lock()
+                .expect("metadata publish barrier mutex should not poison");
+            if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+                *slot = None;
+            }
+        }
+    }
+
+    pub(super) async fn wait_before_metadata_cache_publish(bucket: &str, object: &str) {
+        let barrier = METADATA_CACHE_PUBLISH_BARRIER
+            .get_or_init(|| Mutex::new(None))
+            .lock()
+            .expect("metadata publish barrier mutex should not poison")
+            .as_ref()
+            .filter(|state| state.bucket == bucket && state.object == object)
+            .cloned();
+        if let Some(barrier) = barrier {
+            barrier.arrived.notify_one();
+            barrier.release.notified().await;
+        }
+    }

    fn slow_read_repair_submitter(_request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
        SLOW_READ_REPAIR_SUBMITTER_CALLS.fetch_add(1, Ordering::Relaxed);
@@ -1771,7 +1900,11 @@ mod metadata_cache_tests {
    }

    async fn new_metadata_cache_test_set() -> Arc<SetDisks> {
-        SetDisks::new(
+        new_metadata_cache_test_set_with_ctx(Arc::new(crate::runtime::instance::InstanceContext::new())).await
+    }
+
+    async fn new_metadata_cache_test_set_with_ctx(ctx: Arc<crate::runtime::instance::InstanceContext>) -> Arc<SetDisks> {
+        SetDisks::new_with_instance_ctx(
            "metadata-cache-test".to_string(),
            Arc::new(RwLock::new(Vec::new())),
            4,
@@ -1781,6 +1914,7 @@ mod metadata_cache_tests {
            Vec::new(),
            FormatV3::new(1, 4),
            Vec::new(),
+            ctx,
        )
        .await
    }
@@ -2334,7 +2468,8 @@ mod metadata_cache_tests {
        let parts_metadata = vec![fi.clone()];
        let online_disks = Vec::new();

-        set.cache_get_object_fileinfo("bucket", "object", &fi, &parts_metadata, &online_disks, 0)
+        let generation = set.get_object_metadata_cache_generation("bucket", "object");
+        set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0)
            .await;

        let cached = set
@@ -2353,7 +2488,8 @@ mod metadata_cache_tests {

        let mut deleted = valid_test_fileinfo("deleted-object");
        deleted.deleted = true;
-        set.cache_get_object_fileinfo("bucket", "deleted-object", &deleted, &[deleted.clone()], &[], 0)
+        let generation = set.get_object_metadata_cache_generation("bucket", "deleted-object");
+        set.cache_get_object_fileinfo(("bucket", "deleted-object"), generation, &deleted, &[deleted.clone()], &[], 0)
            .await;
        assert!(
            set.cached_get_object_fileinfo("bucket", "deleted-object").await.is_none(),
@@ -2361,7 +2497,8 @@ mod metadata_cache_tests {
        );

        let invalid = FileInfo::default();
-        set.cache_get_object_fileinfo("bucket", "invalid-object", &invalid, std::slice::from_ref(&invalid), &[], 0)
+        let generation = set.get_object_metadata_cache_generation("bucket", "invalid-object");
+        set.cache_get_object_fileinfo(("bucket", "invalid-object"), generation, &invalid, std::slice::from_ref(&invalid), &[], 0)
            .await;
        assert!(
            set.cached_get_object_fileinfo("bucket", "invalid-object").await.is_none(),
@@ -2376,7 +2513,12 @@ mod metadata_cache_tests {

        set.get_object_metadata_cache
            .insert(
-                GetObjectMetadataCacheKey::new("bucket", "object"),
+                GetObjectMetadataCacheKey::new(
+                    "bucket",
+                    "object",
+                    set.get_object_metadata_cache_generation("bucket", "object")
+                        .expect("metadata cache generation should be active"),
+                ),
                Arc::new(GetObjectMetadataCacheEntry {
                    created_at: Instant::now(),
                    fi: fi.clone(),
@@ -2401,7 +2543,8 @@ mod metadata_cache_tests {
        let set = new_metadata_cache_test_set().await;
        let fi = valid_test_fileinfo("object");

-        set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0)
+        let generation = set.get_object_metadata_cache_generation("bucket", "object");
+        set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
            .await;

        assert!(
@@ -2415,7 +2558,8 @@ mod metadata_cache_tests {
        let set = new_metadata_cache_test_set().await;
        let fi = valid_test_fileinfo("object");

-        set.cache_get_object_fileinfo("bucket", "object", &fi, std::slice::from_ref(&fi), &[], 0)
+        let generation = set.get_object_metadata_cache_generation("bucket", "object");
+        set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
            .await;
        assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_some());

@@ -2426,6 +2570,193 @@ mod metadata_cache_tests {
        );
    }

+    #[tokio::test]
+    #[serial(metadata_cache_publish_barrier)]
+    async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() {
+        let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks(4, 2).await;
+        let bucket = "metadata-cache-production-fence";
+        let object = "object";
+        let disks = set.disks.read().await.clone();
+        for disk in disks.iter().flatten() {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+        let mut reader = PutObjReader::from_vec(vec![7u8; 1024]);
+        set.put_object(bucket, object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("test object should be written");
+
+        let barrier = MetadataCachePublishBarrier::install(bucket, object);
+        let stale_generation = set
+            .get_object_metadata_cache_generation(bucket, object)
+            .expect("metadata cache generation should be active");
+        let reader_set = Arc::clone(&set);
+        let read = tokio::spawn(async move {
+            reader_set
+                .get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
+                .await
+        });
+        barrier.wait_until_paused().await;
+        set.invalidate_get_object_metadata_cache(bucket, object).await;
+        barrier.release();
+        read.await
+            .expect("metadata read task should not panic")
+            .expect("metadata fanout should still return its selected FileInfo");
+
+        assert!(
+            set.get_object_metadata_cache
+                .get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation))
+                .await
+                .is_none(),
+            "production stale publication must not remain under its retired generation key"
+        );
+        assert!(
+            set.cached_get_object_fileinfo(bucket, object).await.is_none(),
+            "the production fanout token captured before invalidation must not publish afterward"
+        );
+    }
+
+    #[tokio::test]
+    async fn metadata_cache_lookup_rechecks_generation_after_get() {
+        let set = new_metadata_cache_test_set().await;
+        let fi = valid_test_fileinfo("object");
+        let generation = set.get_object_metadata_cache_generation("bucket", "object");
+        let generation_index = generation.expect("metadata cache generation should be active").index;
+        set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, std::slice::from_ref(&fi), &[], 0)
+            .await;
+
+        let lookup = set
+            .lookup_cached_get_object_fileinfo_after_get("bucket", "object", || {
+                set.get_object_metadata_cache_generations[generation_index].fetch_add(1, Ordering::AcqRel);
+            })
+            .await;
+
+        assert!(
+            matches!(lookup, MetadataCacheLookup::Miss),
+            "a hit overlapping invalidation must be rejected after the second generation read"
+        );
+    }
+
+    #[tokio::test]
+    async fn metadata_cache_publication_rechecks_generation_after_insert() {
+        let set = new_metadata_cache_test_set().await;
+        let fi = valid_test_fileinfo("object");
+        let generation = set
+            .get_object_metadata_cache_generation("bucket", "object")
+            .expect("metadata cache generation should be active");
+        let generation_index = generation.index;
+        let key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
+        let entry = Arc::new(GetObjectMetadataCacheEntry {
+            created_at: Instant::now(),
+            fi: fi.clone(),
+            parts_metadata: vec![fi],
+            online_disks: Vec::new(),
+            read_quorum: 0,
+        });
+
+        set.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {
+            set.get_object_metadata_cache_generations[generation_index].fetch_add(1, Ordering::AcqRel);
+        })
+        .await;
+        assert!(
+            set.get_object_metadata_cache
+                .get(&GetObjectMetadataCacheKey::new("bucket", "object", generation))
+                .await
+                .is_none(),
+            "post-insert generation change must withdraw the retired entry"
+        );
+        assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
+    }
+
+    #[tokio::test]
+    async fn metadata_cache_generation_advances_monotonically() {
+        let set = new_metadata_cache_test_set().await;
+        let fi = valid_test_fileinfo("object");
+        let initial = set
+            .get_object_metadata_cache_generation("bucket", "object")
+            .expect("initial generation should be active");
+
+        set.invalidate_get_object_metadata_cache("bucket", "object").await;
+        let first = set
+            .get_object_metadata_cache_generation("bucket", "object")
+            .expect("first generation should be active");
+        set.invalidate_get_object_metadata_cache("bucket", "object").await;
+        let second = set
+            .get_object_metadata_cache_generation("bucket", "object")
+            .expect("second generation should be active");
+
+        assert_eq!(first.value, initial.value + 1);
+        assert_eq!(second.value, first.value + 1);
+        set.cache_get_object_fileinfo(("bucket", "object"), Some(first), &fi, std::slice::from_ref(&fi), &[], 0)
+            .await;
+        assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
+    }
+
+    #[tokio::test]
+    async fn metadata_cache_distribution_bypass_uses_set_instance_context() {
+        let distributed_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        distributed_ctx
+            .update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
+            .await;
+        let standalone_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        standalone_ctx
+            .update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
+            .await;
+        let distributed = new_metadata_cache_test_set_with_ctx(distributed_ctx).await;
+        let standalone = new_metadata_cache_test_set_with_ctx(standalone_ctx).await;
+
+        assert_eq!(
+            distributed
+                .get_object_metadata_cache_bypass_reason("bucket", &ObjectOptions::default(), true)
+                .await,
+            Some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
+        );
+        assert_eq!(
+            standalone
+                .get_object_metadata_cache_bypass_reason("bucket", &ObjectOptions::default(), true)
+                .await,
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn metadata_cache_generation_overflow_fails_closed() {
+        let set = new_metadata_cache_test_set().await;
+        let fi = valid_test_fileinfo("object");
+        let generation_index = set
+            .get_object_metadata_cache_generation("bucket", "object")
+            .expect("metadata cache generation should be active")
+            .index;
+        set.get_object_metadata_cache_generations[generation_index].store(u64::MAX - 1, Ordering::Release);
+        let last_generation = set.get_object_metadata_cache_generation("bucket", "object");
+        assert_eq!(last_generation.map(|generation| generation.value), Some(u64::MAX - 1));
+
+        set.invalidate_get_object_metadata_cache("bucket", "object").await;
+
+        assert_eq!(set.get_object_metadata_cache_generation("bucket", "object"), None);
+        set.cache_get_object_fileinfo(("bucket", "object"), last_generation, &fi, std::slice::from_ref(&fi), &[], 0)
+            .await;
+        assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
+        set.invalidate_get_object_metadata_cache("bucket", "object").await;
+        assert_eq!(
+            set.get_object_metadata_cache_generations[generation_index].load(Ordering::Acquire),
+            u64::MAX,
+            "overflow must permanently disable publication instead of wrapping"
+        );
+    }
+
+    #[tokio::test]
+    async fn metadata_cache_invalidate_all_fences_late_insert() {
+        let set = new_metadata_cache_test_set().await;
+        let fi = valid_test_fileinfo("object");
+        let stale_generation = set.get_object_metadata_cache_generation("bucket", "object");
+
+        set.invalidate_all_get_object_metadata_cache();
+
+        set.cache_get_object_fileinfo(("bucket", "object"), stale_generation, &fi, std::slice::from_ref(&fi), &[], 0)
+            .await;
+        assert!(set.cached_get_object_fileinfo("bucket", "object").await.is_none());
+    }
+
    #[tokio::test]
    async fn get_object_metadata_cache_prunes_when_capacity_is_reached() {
        // moka handles capacity eviction automatically via the configured max_capacity.
@@ -2433,7 +2764,8 @@ mod metadata_cache_tests {
        let set = new_metadata_cache_test_set().await;
        let fresh_fi = valid_test_fileinfo("fresh-object");

-        set.cache_get_object_fileinfo("bucket", "fresh-object", &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0)
+        let generation = set.get_object_metadata_cache_generation("bucket", "fresh-object");
+        set.cache_get_object_fileinfo(("bucket", "fresh-object"), generation, &fresh_fi, std::slice::from_ref(&fresh_fi), &[], 0)
            .await;

        assert!(