fix(ecstore): require commit quorum for latest metadata (#4117)

crates/ecstore/src/cache_value/metacache_set.rs

@@ -19,7 +19,7 @@ use futures::future::join_all;
 use metrics::counter;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
 use std::{
-    collections::VecDeque,
+    collections::{HashSet, VecDeque},
     future::Future,
     pin::Pin,
     sync::{Arc, OnceLock},
@@ -41,6 +41,33 @@ pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 
+#[derive(Clone, Default)]
+pub(crate) struct FallbackClaimTracker {
+    claimed: Arc<TokioMutex<HashSet<String>>>,
+}
+
+impl FallbackClaimTracker {
+    pub(crate) async fn claim_disk(&self, disk: &DiskStore) {
+        self.claimed.lock().await.insert(disk.endpoint().to_string());
+    }
+
+    pub(crate) async fn claimed_keys(&self) -> HashSet<String> {
+        self.claimed.lock().await.clone()
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn claim_test_fallback(&self) {
+        let mut claimed = self.claimed.lock().await;
+        let key = format!("test-fallback-{}", claimed.len());
+        claimed.insert(key);
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn contains_key(&self, key: &str) -> bool {
+        self.claimed.lock().await.contains(key)
+    }
+}
+
 #[derive(Debug)]
 enum PeekOutcome {
     Ready(Option<MetaCacheEntry>),
@@ -129,9 +156,28 @@ impl Clone for ListPathRawOptions {
 }
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
+    list_path_raw_inner(rx, opts, None).await
+}
+
+pub(crate) async fn list_path_raw_with_claim_tracker(
+    rx: CancellationToken,
+    opts: ListPathRawOptions,
+    claim_tracker: FallbackClaimTracker,
+) -> disk::error::Result<()> {
+    list_path_raw_inner(rx, opts, Some(claim_tracker)).await
+}
+
+async fn list_path_raw_inner(
+    rx: CancellationToken,
+    opts: ListPathRawOptions,
+    fallback_claim_tracker: Option<FallbackClaimTracker>,
+) -> disk::error::Result<()> {
     if opts.disks.is_empty() {
         return Err(DiskError::ErasureReadQuorum);
     }
+    if opts.min_disks > opts.disks.len() {
+        return Err(DiskError::ErasureReadQuorum);
+    }
 
     let log_bucket = opts.bucket.clone();
     let log_path = opts.path.clone();
@@ -151,6 +197,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     for (disk_idx, disk) in opts.disks.iter().enumerate() {
         let opdisk = disk.clone();
         let opts_clone = opts.clone();
+        let fallback_claim_tracker = fallback_claim_tracker.clone();
         let fds_clone = fds.clone();
         #[cfg(test)]
         let test_fallbacks_clone = test_fallbacks.clone();
@@ -276,6 +323,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
             while need_fallback {
                 #[cfg(test)]
                 if let Some(behavior) = take_fallback_candidate(&test_fallbacks_clone).await {
+                    if let Some(claim_tracker) = fallback_claim_tracker.as_ref() {
+                        claim_tracker.claim_test_fallback().await;
+                    }
                     match behavior {
                         TestReaderBehavior::Eof => {
                             need_fallback = false;
@@ -341,6 +391,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                     record_producer_error(&producer_errs_clone, disk_idx, &err);
                     return Err(err);
                 };
+                if let Some(claim_tracker) = fallback_claim_tracker.as_ref() {
+                    claim_tracker.claim_disk(&disk).await;
+                }
 
                 let fallback_walk_started = std::time::Instant::now();
                 match disk
@@ -797,6 +850,22 @@ mod tests {
         assert_eq!(err, DiskError::ErasureReadQuorum);
     }
 
+    #[tokio::test]
+    async fn list_path_raw_rejects_impossible_min_disks() {
+        let err = list_path_raw(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None, None],
+                min_disks: 3,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect_err("impossible listing quorum should fail before producing partial results");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
+
     #[test]
     fn missing_path_error_classification_excludes_actionable_failures() {
         assert!(is_missing_path_error(&DiskError::FileNotFound));
@@ -889,6 +958,27 @@ mod tests {
         assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &[2]);
     }
 
+    #[tokio::test]
+    async fn list_path_raw_records_claimed_fallback_candidates() {
+        let claim_tracker = FallbackClaimTracker::default();
+
+        list_path_raw_with_claim_tracker(
+            CancellationToken::new(),
+            ListPathRawOptions {
+                disks: vec![None],
+                min_disks: 1,
+                test_reader_behaviors: vec![TestReaderBehavior::PrimaryErrorThenFallback(DiskError::DiskNotFound)],
+                test_fallback_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
+                ..Default::default()
+            },
+            claim_tracker.clone(),
+        )
+        .await
+        .expect("fallback producer should restore the single logical reader");
+
+        assert!(claim_tracker.contains_key("test-fallback-0").await);
+    }
+
     #[tokio::test]
     async fn list_path_raw_returns_timeout_when_reader_stalls_before_completion() {
         let err = list_path_raw(

crates/ecstore/src/metadata/set_disk.rs

@@ -13,6 +13,14 @@
 // limitations under the License.
 
 use super::*;
+use rustfs_utils::http;
+
+#[derive(Clone, Copy)]
+struct FileInfoIdentityGroup {
+    hash: [u8; 32],
+    count: usize,
+    mod_time: Option<OffsetDateTime>,
+}
 
 impl SetDisks {
     pub(super) fn all_not_found_metadata(errs: &[Option<DiskError>]) -> bool {
@@ -338,6 +346,90 @@ impl SetDisks {
         (new_disk, mod_time, None)
     }
 
+    fn usable_fileinfo_count(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> (usize, bool) {
+        let mut has_read_error = false;
+        let mut usable_metadata = 0;
+        for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
+            if err.is_some() {
+                has_read_error = true;
+                continue;
+            }
+
+            if meta.is_valid() {
+                usable_metadata += 1;
+            }
+        }
+
+        (usable_metadata, has_read_error)
+    }
+
+    pub(super) fn latest_fileinfo_selection_quorum(
+        version_id: &str,
+        parts_metadata: &[FileInfo],
+        errs: &[Option<DiskError>],
+        read_quorum: usize,
+        write_quorum: usize,
+    ) -> usize {
+        if !version_id.is_empty() || write_quorum <= read_quorum {
+            return read_quorum;
+        }
+
+        let (usable_metadata, has_read_error) = Self::usable_fileinfo_count(parts_metadata, errs);
+
+        if usable_metadata < write_quorum {
+            return read_quorum;
+        }
+
+        if !has_read_error {
+            return write_quorum;
+        }
+
+        let mut identity_counts = HashMap::with_capacity(usable_metadata);
+        for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
+            if err.is_some() || !meta.is_valid() {
+                continue;
+            }
+
+            let key = Self::file_info_quorum_hash(meta);
+
+            let count = identity_counts.entry(key).or_insert(0);
+            *count += 1;
+            if *count >= write_quorum {
+                return write_quorum;
+            }
+        }
+
+        read_quorum
+    }
+
+    pub(super) fn select_valid_fileinfo(
+        disks: &[Option<DiskStore>],
+        parts_metadata: &[FileInfo],
+        errs: &[Option<DiskError>],
+        version_id: &str,
+        read_quorum: usize,
+        write_quorum: usize,
+    ) -> disk::error::Result<(Vec<Option<DiskStore>>, FileInfo, usize)> {
+        let selection_quorum =
+            Self::latest_fileinfo_selection_quorum(version_id, parts_metadata, errs, read_quorum, write_quorum);
+        let (usable_metadata, has_read_error) = Self::usable_fileinfo_count(parts_metadata, errs);
+
+        if version_id.is_empty()
+            && write_quorum > read_quorum
+            && has_read_error
+            && usable_metadata >= write_quorum
+            && selection_quorum == read_quorum
+        {
+            let (online_disks, fi) = Self::pick_degraded_latest_fileinfo(disks, parts_metadata, errs, read_quorum, write_quorum)?;
+            return Ok((online_disks, fi, read_quorum));
+        }
+
+        let (online_disks, mod_time, etag) = Self::list_online_disks(disks, parts_metadata, errs, selection_quorum);
+        let fi = Self::pick_valid_fileinfo(parts_metadata, mod_time, etag, selection_quorum)?;
+
+        Ok((online_disks, fi, selection_quorum))
+    }
+
     pub(super) fn pick_valid_fileinfo(
         metas: &[FileInfo],
         mod_time: Option<OffsetDateTime>,
@@ -347,52 +439,271 @@ impl SetDisks {
         Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
     }
 
+    fn update_hash_bytes(hasher: &mut Sha256, value: &[u8]) {
+        hasher.update(value.len().to_le_bytes());
+        hasher.update(value);
+    }
+
+    fn update_hash_str(hasher: &mut Sha256, value: &str) {
+        Self::update_hash_bytes(hasher, value.as_bytes());
+    }
+
+    fn update_hash_optional_uuid(hasher: &mut Sha256, value: Option<Uuid>) {
+        if let Some(value) = value {
+            hasher.update([1]);
+            hasher.update(value.as_bytes());
+        } else {
+            hasher.update([0]);
+        }
+    }
+
+    fn update_hash_optional_time(hasher: &mut Sha256, value: Option<OffsetDateTime>) {
+        if let Some(value) = value {
+            hasher.update([1]);
+            hasher.update(value.unix_timestamp_nanos().to_le_bytes());
+        } else {
+            hasher.update([0]);
+        }
+    }
+
+    fn update_hash_optional_u32(hasher: &mut Sha256, value: Option<u32>) {
+        if let Some(value) = value {
+            hasher.update([1]);
+            hasher.update(value.to_le_bytes());
+        } else {
+            hasher.update([0]);
+        }
+    }
+
+    fn update_hash_optional_u64(hasher: &mut Sha256, value: Option<u64>) {
+        if let Some(value) = value {
+            hasher.update([1]);
+            hasher.update(value.to_le_bytes());
+        } else {
+            hasher.update([0]);
+        }
+    }
+
+    fn update_hash_optional_bytes(hasher: &mut Sha256, value: Option<&Bytes>) {
+        if let Some(value) = value {
+            hasher.update([1]);
+            Self::update_hash_bytes(hasher, value);
+        } else {
+            hasher.update([0]);
+        }
+    }
+
+    fn update_hash_optional_str(hasher: &mut Sha256, value: Option<&str>) {
+        if let Some(value) = value {
+            hasher.update([1]);
+            Self::update_hash_str(hasher, value);
+        } else {
+            hasher.update([0]);
+        }
+    }
+
+    fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
+        value
+            .get(..prefix.len())
+            .is_some_and(|value_prefix| value_prefix.eq_ignore_ascii_case(prefix))
+    }
+
+    fn internal_metadata_suffix(name: &str) -> Option<&str> {
+        name.get(http::RUSTFS_INTERNAL_PREFIX.len()..)
+            .filter(|_| Self::starts_with_ignore_ascii_case(name, http::RUSTFS_INTERNAL_PREFIX))
+            .or_else(|| {
+                name.get(http::MINIO_INTERNAL_PREFIX.len()..)
+                    .filter(|_| Self::starts_with_ignore_ascii_case(name, http::MINIO_INTERNAL_PREFIX))
+            })
+    }
+
+    fn is_replication_quorum_metadata_key(name: &str) -> bool {
+        if name.eq_ignore_ascii_case(http::AMZ_BUCKET_REPLICATION_STATUS) {
+            return true;
+        }
+
+        let Some(suffix) = Self::internal_metadata_suffix(name) else {
+            return false;
+        };
+
+        suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICA_STATUS)
+            || suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICA_TIMESTAMP)
+            || suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICATION_STATUS)
+            || suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICATION_TIMESTAMP)
+            || suffix.eq_ignore_ascii_case(http::SUFFIX_PURGESTATUS)
+            || Self::starts_with_ignore_ascii_case(suffix, http::SUFFIX_REPLICATION_RESET_ARN_PREFIX)
+    }
+
+    fn update_hash_quorum_metadata_map(hasher: &mut Sha256, entries: &HashMap<String, String>) {
+        let mut entries = entries
+            .iter()
+            .filter(|(name, _)| !Self::is_replication_quorum_metadata_key(name))
+            .collect::<Vec<_>>();
+        entries.sort_by(|left, right| left.0.cmp(right.0));
+        hasher.update(entries.len().to_le_bytes());
+        for (name, value) in entries {
+            Self::update_hash_str(hasher, name);
+            Self::update_hash_str(hasher, value);
+        }
+    }
+
+    fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
+        let mut hasher = Sha256::new();
+        Self::update_file_info_quorum_hash(&mut hasher, meta);
+        let digest = hasher.finalize();
+        let mut key = [0u8; 32];
+        key.copy_from_slice(digest.as_slice());
+        key
+    }
+
     fn update_file_info_quorum_hash(hasher: &mut Sha256, meta: &FileInfo) {
         hasher.update(meta.size.to_le_bytes());
-        hasher.update([meta.deleted as u8, meta.mark_deleted as u8]);
+        hasher.update([u8::from(meta.deleted), u8::from(meta.mark_deleted)]);
+        hasher.update([u8::from(meta.expire_restored)]);
+        Self::update_hash_optional_time(hasher, meta.mod_time);
+        Self::update_hash_str(hasher, &meta.transition_status);
+        Self::update_hash_str(hasher, &meta.transition_tier);
+        Self::update_hash_str(hasher, &meta.transitioned_objname);
+        Self::update_hash_optional_uuid(hasher, meta.transition_version_id);
+        Self::update_hash_optional_u32(hasher, meta.mode);
+        Self::update_hash_optional_u64(hasher, meta.written_by_version);
 
-        if let Some(version_id) = meta.version_id {
-            hasher.update(version_id.as_bytes());
-        }
+        Self::update_hash_optional_uuid(hasher, meta.version_id);
+        Self::update_hash_optional_uuid(hasher, meta.data_dir);
 
-        if let Some(data_dir) = meta.data_dir {
-            hasher.update(data_dir.as_bytes());
-        }
+        Self::update_hash_optional_bytes(hasher, meta.checksum.as_ref());
 
-        if let Some(checksum) = &meta.checksum {
-            hasher.update(checksum);
-        }
+        Self::update_hash_quorum_metadata_map(hasher, &meta.metadata);
 
+        hasher.update(meta.parts.len().to_le_bytes());
         for part in meta.parts.iter() {
-            hasher.update(format!("part.{}", part.number).as_bytes());
-            hasher.update(format!("part.{}", part.size).as_bytes());
+            hasher.update(part.number.to_le_bytes());
+            hasher.update(part.size.to_le_bytes());
             hasher.update(part.actual_size.to_le_bytes());
-            hasher.update(part.etag.as_bytes());
+            Self::update_hash_str(hasher, &part.etag);
 
-            if let Some(mod_time) = part.mod_time {
-                hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
-            }
+            Self::update_hash_optional_time(hasher, part.mod_time);
 
-            if let Some(index) = &part.index {
-                hasher.update(index);
-            }
+            Self::update_hash_optional_bytes(hasher, part.index.as_ref());
+            Self::update_hash_optional_str(hasher, part.error.as_deref());
 
             if let Some(checksums) = &part.checksums {
                 let mut checksum_entries = checksums.iter().collect::<Vec<_>>();
                 checksum_entries.sort_by(|left, right| left.0.cmp(right.0));
+                hasher.update(checksum_entries.len().to_le_bytes());
                 for (name, value) in checksum_entries {
-                    hasher.update(name.as_bytes());
-                    hasher.update(value.as_bytes());
+                    Self::update_hash_str(hasher, name);
+                    Self::update_hash_str(hasher, value);
                 }
+            } else {
+                hasher.update(0usize.to_le_bytes());
             }
         }
 
         if !meta.deleted && meta.size != 0 {
-            hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes());
-            hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes());
+            hasher.update(meta.erasure.data_blocks.to_le_bytes());
+            hasher.update(meta.erasure.parity_blocks.to_le_bytes());
+            hasher.update(meta.erasure.distribution.len().to_le_bytes());
+            for disk_index in meta.erasure.distribution.iter() {
+                hasher.update(disk_index.to_le_bytes());
+            }
         }
     }
 
+    fn latest_fileinfo_identity_groups(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<FileInfoIdentityGroup> {
+        let mut groups: Vec<FileInfoIdentityGroup> = Vec::with_capacity(parts_metadata.len());
+        for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
+            if err.is_some() || !meta.is_valid() {
+                continue;
+            }
+
+            let hash = Self::file_info_quorum_hash(meta);
+            if let Some(group) = groups.iter_mut().find(|group| group.hash == hash) {
+                group.count += 1;
+                continue;
+            }
+
+            groups.push(FileInfoIdentityGroup {
+                hash,
+                count: 1,
+                mod_time: meta.mod_time,
+            });
+        }
+
+        groups
+    }
+
+    fn pick_fileinfo_identity(
+        disks: &[Option<DiskStore>],
+        parts_metadata: &[FileInfo],
+        errs: &[Option<DiskError>],
+        hash: [u8; 32],
+        quorum: usize,
+    ) -> disk::error::Result<(Vec<Option<DiskStore>>, FileInfo)> {
+        let mut online_disks = vec![None; disks.len()];
+        let mut selected = None;
+        let mut count = 0;
+
+        for (i, ((meta, err), disk)) in parts_metadata.iter().zip(errs.iter()).zip(disks.iter()).enumerate() {
+            if err.is_some() || !meta.is_valid() || Self::file_info_quorum_hash(meta) != hash {
+                continue;
+            }
+
+            count += 1;
+            online_disks[i].clone_from(disk);
+            if selected.is_none() {
+                selected = Some(meta.clone());
+            }
+        }
+
+        if count < quorum {
+            return Err(DiskError::ErasureReadQuorum);
+        }
+
+        selected
+            .map(|mut fi| {
+                fi.is_latest = fi.successor_mod_time.is_none();
+                (online_disks, fi)
+            })
+            .ok_or(DiskError::ErasureReadQuorum)
+    }
+
+    fn pick_degraded_latest_fileinfo(
+        disks: &[Option<DiskStore>],
+        parts_metadata: &[FileInfo],
+        errs: &[Option<DiskError>],
+        read_quorum: usize,
+        write_quorum: usize,
+    ) -> disk::error::Result<(Vec<Option<DiskStore>>, FileInfo)> {
+        let mut groups = Self::latest_fileinfo_identity_groups(parts_metadata, errs);
+        if groups.is_empty() {
+            return Err(DiskError::ErasureReadQuorum);
+        }
+
+        groups.sort_by(|left, right| right.mod_time.cmp(&left.mod_time).then_with(|| right.count.cmp(&left.count)));
+        let latest_mod_time = groups[0].mod_time;
+
+        let mut older_start = 0;
+        while older_start < groups.len() && groups[older_start].mod_time == latest_mod_time {
+            if groups[older_start].count >= write_quorum {
+                return Self::pick_fileinfo_identity(disks, parts_metadata, errs, groups[older_start].hash, write_quorum);
+            }
+            older_start += 1;
+        }
+
+        if older_start > 1 {
+            return Err(DiskError::ErasureReadQuorum);
+        }
+
+        for group in groups.iter().skip(older_start) {
+            if group.count >= read_quorum {
+                return Self::pick_fileinfo_identity(disks, parts_metadata, errs, group.hash, read_quorum);
+            }
+        }
+
+        Err(DiskError::ErasureReadQuorum)
+    }
+
     pub(super) fn find_file_info_in_quorum(
         metas: &[FileInfo],
         mod_time: &Option<OffsetDateTime>,
@@ -405,7 +716,6 @@ impl SetDisks {
         }
 
         let mut meta_hashes = vec![None; metas.len()];
-        let mut hasher = Sha256::new();
 
         for (i, meta) in metas.iter().enumerate() {
             if !meta.is_valid() {
@@ -437,8 +747,6 @@ impl SetDisks {
             let mod_valid = mod_time == &meta.mod_time;
 
             if etag_only || mod_valid {
-                Self::update_file_info_quorum_hash(&mut hasher, meta);
-
                 if meta.is_remote() {
                     // TODO:
                 }
@@ -447,9 +755,7 @@ impl SetDisks {
 
                 // TODO: IsCompressed
 
-                meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice()));
-
-                hasher.reset();
+                meta_hashes[i] = Some(Self::file_info_quorum_hash(meta));
             } else {
                 debug!(
                     index = i,
@@ -462,7 +768,7 @@ impl SetDisks {
 
         let mut count_map = HashMap::new();
 
-        for hash in meta_hashes.iter().flatten() {
+        for hash in meta_hashes.iter().flatten().copied() {
             *count_map.entry(hash).or_insert(0) += 1;
         }
 
@@ -495,7 +801,7 @@ impl SetDisks {
         for (i, op_hash) in meta_hashes.iter().enumerate() {
             if let Some(hash) = op_hash
                 && let Some(max_hash) = max_val
-                && hash == max_hash
+                && *hash == max_hash
                 && metas[i].is_valid()
             {
                 if !found {
@@ -504,7 +810,7 @@ impl SetDisks {
                 }
 
                 let props = ObjProps {
-                    mod_time: metas[i].mod_time,
+                    successor_mod_time: metas[i].successor_mod_time,
                     num_versions: metas[i].num_versions,
                 };
 
@@ -517,9 +823,9 @@ impl SetDisks {
 
             for (val, &count) in &valid_obj_map {
                 if count >= quorum {
-                    fi.mod_time = val.mod_time;
+                    fi.successor_mod_time = val.successor_mod_time;
                     fi.num_versions = val.num_versions;
-                    fi.is_latest = val.mod_time.is_none();
+                    fi.is_latest = val.successor_mod_time.is_none();
 
                     break;
                 }

crates/ecstore/src/set_disk/mod.rs

@@ -97,7 +97,8 @@ use rustfs_common::heal_channel::{
 use rustfs_config::MI_B;
 use rustfs_filemeta::{
     FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
-    RawFileInfo, ReplicateDecision, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw, merge_file_meta_versions,
+    RawFileInfo, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw,
+    merge_file_meta_versions,
 };
 use rustfs_io_metrics::{
     record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
@@ -3475,7 +3476,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         };
 
         let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
-            Ok((r, w)) => (r as usize, w as usize),
+            Ok((r, w)) => (
+                usize::try_from(r)
+                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![src_bucket, src_object]))?,
+                usize::try_from(w)
+                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![src_bucket, src_object]))?,
+            ),
             Err(mut err) => {
                 if err == DiskError::ErasureReadQuorum
                     && !src_bucket.starts_with(RUSTFS_META_BUCKET)
@@ -3494,10 +3500,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let (online_disks, mod_time, etag) = Self::list_online_disks(&disks, &metas, &errs, read_quorum);
-
-        let mut fi = Self::pick_valid_fileinfo(&metas, mod_time, etag, read_quorum)
-            .map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
+        let src_version_id = src_opts.version_id.as_deref().unwrap_or_default();
+        let (online_disks, mut fi, _) =
+            Self::select_valid_fileinfo(&disks, &metas, &errs, src_version_id, read_quorum, write_quorum)
+                .map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
 
         if fi.deleted {
             if src_opts.version_id.is_none() {
@@ -4078,8 +4084,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let read_quorum = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
-            Ok((res, _)) => res,
+        let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
+            Ok((read_quorum, write_quorum)) => (read_quorum, write_quorum),
             Err(mut err) => {
                 if err == DiskError::ErasureReadQuorum
                     && !bucket.starts_with(RUSTFS_META_BUCKET)
@@ -4098,11 +4104,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let read_quorum = read_quorum as usize;
+        let read_quorum =
+            usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
+        let write_quorum = usize::try_from(write_quorum)
+            .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
 
-        let (online_disks, mod_time, etag) = Self::list_online_disks(&disks, &metas, &errs, read_quorum);
-
-        let mut fi = Self::pick_valid_fileinfo(&metas, mod_time, etag, read_quorum)
+        let version_id = opts.version_id.as_deref().unwrap_or_default();
+        let (online_disks, mut fi, _) = Self::select_valid_fileinfo(&disks, &metas, &errs, version_id, read_quorum, write_quorum)
             .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
 
         if fi.deleted {
@@ -6045,13 +6053,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
 
 #[derive(Debug, PartialEq, Eq)]
 struct ObjProps {
-    mod_time: Option<OffsetDateTime>,
+    successor_mod_time: Option<OffsetDateTime>,
     num_versions: usize,
 }
 
 impl Hash for ObjProps {
     fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.mod_time.hash(state);
+        self.successor_mod_time.hash(state);
         self.num_versions.hash(state);
     }
 }
@@ -8062,6 +8070,34 @@ mod tests {
         }
     }
 
+    fn decoded_quorum_test_fileinfo_with_metadata(
+        mod_time: OffsetDateTime,
+        data_dir: Uuid,
+        part_etag: &str,
+        erasure_index: usize,
+        extra_metadata: &[(&str, &str)],
+    ) -> FileInfo {
+        let mut fi = quorum_test_fileinfo(mod_time, data_dir, part_etag, erasure_index);
+        for (name, value) in extra_metadata {
+            fi.metadata.insert((*name).to_string(), (*value).to_string());
+        }
+
+        let mut meta = FileMeta::new();
+        meta.add_version(fi).expect("test file metadata should accept object version");
+        let encoded = meta.marshal_msg().expect("test file metadata should marshal");
+        rustfs_filemeta::get_file_info(
+            &encoded,
+            "bucket",
+            "object",
+            "",
+            rustfs_filemeta::FileInfoOpts {
+                data: false,
+                include_free_versions: false,
+            },
+        )
+        .expect("test file metadata should decode as file info")
+    }
+
     #[test]
     fn test_find_file_info_in_quorum_uses_part_identity() {
         let mod_time = OffsetDateTime::now_utc();
@@ -8096,6 +8132,319 @@ mod tests {
         assert_eq!(err, DiskError::ErasureReadQuorum);
     }
 
+    #[test]
+    fn test_latest_fileinfo_selection_quorum_requires_write_quorum_when_full_metadata_is_available() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let metas = vec![
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 1),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 2),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-b", 3),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-b", 4),
+        ];
+        let errs = vec![None, None, None, None];
+
+        let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
+
+        assert_eq!(quorum, 3);
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_quorum_preserves_read_quorum_for_version_or_degraded_reads() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let metas = vec![
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 1),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 2),
+            FileInfo::default(),
+            FileInfo::default(),
+        ];
+        let degraded_errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
+        let clean_errs = vec![None, None, None, None];
+
+        assert_eq!(SetDisks::latest_fileinfo_selection_quorum("", &metas, &degraded_errs, 2, 3), 2);
+        assert_eq!(SetDisks::latest_fileinfo_selection_quorum("version-id", &metas, &clean_errs, 2, 3), 2);
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_quorum_keeps_read_quorum_for_partial_overwrite_with_read_error() {
+        let old_mod_time = OffsetDateTime::now_utc();
+        let new_mod_time = old_mod_time + time::Duration::seconds(1);
+        let old_data_dir = Uuid::new_v4();
+        let new_data_dir = Uuid::new_v4();
+        let metas = vec![
+            quorum_test_fileinfo(old_mod_time, old_data_dir, "part-etag-old", 1),
+            quorum_test_fileinfo(old_mod_time, old_data_dir, "part-etag-old", 2),
+            quorum_test_fileinfo(new_mod_time, new_data_dir, "part-etag-new", 3),
+            FileInfo::default(),
+        ];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
+        let (online_disks, mod_time, etag) = SetDisks::list_online_disks(&vec![None; metas.len()], &metas, &errs, quorum);
+        let fi = SetDisks::pick_valid_fileinfo(&metas, mod_time, etag, quorum)
+            .expect("old metadata should remain readable with read quorum");
+
+        assert_eq!(quorum, 2);
+        assert_eq!(online_disks.len(), metas.len());
+        assert_eq!(fi.data_dir, Some(old_data_dir));
+        assert_eq!(fi.parts[0].etag, "part-etag-old");
+
+        let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
+            .expect("old metadata should remain selectable with read quorum");
+        assert_eq!(selected_quorum, 2);
+        assert_eq!(selected.data_dir, Some(old_data_dir));
+        assert_eq!(selected.parts[0].etag, "part-etag-old");
+        assert!(selected.is_latest);
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_rejects_partial_latest_read_quorum_with_read_error() {
+        let old_mod_time = OffsetDateTime::now_utc();
+        let new_mod_time = old_mod_time + time::Duration::seconds(1);
+        let old_data_dir = Uuid::new_v4();
+        let new_data_dir = Uuid::new_v4();
+        let metas = vec![
+            quorum_test_fileinfo(new_mod_time, new_data_dir, "part-etag-new", 1),
+            quorum_test_fileinfo(new_mod_time, new_data_dir, "part-etag-new", 2),
+            quorum_test_fileinfo(old_mod_time, old_data_dir, "part-etag-old", 3),
+            FileInfo::default(),
+        ];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
+
+        assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_preserves_degraded_read_quorum_without_competing_latest() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let metas = vec![
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 2),
+            FileInfo::default(),
+            FileInfo::default(),
+        ];
+        let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
+
+        let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
+            .expect("read quorum should remain enough when no competing latest is visible");
+
+        assert_eq!(selected_quorum, 2);
+        assert_eq!(selected.data_dir, Some(data_dir));
+        assert_eq!(selected.parts[0].etag, "part-etag-old");
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_ignores_derived_version_stack_drift() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let mut latest_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
+        latest_meta.is_latest = true;
+        latest_meta.num_versions = 1;
+
+        let mut stale_stack_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
+        stale_stack_meta.is_latest = false;
+        stale_stack_meta.successor_mod_time = Some(mod_time + time::Duration::seconds(1));
+        stale_stack_meta.num_versions = 2;
+
+        let mut newer_stack_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
+        newer_stack_meta.is_latest = false;
+        newer_stack_meta.successor_mod_time = Some(mod_time + time::Duration::seconds(2));
+        newer_stack_meta.num_versions = 3;
+
+        let metas = vec![latest_meta, stale_stack_meta, newer_stack_meta, FileInfo::default()];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
+            .expect("same object version should stay readable despite derived version stack drift");
+
+        assert_eq!(selected_quorum, 3);
+        assert_eq!(selected.data_dir, Some(data_dir));
+        assert_eq!(selected.parts[0].etag, "part-etag");
+        assert_eq!(selected.mod_time, Some(mod_time));
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_uses_successor_mod_time_quorum_for_latest_flag() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let mut stale_stack_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
+        stale_stack_meta.is_latest = false;
+        stale_stack_meta.successor_mod_time = Some(mod_time + time::Duration::seconds(1));
+        stale_stack_meta.num_versions = 2;
+
+        let mut latest_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
+        latest_meta_a.is_latest = true;
+        latest_meta_a.num_versions = 1;
+        let mut latest_meta_b = latest_meta_a.clone();
+        latest_meta_b.erasure.index = 3;
+
+        let metas = vec![stale_stack_meta, latest_meta_a, latest_meta_b];
+
+        let selected = SetDisks::find_file_info_in_quorum(&metas, &Some(mod_time), &None, 2)
+            .expect("latest flag should be derived from successor mod time quorum");
+
+        assert!(selected.is_latest);
+        assert_eq!(selected.successor_mod_time, None);
+        assert_eq!(selected.num_versions, 1);
+        assert_eq!(selected.mod_time, Some(mod_time));
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_ignores_replication_state_drift() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let replication_status_key = format!(
+            "{}{}",
+            rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
+            rustfs_utils::http::SUFFIX_REPLICATION_STATUS
+        );
+        let replication_timestamp_key = format!(
+            "{}{}",
+            rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
+            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
+        );
+        let replication_reset_key = format!(
+            "{}{}target-a",
+            rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
+            rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX
+        );
+        let meta_a = decoded_quorum_test_fileinfo_with_metadata(
+            mod_time,
+            data_dir,
+            "part-etag",
+            1,
+            &[
+                (&replication_status_key, "target-a=COMPLETED;"),
+                (&replication_timestamp_key, "2024-01-01T00:00:00Z"),
+                (&replication_reset_key, "COMPLETED"),
+            ],
+        );
+        let meta_b = decoded_quorum_test_fileinfo_with_metadata(
+            mod_time,
+            data_dir,
+            "part-etag",
+            2,
+            &[
+                (&replication_status_key, "target-a=PENDING;"),
+                (&replication_timestamp_key, "2024-01-01T00:00:01Z"),
+                (&replication_reset_key, "PENDING"),
+            ],
+        );
+        let meta_c = decoded_quorum_test_fileinfo_with_metadata(
+            mod_time,
+            data_dir,
+            "part-etag",
+            3,
+            &[
+                (&replication_status_key, "target-a=FAILED;"),
+                (&replication_timestamp_key, "2024-01-01T00:00:02Z"),
+                (&replication_reset_key, "FAILED"),
+            ],
+        );
+        assert!(meta_a.replication_state_internal.is_some());
+        assert_eq!(
+            meta_a
+                .metadata
+                .get(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)
+                .map(String::as_str),
+            Some("COMPLETED")
+        );
+
+        let metas = vec![meta_a, meta_b, meta_c, FileInfo::default()];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
+            .expect("replication status drift should not split readable object identity");
+
+        assert_eq!(selected_quorum, 3);
+        assert_eq!(selected.data_dir, Some(data_dir));
+        assert_eq!(selected.parts[0].etag, "part-etag");
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_rejects_same_modtime_metadata_split_without_write_quorum() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let mut old_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
+        let mut old_meta_b = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
+        let mut partial_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
+        old_meta_a.metadata.insert("x-amz-meta-color".to_string(), "blue".to_string());
+        old_meta_b.metadata.insert("x-amz-meta-color".to_string(), "blue".to_string());
+        partial_meta
+            .metadata
+            .insert("x-amz-meta-color".to_string(), "red".to_string());
+        let metas = vec![old_meta_a, old_meta_b, partial_meta, FileInfo::default()];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
+        let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
+
+        assert_eq!(quorum, 2);
+        assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_rejects_same_modtime_partial_metadata_read_quorum() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let mut old_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
+        let mut partial_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
+        let mut partial_meta_b = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
+        old_meta.metadata.insert("x-amz-meta-color".to_string(), "blue".to_string());
+        partial_meta_a
+            .metadata
+            .insert("x-amz-meta-color".to_string(), "red".to_string());
+        partial_meta_b
+            .metadata
+            .insert("x-amz-meta-color".to_string(), "red".to_string());
+        let metas = vec![old_meta, partial_meta_a, partial_meta_b, FileInfo::default()];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
+
+        assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_rejects_same_modtime_transition_split_without_write_quorum() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let old_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
+        let old_meta_b = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
+        let mut partial_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
+        partial_meta.transition_status = TRANSITION_COMPLETE.to_string();
+        partial_meta.transition_tier = "WARM".to_string();
+        partial_meta.transitioned_objname = "remote/object".to_string();
+        partial_meta.transition_version_id = Some(Uuid::new_v4());
+        let metas = vec![old_meta_a, old_meta_b, partial_meta, FileInfo::default()];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
+        let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
+
+        assert_eq!(quorum, 2);
+        assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
+    }
+
+    #[test]
+    fn test_latest_fileinfo_selection_quorum_uses_write_quorum_for_degraded_committed_identity() {
+        let mod_time = OffsetDateTime::now_utc();
+        let data_dir = Uuid::new_v4();
+        let metas = vec![
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 1),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 2),
+            quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 3),
+            FileInfo::default(),
+        ];
+        let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
+
+        assert_eq!(SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3), 3);
+    }
+
     #[test]
     fn test_list_object_parities() {
         // Test extracting parity counts from file info

crates/ecstore/src/set_disk/read.rs

@@ -2362,7 +2362,7 @@ impl SetDisks {
         let _min_disks = self.set_drive_count - self.default_parity_count;
 
         let metadata_resolve_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
-        let (read_quorum, _) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
+        let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
             .map_err(|err| to_object_err(err.into(), vec![bucket, object]))
         {
             Ok(v) => v,
@@ -2378,8 +2378,8 @@ impl SetDisks {
         };
         let read_quorum =
             usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
-        metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, read_quorum);
-
+        let write_quorum = usize::try_from(write_quorum)
+            .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
             error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
             record_get_stage_duration_if_enabled(
@@ -2390,9 +2390,9 @@ impl SetDisks {
             return Err(to_object_err(err.into(), vec![bucket, object]));
         }
 
-        let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum);
-
-        let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum)?;
+        let (op_online_disks, fi, fileinfo_selection_quorum) =
+            Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
+        metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, fileinfo_selection_quorum);
         if errs.iter().any(|err| err.is_some()) {
             let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
             submit_read_repair_heal(