fix(storage): address pending metadata and health gaps (#4380)

crates/ecstore/src/bucket/metadata_sys.rs

@@ -39,19 +39,26 @@ use std::{collections::HashMap, sync::Arc};
 use time::OffsetDateTime;
 use tokio::sync::RwLock;
 use tokio::time::sleep;
-use tracing::error;
+use tokio_util::sync::CancellationToken;
+use tracing::{error, warn};
 
 lazy_static! {
     pub static ref GLOBAL_BUCKET_METADATA_SYS: OnceLock<Arc<RwLock<BucketMetadataSys>>> = OnceLock::new();
 }
 
+const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
+
 pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
     let mut sys = BucketMetadataSys::new(api);
     sys.init(buckets).await;
 
     let sys = Arc::new(RwLock::new(sys));
 
-    GLOBAL_BUCKET_METADATA_SYS.set(sys).unwrap();
+    GLOBAL_BUCKET_METADATA_SYS.set(sys.clone()).unwrap();
+
+    if runtime_sources::setup_is_dist_erasure().await {
+        start_refresh_buckets_metadata_loop(sys);
+    }
 }
 
 pub fn get_global_bucket_metadata_sys() -> Option<Arc<RwLock<BucketMetadataSys>>> {
@@ -85,6 +92,67 @@ pub async fn remove_bucket_metadata(bucket: &str) -> Result<bool> {
     Ok(lock.remove(bucket).await)
 }
 
+fn start_refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>) {
+    let Some(cancel_token) = runtime_sources::background_services_cancel_token().cloned() else {
+        warn!("bucket metadata refresh loop skipped because background cancellation token is not initialized");
+        return;
+    };
+
+    tokio::spawn(async move {
+        refresh_buckets_metadata_loop(sys, cancel_token).await;
+    });
+}
+
+async fn refresh_buckets_metadata_loop(sys: Arc<RwLock<BucketMetadataSys>>, cancel_token: CancellationToken) {
+    loop {
+        if !wait_refresh_interval_or_cancel(&cancel_token, BUCKET_METADATA_REFRESH_INTERVAL).await {
+            break;
+        }
+        refresh_buckets_metadata_once(sys.clone()).await;
+    }
+}
+
+async fn wait_refresh_interval_or_cancel(cancel_token: &CancellationToken, interval: Duration) -> bool {
+    tokio::select! {
+        _ = cancel_token.cancelled() => false,
+        _ = sleep(interval) => true,
+    }
+}
+
+async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
+    let buckets = {
+        let sys = sys.read().await;
+        sys.bucket_names().await
+    };
+    if buckets.is_empty() {
+        return;
+    }
+
+    let count = runtime_sources::endpoint_erasure_set_count()
+        .map(|count| count * 10)
+        .unwrap_or(10)
+        .max(1);
+    let mut failed_buckets = HashSet::new();
+
+    for chunk in buckets.chunks(count) {
+        let sys = sys.read().await;
+        sys.concurrent_load(chunk, &mut failed_buckets).await;
+    }
+
+    if !failed_buckets.is_empty() {
+        warn!(
+            failed_bucket_count = failed_buckets.len(),
+            "bucket metadata refresh loop left buckets queued for retry"
+        );
+    }
+}
+
+async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
+    BucketTargetSys::get()
+        .update_all_targets(bucket, bm.bucket_target_config.as_ref())
+        .await;
+}
+
 pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
     let sys = get_bucket_metadata_sys()?;
     let lock = sys.read().await;
@@ -295,10 +363,6 @@ impl BucketMetadataSys {
         let mut initialized = self.initialized.write().await;
         *initialized = true;
 
-        if runtime_sources::setup_is_dist_erasure().await {
-            // TODO: refresh_buckets_metadata_loop
-        }
-
         Ok(())
     }
 
@@ -325,18 +389,11 @@ impl BucketMetadataSys {
 
         let results = join_all(futures).await;
 
-        let mut idx = 0;
-
-        let mut mp = self.metadata_map.write().await;
-
-        // TODO: EventNotifier
-        for res in results {
+        for (idx, res) in results.into_iter().enumerate() {
             match res {
                 Ok(res) => {
                     if let Some(bucket) = buckets.get(idx) {
-                        let x = Arc::new(res);
-                        mp.insert(bucket.clone(), x.clone());
-                        BucketTargetSys::get().set(bucket, &x).await;
+                        self.set(bucket.clone(), Arc::new(res)).await;
                     }
                 }
                 Err(e) => {
@@ -346,8 +403,6 @@ impl BucketMetadataSys {
                     }
                 }
             }
-
-            idx += 1;
         }
     }
 
@@ -367,7 +422,9 @@ impl BucketMetadataSys {
     pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
         if !is_meta_bucketname(&bucket) {
             let mut map = self.metadata_map.write().await;
-            map.insert(bucket, bm);
+            map.insert(bucket.clone(), bm.clone());
+            drop(map);
+            sync_bucket_target_sys(&bucket, &bm).await;
         }
     }
 
@@ -379,7 +436,12 @@ impl BucketMetadataSys {
             return false;
         }
         let mut map = self.metadata_map.write().await;
-        map.remove(bucket).is_some()
+        let removed = map.remove(bucket).is_some();
+        drop(map);
+        if removed {
+            BucketTargetSys::get().delete(bucket).await;
+        }
+        removed
     }
 
     async fn _reset(&mut self) {
@@ -466,6 +528,10 @@ impl BucketMetadataSys {
         Ok(())
     }
 
+    async fn bucket_names(&self) -> Vec<String> {
+        self.metadata_map.read().await.keys().cloned().collect()
+    }
+
     pub async fn get_config_from_disk(&self, bucket: &str) -> Result<BucketMetadata> {
         if is_meta_bucketname(bucket) {
             return Err(Error::other("errInvalidArgument"));
@@ -498,6 +564,8 @@ impl BucketMetadataSys {
 
             let bm = Arc::new(bm);
             map.insert(bucket.to_string(), bm.clone());
+            drop(map);
+            sync_bucket_target_sys(bucket, &bm).await;
 
             Ok((bm, true))
         }
@@ -697,13 +765,9 @@ impl BucketMetadataSys {
     }
 
     pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
-        let (bm, reload) = self.get_config(bucket).await?;
+        let (bm, _) = self.get_config(bucket).await?;
 
         if let Some(config) = &bm.replication_config {
-            if reload {
-                // TODO: globalBucketTargetSys
-            }
-            //println!("549 {:?}", config.clone());
             Ok((config.clone(), bm.replication_config_updated_at))
         } else {
             Err(Error::ConfigNotFound)
@@ -711,17 +775,95 @@ impl BucketMetadataSys {
     }
 
     pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
-        let (bm, reload) = self.get_config(bucket).await?;
+        let (bm, _) = self.get_config(bucket).await?;
 
         if let Some(config) = &bm.bucket_target_config {
-            if reload {
-                // TODO: globalBucketTargetSys
-                //config.
-            }
-
             Ok(config.clone())
         } else {
             Err(Error::ConfigNotFound)
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
+    use serial_test::serial;
+    use tokio::time::timeout;
+
+    fn target(bucket: &str, id: &str) -> BucketTarget {
+        BucketTarget {
+            source_bucket: bucket.to_string(),
+            endpoint: format!("{id}.example.com:9000"),
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                ..Default::default()
+            }),
+            target_bucket: format!("{bucket}-{id}"),
+            arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
+            target_type: BucketTargetType::ReplicationService,
+            region: "us-east-1".to_string(),
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn metadata_reload_syncs_bucket_target_sys() {
+        let bucket = "metadata-reload-targets";
+        let target_sys = BucketTargetSys::get();
+        target_sys.delete(bucket).await;
+
+        let mut bm = BucketMetadata::new(bucket);
+        bm.bucket_target_config = Some(BucketTargets {
+            targets: vec![target(bucket, "fresh")],
+        });
+
+        sync_bucket_target_sys(bucket, &bm).await;
+
+        let targets = target_sys
+            .list_bucket_targets(bucket)
+            .await
+            .expect("target sync should publish bucket targets");
+        assert_eq!(targets.targets.len(), 1);
+        assert_eq!(targets.targets[0].arn, format!("arn:rustfs:replication:us-east-1:{bucket}:fresh"));
+
+        target_sys.delete(bucket).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() {
+        let bucket = "metadata-clear-targets";
+        let target_sys = BucketTargetSys::get();
+        target_sys.delete(bucket).await;
+        target_sys
+            .targets_map
+            .write()
+            .await
+            .insert(bucket.to_string(), vec![target(bucket, "stale")]);
+
+        let bm = BucketMetadata::new(bucket);
+        sync_bucket_target_sys(bucket, &bm).await;
+
+        assert!(target_sys.list_bucket_targets(bucket).await.is_err());
+        target_sys.delete(bucket).await;
+    }
+
+    #[tokio::test]
+    async fn refresh_wait_exits_when_cancelled() {
+        let cancel_token = CancellationToken::new();
+        cancel_token.cancel();
+
+        let should_refresh = timeout(
+            Duration::from_millis(100),
+            wait_refresh_interval_or_cancel(&cancel_token, Duration::from_secs(60)),
+        )
+        .await
+        .expect("cancelled refresh wait should not sleep until the interval");
+
+        assert!(!should_refresh);
+    }
+}

crates/ecstore/src/diagnostics/admin_server_info.rs

@@ -21,7 +21,7 @@ use crate::data_usage::load_data_usage_cache;
 use crate::storage_api_contracts::admin::StorageAdminApi;
 use rustfs_common::heal_channel::DriveState;
 use rustfs_madmin::{
-    BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties,
+    BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, MemStats, ServerProperties,
 };
 use rustfs_protos::{
     models::{PingBody, PingBodyBuilder},
@@ -128,10 +128,22 @@ pub async fn get_local_server_property() -> ServerProperties {
     let addr = runtime_sources::local_node_name().await;
     let mut pool_numbers = HashSet::new();
     let mut network = HashMap::new();
+    let (mem_stats, max_procs, num_cpu) = collect_runtime_server_stats();
 
     let endpoints = match runtime_sources::endpoint_pools() {
         Some(eps) => eps,
-        None => return ServerProperties::default(),
+        None => {
+            return ServerProperties {
+                state: ITEM_INITIALIZING.to_string(),
+                endpoint: addr,
+                uptime: runtime_sources::boot_uptime_secs(),
+                version: get_commit_id(),
+                mem_stats,
+                max_procs,
+                num_cpu,
+                ..Default::default()
+            };
+        }
     };
     for ep in endpoints.as_ref().iter() {
         for endpoint in ep.endpoints.as_ref().iter() {
@@ -154,14 +166,14 @@ pub async fn get_local_server_property() -> ServerProperties {
         }
     }
 
-    // todo: mem collect
-    // let mem_stats =
-
     let mut props = ServerProperties {
         endpoint: addr,
         uptime: runtime_sources::boot_uptime_secs(),
         network,
         version: get_commit_id(),
+        mem_stats,
+        max_procs,
+        num_cpu,
         ..Default::default()
     };
 
@@ -189,6 +201,15 @@ pub async fn get_local_server_property() -> ServerProperties {
     props
 }
 
+fn collect_runtime_server_stats() -> (MemStats, u64, u64) {
+    let num_cpu = u64::try_from(num_cpus::get()).unwrap_or(u64::MAX);
+    let max_procs = std::thread::available_parallelism()
+        .map(|parallelism| u64::try_from(parallelism.get()).unwrap_or(u64::MAX))
+        .unwrap_or(num_cpu.max(1));
+
+    (rustfs_madmin::health::collect_mem_stats(), max_procs, num_cpu)
+}
+
 pub async fn get_server_info(get_pools: bool) -> InfoMessage {
     let nowt: OffsetDateTime = OffsetDateTime::now_utc();
 
@@ -393,7 +414,7 @@ mod tests {
 
     use crate::runtime::sources as runtime_sources;
 
-    use super::get_server_info;
+    use super::{get_local_server_property, get_server_info};
 
     #[serial]
     #[tokio::test]
@@ -403,4 +424,17 @@ mod tests {
 
         assert_eq!(info.deployment_id, expected_deployment_id);
     }
+
+    #[serial]
+    #[tokio::test]
+    async fn local_server_property_includes_runtime_stats_without_endpoint_pools() {
+        let props = get_local_server_property().await;
+
+        assert!(props.num_cpu > 0);
+        assert!(props.max_procs > 0);
+        assert!(
+            props.mem_stats.alloc > 0 || props.mem_stats.total_alloc > 0 || props.mem_stats.heap_alloc > 0,
+            "memory stats should not remain fixed placeholders"
+        );
+    }
 }

crates/ecstore/src/disk/local.rs

@@ -36,7 +36,7 @@ use crate::erasure::coding::bitrot_verify;
 use crate::runtime::sources as runtime_sources;
 use bytes::Bytes;
 use metrics::counter;
-use parking_lot::RwLock as ParkingLotRwLock;
+use parking_lot::{Mutex as ParkingLotMutex, RwLock as ParkingLotRwLock};
 use rustfs_filemeta::{
     Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
     get_file_info, read_xl_meta_no_data,
@@ -1461,6 +1461,7 @@ pub struct LocalDisk {
     pub major: u64,
     pub minor: u64,
     pub nrrequests: u64,
+    scan_locks: Arc<ParkingLotMutex<HashSet<(String, String)>>>,
     // Performance optimization fields
     path_cache: Arc<ParkingLotRwLock<HashMap<String, PathBuf>>>,
     current_dir: Arc<OnceLock<PathBuf>>,
@@ -1698,6 +1699,7 @@ impl LocalDisk {
             minor: Default::default(),
             major: Default::default(),
             nrrequests: Default::default(),
+            scan_locks: Arc::new(ParkingLotMutex::new(HashSet::new())),
             // // format_legacy,
             // format_file_info: Mutex::new(format_meta),
             // format_data: Mutex::new(format_data),
@@ -2060,6 +2062,20 @@ impl LocalDisk {
         reject_local_disk_symlink_components(&self.root, path)
     }
 
+    fn try_acquire_scan_lock(&self, opts: &WalkDirOptions) -> Result<LocalScanLockGuard> {
+        let key = local_disk_scan_lock_key(&opts.bucket, &opts.base_dir, opts.filter_prefix.as_deref());
+        let mut scan_locks = self.scan_locks.lock();
+
+        if !scan_locks.insert(key.clone()) {
+            return Err(DiskError::DiskOngoingReq);
+        }
+
+        Ok(LocalScanLockGuard {
+            scan_locks: Arc::clone(&self.scan_locks),
+            key,
+        })
+    }
+
     // Batch path generation with single lock acquisition
     fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
         let mut results = Vec::with_capacity(requests.len());
@@ -3015,6 +3031,34 @@ impl Drop for ScanGuard {
     }
 }
 
+struct LocalScanLockGuard {
+    scan_locks: Arc<ParkingLotMutex<HashSet<(String, String)>>>,
+    key: (String, String),
+}
+
+impl Drop for LocalScanLockGuard {
+    fn drop(&mut self) {
+        self.scan_locks.lock().remove(&self.key);
+    }
+}
+
+fn local_disk_scan_lock_key(bucket: &str, base_dir: &str, filter_prefix: Option<&str>) -> (String, String) {
+    let mut prefix = base_dir.trim_matches('/').to_owned();
+    if let Some(filter_prefix) = filter_prefix
+        .map(|prefix| prefix.trim_matches('/'))
+        .filter(|prefix| !prefix.is_empty())
+    {
+        if prefix.is_empty() {
+            prefix.push_str(filter_prefix);
+        } else {
+            prefix.push_str(SLASH_SEPARATOR);
+            prefix.push_str(filter_prefix);
+        }
+    }
+
+    (bucket.to_owned(), prefix)
+}
+
 fn is_root_path(path: impl AsRef<Path>) -> bool {
     path.as_ref().components().count() == 1 && path.as_ref().has_root()
 }
@@ -3850,6 +3894,8 @@ impl DiskAPI for LocalDisk {
             return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
         }
 
+        let _scan_lock = self.try_acquire_scan_lock(&opts)?;
+
         let mut wr = wr;
 
         let mut out = MetacacheWriter::new(&mut wr);
@@ -4828,7 +4874,7 @@ mod test {
     use std::io;
     use std::pin::Pin;
     use std::task::{Context, Poll};
-    use tokio::io::{AsyncReadExt, ReadBuf};
+    use tokio::io::{AsyncReadExt, AsyncWrite, ReadBuf};
 
     fn test_file_info(name: &str, version_id: Uuid, data_dir: Option<Uuid>, data: Option<Bytes>) -> FileInfo {
         let size = data
@@ -4859,6 +4905,82 @@ mod test {
         }
     }
 
+    struct BlockingScanWriter {
+        entered_tx: Option<tokio::sync::oneshot::Sender<()>>,
+    }
+
+    impl AsyncWrite for BlockingScanWriter {
+        fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
+            if let Some(tx) = self.entered_tx.take() {
+                let _ = tx.send(());
+            }
+            Poll::Pending
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Pending
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Pending
+        }
+    }
+
+    #[tokio::test]
+    async fn test_local_disk_scan_rejects_concurrent_same_prefix_and_releases_on_cancel() {
+        use tempfile::tempdir;
+
+        let dir = tempdir().expect("temp dir should be created");
+        let bucket = "test-bucket";
+        let object_dir = dir.path().join(bucket).join("prefix/object");
+        fs::create_dir_all(&object_dir).await.expect("object dir should be created");
+        fs::write(object_dir.join(STORAGE_FORMAT_FILE), b"meta")
+            .await
+            .expect("object metadata should be written");
+
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
+        let opts = WalkDirOptions {
+            bucket: bucket.to_string(),
+            base_dir: "prefix/".to_string(),
+            recursive: true,
+            ..Default::default()
+        };
+
+        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
+        let first_disk = Arc::clone(&disk);
+        let first_opts = opts.clone();
+        let mut blocking_writer = BlockingScanWriter {
+            entered_tx: Some(entered_tx),
+        };
+        let first_scan = tokio::spawn(async move { first_disk.walk_dir(first_opts, &mut blocking_writer).await });
+
+        entered_rx.await.expect("first scan should enter write path");
+
+        let mut second_writer = tokio::io::sink();
+        let second_scan = disk.walk_dir(opts.clone(), &mut second_writer).await;
+        assert!(
+            matches!(second_scan, Err(DiskError::DiskOngoingReq)),
+            "concurrent scan of same bucket and prefix must be rejected, got {second_scan:?}"
+        );
+
+        first_scan.abort();
+        assert!(
+            first_scan
+                .await
+                .expect_err("first scan task should be cancelled")
+                .is_cancelled(),
+            "aborting the blocked scan should cancel the task"
+        );
+
+        let mut after_cancel_writer = tokio::io::sink();
+        let after_cancel = disk.walk_dir(opts, &mut after_cancel_writer).await;
+        assert!(
+            after_cancel.is_ok(),
+            "cancelled scan must release the bucket/prefix lock, got {after_cancel:?}"
+        );
+    }
+
     #[tokio::test]
     async fn test_rename_data_writes_old_metadata_backup_before_non_inline_undo() {
         use tempfile::tempdir;

crates/ecstore/src/set_disk/metadata.rs

@@ -235,7 +235,8 @@ impl SetDisks {
 
     pub(super) fn list_object_parities(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<i32> {
         let total_shards = parts_metadata.len();
-        let half = total_shards as i32 / 2;
+        let total_shards_i32 = i32::try_from(total_shards).unwrap_or(i32::MAX);
+        let half = total_shards_i32 / 2;
         let mut parities: Vec<i32> = vec![-1; total_shards];
 
         for (index, metadata) in parts_metadata.iter().enumerate() {
@@ -251,11 +252,12 @@ impl SetDisks {
 
             if metadata.deleted || metadata.size == 0 {
                 parities[index] = half;
-            // } else if metadata.transition_status == "TransitionComplete" {
-            // TODO: metadata.transition_status
-            //     parities[index] = total_shards - (total_shards / 2 + 1);
+            } else if metadata.transition_status == TRANSITION_COMPLETE {
+                let majority_metadata_parity = total_shards_i32 - (half + 1);
+                let erasure_parity = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(i32::MAX);
+                parities[index] = majority_metadata_parity.max(erasure_parity);
             } else {
-                parities[index] = metadata.erasure.parity_blocks as i32;
+                parities[index] = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(i32::MAX);
             }
         }
         parities
@@ -502,6 +504,12 @@ impl SetDisks {
         }
     }
 
+    fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
+        meta.metadata
+            .keys()
+            .any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
+    }
+
     fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
         value
             .get(..prefix.len())
@@ -560,6 +568,11 @@ impl SetDisks {
         hasher.update(meta.size.to_le_bytes());
         hasher.update([u8::from(meta.deleted), u8::from(meta.mark_deleted)]);
         hasher.update([u8::from(meta.expire_restored)]);
+        hasher.update([
+            u8::from(meta.is_remote()),
+            u8::from(Self::file_info_has_encryption_metadata(meta)),
+            u8::from(meta.is_compressed()),
+        ]);
         Self::update_hash_optional_time(hasher, meta.mod_time);
         Self::update_hash_str(hasher, &meta.transition_status);
         Self::update_hash_str(hasher, &meta.transition_tier);
@@ -747,14 +760,6 @@ impl SetDisks {
             let mod_valid = mod_time == &meta.mod_time;
 
             if etag_only || mod_valid {
-                if meta.is_remote() {
-                    // TODO:
-                }
-
-                // TODO: IsEncrypted
-
-                // TODO: IsCompressed
-
                 meta_hashes[i] = Some(Self::file_info_quorum_hash(meta));
             } else {
                 debug!(
@@ -1003,6 +1008,104 @@ impl SetDisks {
 mod tests {
     use super::*;
 
+    fn metadata_quorum_test_fileinfo(mod_time: OffsetDateTime, erasure_index: usize) -> FileInfo {
+        let mut fi = FileInfo::new("bucket/object", 2, 2);
+        fi.name = "bucket/object".to_string();
+        fi.size = 8 * 1024 * 1024;
+        fi.mod_time = Some(mod_time);
+        fi.data_dir = Some(Uuid::new_v4());
+        fi.metadata.insert("etag".to_string(), "object-etag".to_string());
+        fi.add_object_part(1, "part-etag".to_string(), 8 * 1024 * 1024, Some(mod_time), 8 * 1024 * 1024, None, None);
+        fi.erasure.index = erasure_index;
+        fi
+    }
+
+    fn transition_metadata_quorum_fileinfo(erasure_index: usize) -> FileInfo {
+        let mut fi = FileInfo::new("bucket/object", 5, 1);
+        fi.name = "bucket/object".to_string();
+        fi.size = 8 * 1024 * 1024;
+        fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"));
+        fi.metadata.insert("etag".to_string(), "object-etag".to_string());
+        fi.transition_status = TRANSITION_COMPLETE.to_string();
+        fi.transition_tier = "WARM".to_string();
+        fi.transitioned_objname = "remote/object".to_string();
+        fi.transition_version_id = Some(Uuid::new_v4());
+        fi.erasure.index = erasure_index;
+        fi
+    }
+
+    fn expect_metadata_quorum_error(metas: Vec<FileInfo>, mod_time: OffsetDateTime, message: &str) {
+        let err = SetDisks::find_file_info_in_quorum(&metas, &Some(mod_time), &None, 3).expect_err(message);
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
+
+    #[test]
+    fn metadata_quorum_uses_simple_majority_for_transitioned_objects() {
+        let parts_metadata = (1..=6).map(transition_metadata_quorum_fileinfo).collect::<Vec<_>>();
+        let errs = vec![None; parts_metadata.len()];
+
+        let parities = SetDisks::list_object_parities(&parts_metadata, &errs);
+        let quorum = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, 1)
+            .expect("transitioned metadata should resolve object quorum");
+
+        assert_eq!(parities, vec![2; 6]);
+        assert_eq!(quorum, (4, 4));
+    }
+
+    #[test]
+    fn find_file_info_in_quorum_rejects_encrypted_plain_metadata_split() {
+        let mod_time = OffsetDateTime::now_utc();
+        let mut encrypted_a = metadata_quorum_test_fileinfo(mod_time, 1);
+        let mut encrypted_b = metadata_quorum_test_fileinfo(mod_time, 2);
+        let plain = metadata_quorum_test_fileinfo(mod_time, 3);
+        encrypted_a
+            .metadata
+            .insert("x-rustfs-encryption-key".to_string(), "encrypted-key".to_string());
+        encrypted_b
+            .metadata
+            .insert("x-rustfs-encryption-key".to_string(), "encrypted-key".to_string());
+
+        expect_metadata_quorum_error(
+            vec![encrypted_a, encrypted_b, plain],
+            mod_time,
+            "mixed encrypted and plain metadata must not reach quorum",
+        );
+    }
+
+    #[test]
+    fn find_file_info_in_quorum_rejects_compressed_plain_metadata_split() {
+        let mod_time = OffsetDateTime::now_utc();
+        let mut compressed_a = metadata_quorum_test_fileinfo(mod_time, 1);
+        let mut compressed_b = metadata_quorum_test_fileinfo(mod_time, 2);
+        let plain = metadata_quorum_test_fileinfo(mod_time, 3);
+        http::insert_str(&mut compressed_a.metadata, http::SUFFIX_COMPRESSION, "lz4".to_string());
+        http::insert_str(&mut compressed_b.metadata, http::SUFFIX_COMPRESSION, "lz4".to_string());
+
+        expect_metadata_quorum_error(
+            vec![compressed_a, compressed_b, plain],
+            mod_time,
+            "mixed compressed and plain metadata must not reach quorum",
+        );
+    }
+
+    #[test]
+    fn find_file_info_in_quorum_rejects_remote_local_metadata_split() {
+        let mod_time = OffsetDateTime::now_utc();
+        let mut remote = metadata_quorum_test_fileinfo(mod_time, 1);
+        let local_a = metadata_quorum_test_fileinfo(mod_time, 2);
+        let local_b = metadata_quorum_test_fileinfo(mod_time, 3);
+        remote.transition_status = TRANSITION_COMPLETE.to_string();
+        remote.transition_tier = "WARM".to_string();
+        remote.transitioned_objname = "remote/object".to_string();
+        remote.transition_version_id = Some(Uuid::new_v4());
+
+        expect_metadata_quorum_error(
+            vec![remote, local_a, local_b],
+            mod_time,
+            "mixed remote and local metadata must not reach quorum",
+        );
+    }
+
     async fn shuffle_test_disks(tempdir: &tempfile::TempDir, count: usize) -> Vec<Option<DiskStore>> {
         let endpoint =
             Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");