diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index 6dcd1745d..eff552c0a 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d -sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926 +sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81 +sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json index 55cd0a85e..07321d052 100644 --- a/.config/ecstore-required-tests.json +++ b/.config/ecstore-required-tests.json @@ -29,7 +29,7 @@ { "invariant": "list-uncommitted-version", "suite": "rustfs-filemeta", - "name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge" + "name": "metacache::tests::resolve_merge_keeps_the_newest_version_at_listing_quorum" }, { "invariant": "minio-object-fixture", diff --git a/crates/e2e_test/src/degraded_listing_availability_test.rs b/crates/e2e_test/src/degraded_listing_availability_test.rs index 53929ce4d..6004f36b8 100644 --- a/crates/e2e_test/src/degraded_listing_availability_test.rs +++ b/crates/e2e_test/src/degraded_listing_availability_test.rs @@ -20,15 +20,14 @@ //! offline before heal converges, a strict latest-listing quorum of 3 can only //! ever observe 2 copies, so ListObjectsV2 silently dropped the object even //! though GetObject (read quorum 2) still succeeded. Exposed by the flaky -//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the -//! product fix relaxes the listing's required object quorum by the number of -//! set drives the listing could not consult (see -//! `latest_listing_required_object_quorum` in -//! `crates/ecstore/src/store/list_objects.rs`). +//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853). The +//! listing now follows MinIO's rule: an entry needs `ceil(asked/2)` copies +//! among the online drives it asked, so three online drives need two copies +//! (see `SetDisks::list_path` in `crates/ecstore/src/store/list_objects.rs`). #[cfg(test)] mod tests { - use crate::common::{RustFSTestClusterEnvironment, init_logging}; + use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::Client; use bytes::Bytes; use std::collections::HashSet; @@ -51,13 +50,13 @@ mod tests { format!("degraded-object-{idx:02}") } - async fn list_all_keys(client: &Client) -> Result, Box> { + async fn list_all_keys(client: &Client, bucket: &str) -> Result, Box> { let mut keys = HashSet::new(); let mut continuation_token: Option = None; loop { let response = client .list_objects_v2() - .bucket(BUCKET) + .bucket(bucket) .set_continuation_token(continuation_token.clone()) .send() .await?; @@ -147,7 +146,7 @@ mod tests { let expected: HashSet = (0..OBJECT_COUNT).map(object_key).collect(); let deadline = Instant::now() + LISTING_DEADLINE; let listed = loop { - let listed = match list_all_keys(&client).await { + let listed = match list_all_keys(&client, BUCKET).await { Ok(keys) => keys, Err(error) if Instant::now() < deadline => { info!("retrying degraded listing: {error}"); @@ -171,4 +170,97 @@ mod tests { Ok(()) } + + /// 4-node x 4-drive cluster (one 16-drive set), #8828 shape: + /// 1. PUT objects while the storage class is EC:8, so each is 8+8. + /// 2. Restart with the EC:4 default, as after a storage-class change. + /// 3. Stop two nodes (8 drives). Each object keeps eight data shards, so + /// GET succeeds and ListObjectsV2 must list every key: the listing + /// quorum is half of the eight drives it can ask. + /// 4. Stop a third node. Four online drives are below the set's read + /// quorum, so LIST must report 503 instead of a partial page. + #[tokio::test] + async fn ec8_objects_stay_listable_with_half_of_an_ec4_set_offline() -> TestResult { + const EC8_BUCKET: &str = "degraded-listing-ec8"; + init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?; + cluster.set_env("RUSTFS_HEAL_ENABLED", "false"); + cluster.set_env("RUSTFS_SCANNER_ENABLED", "false"); + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:8"); + cluster.start().await?; + cluster.create_test_bucket(EC8_BUCKET).await?; + let client = cluster.create_s3_client(0)?; + for idx in 0..OBJECT_COUNT { + let key = object_key(idx); + client + .put_object() + .bucket(EC8_BUCKET) + .key(&key) + .body(Bytes::from(format!("ec8 listing payload {idx}")).into()) + .send() + .await + .map_err(|error| format!("EC:8 PUT for {key} failed: {error}"))?; + } + + info!("restarting with the EC:4 default storage class"); + cluster.stop(); + // A later entry for the same variable wins when the node process starts. + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"); + cluster.start().await?; + let client = cluster.create_s3_client(0)?; + + info!("stopping nodes 2 and 3 (eight of sixteen drives)"); + cluster.stop_node(2)?; + cluster.stop_node(3)?; + for idx in 0..OBJECT_COUNT { + let key = object_key(idx); + let deadline = Instant::now() + GET_RETRY_DEADLINE; + loop { + match client.get_object().bucket(EC8_BUCKET).key(&key).send().await { + Ok(response) => { + let body = response.body.collect().await?.into_bytes(); + assert_eq!(body.as_ref(), format!("ec8 listing payload {idx}").as_bytes()); + break; + } + Err(error) if Instant::now() < deadline => { + info!("retrying EC:8 GET for {key}: {error}"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + Err(error) => return Err(format!("EC:8 object {key} failed GET with eight drives: {error}").into()), + } + } + } + + let expected: HashSet = (0..OBJECT_COUNT).map(object_key).collect(); + let deadline = Instant::now() + LISTING_DEADLINE; + loop { + match list_all_keys(&client, EC8_BUCKET).await { + Ok(listed) if listed == expected => break, + Ok(listed) => assert!( + Instant::now() < deadline, + "EC:8 objects readable with eight drives stayed missing from ListObjectsV2: listed={listed:?}" + ), + Err(error) => assert!( + Instant::now() < deadline, + "ListObjectsV2 kept failing with eight of sixteen drives online: {error}" + ), + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + + info!("stopping node 1 (four of sixteen drives remain)"); + cluster.stop_node(1)?; + let deadline = Instant::now() + LISTING_DEADLINE; + loop { + match client.list_objects_v2().bucket(EC8_BUCKET).send().await { + Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => break, + Err(error) => assert!(Instant::now() < deadline, "below the set read quorum LIST must report 503, got {error}"), + Ok(response) => panic!("below the set read quorum LIST must fail, but listed {} keys", response.contents().len()), + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + + Ok(()) + } } diff --git a/crates/ecstore/src/cache_value/metacache_set.rs b/crates/ecstore/src/cache_value/metacache_set.rs index 5dc3b3668..0afecc7a3 100644 --- a/crates/ecstore/src/cache_value/metacache_set.rs +++ b/crates/ecstore/src/cache_value/metacache_set.rs @@ -43,11 +43,6 @@ const EVENT_METACACHE_LISTING: &str = "metacache_listing"; pub type AgreedFn = Box Pin> + Send>> + Send + 'static>; pub type PartialFn = Box]) -> Pin + Send>> + Send + 'static>; -pub(crate) type PartialResultFn = Box< - dyn Fn(MetaCacheEntries, &[Option]) -> Pin> + Send>> - + Send - + 'static, ->; type FinishedFn = Box]) -> Pin + Send>> + Send + 'static>; #[derive(Clone, Default)] @@ -60,6 +55,7 @@ impl FallbackClaimTracker { self.claimed.lock().await.insert(disk.endpoint().to_string()); } + #[cfg(test)] pub(crate) async fn claimed_keys(&self) -> HashSet { self.claimed.lock().await.clone() } @@ -317,7 +313,17 @@ pub struct ListPathRawOutcome { pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> { let rx = rx.child_token(); let _cancel_guard = rx.clone().drop_guard(); - list_path_raw_inner(rx, opts, None, None).await.map(|_| ()) + list_path_raw_inner(rx, opts, None).await.map(|_| ()) +} + +/// Like [`list_path_raw`], but reports how the merged stream ended. +pub(crate) async fn list_path_raw_with_outcome( + rx: CancellationToken, + opts: ListPathRawOptions, +) -> disk::error::Result { + let rx = rx.child_token(); + let _cancel_guard = rx.clone().drop_guard(); + list_path_raw_inner(rx, opts, None).await } pub(crate) async fn list_path_raw_with_claim_tracker( @@ -327,25 +333,13 @@ pub(crate) async fn list_path_raw_with_claim_tracker( ) -> disk::error::Result { let rx = rx.child_token(); let _cancel_guard = rx.clone().drop_guard(); - list_path_raw_inner(rx, opts, Some(claim_tracker), None).await -} - -pub(crate) async fn list_path_raw_with_partial_result( - rx: CancellationToken, - opts: ListPathRawOptions, - claim_tracker: FallbackClaimTracker, - partial_result: PartialResultFn, -) -> disk::error::Result { - let rx = rx.child_token(); - let _cancel_guard = rx.clone().drop_guard(); - list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await + list_path_raw_inner(rx, opts, Some(claim_tracker)).await } async fn list_path_raw_inner( rx: CancellationToken, opts: ListPathRawOptions, fallback_claim_tracker: Option, - partial_result: Option, ) -> disk::error::Result { if opts.disks.is_empty() { return Err(DiskError::ErasureReadQuorum); @@ -1020,13 +1014,7 @@ async fn list_path_raw_inner( } } - if let Some(partial_fn) = partial_result.as_ref() { - tokio::select! { - biased; - _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }), - result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?, - } - } else if let Some(partial_fn) = opts.partial.as_ref() { + if let Some(partial_fn) = opts.partial.as_ref() { tokio::select! { biased; _ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }), @@ -1308,7 +1296,6 @@ mod tests { ..Default::default() }, None, - None, ) .await .expect("listing should complete"); @@ -1333,7 +1320,6 @@ mod tests { ..Default::default() }, None, - None, ) .await .expect("listing should complete"); @@ -1827,39 +1813,6 @@ mod tests { ); } - #[tokio::test] - async fn list_path_raw_propagates_partial_callback_failure() { - let first = MetaCacheEntry { - name: "bucket/object-a".to_string(), - metadata: vec![1], - ..Default::default() - }; - let second = MetaCacheEntry { - name: "bucket/object-b".to_string(), - metadata: vec![2], - ..Default::default() - }; - - let err = list_path_raw_with_partial_result( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None], - min_disks: 2, - test_reader_behaviors: vec![ - TestReaderBehavior::Entries(vec![first]), - TestReaderBehavior::Entries(vec![second]), - ], - ..Default::default() - }, - FallbackClaimTracker::default(), - Box::new(move |_, _| Box::pin(async { Err(DiskError::ErasureReadQuorum) })), - ) - .await - .expect_err("a resolution failure after reader advancement must fail the listing"); - - assert_eq!(err, DiskError::ErasureReadQuorum); - } - #[tokio::test] async fn list_path_raw_cancels_a_blocked_partial_callback() { let cancel = CancellationToken::new(); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 67b391946..33b48fbe5 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -160,7 +160,7 @@ use rustfs_utils::http::{ use rustfs_utils::{ HashAlgorithm, crypto::hex, - path::{SLASH_SEPARATOR, encode_dir_object, encode_dir_object_ref, has_suffix, object_name_disk_identity, path_join_buf}, + path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, object_name_disk_identity, path_join_buf}, }; use sha2::Sha256; use std::borrow::Cow; @@ -5247,40 +5247,6 @@ impl SetDisks { .with_attempt(attempt)) } - pub(crate) async fn read_listing_metadata_after_namespace_barrier( - &self, - disks: &[Option], - bucket: &str, - object: &str, - ) -> Result<(MetaCacheEntries, usize)> { - // This is only called after the lock-free LIST fast path observes an - // unresolved metadata generation. Waiting on the ordinary object read - // lock establishes a publication boundary with concurrent overwrites. - let stored_object = encode_dir_object_ref(object); - let _guard = self - .acquire_read_lock_diag("list_object_reconcile", bucket, stored_object.as_ref()) - .await?; - let (raw_entries, errors) = Self::read_all_raw_file_info(disks, bucket, stored_object.as_ref(), false).await; - let confirmed_absent = errors - .iter() - .flatten() - .filter(|err| DiskError::is_err_object_not_found(err) || DiskError::is_err_version_not_found(err)) - .count(); - let entries = raw_entries - .into_iter() - .map(|raw| { - raw.filter(|raw| !raw.buf.is_empty()).map(|raw| MetaCacheEntry { - name: object.to_owned(), - metadata: raw.buf, - cached: None, - reusable: false, - }) - }) - .collect(); - - Ok((MetaCacheEntries(entries), confirmed_absent)) - } - async fn acquire_write_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result { crate::hp_guard!("SetDisks::acquire_write_lock"); let diag_enabled = is_object_lock_diag_enabled(); @@ -16477,54 +16443,6 @@ mod tests { assert_eq!(err, StorageError::PreconditionFailed); } - #[tokio::test] - async fn listing_metadata_reconcile_encodes_directory_object_and_waits_for_writer() { - let set_disks = make_local_bucket_test_set_disks().await; - let bucket = "bucket-list-reconcile-lock"; - let object = "directory/"; - let stored_object = encode_dir_object(object); - set_disks - .make_bucket(bucket, &MakeBucketOptions::default()) - .await - .expect("bucket should be created"); - let mut reader = PutObjReader::from_vec(b"directory marker".to_vec()); - set_disks - .put_object(bucket, &stored_object, &mut reader, &ObjectOptions::default()) - .await - .expect("directory object should be written under its encoded key"); - let disks = set_disks.disks.read().await.clone(); - let namespace_lock = set_disks - .new_ns_lock(bucket, &stored_object) - .await - .expect("namespace lock should be created"); - let writer_guard = namespace_lock - .get_write_lock(std::time::Duration::from_secs(30)) - .await - .expect("writer lock should be acquired"); - let read = set_disks.read_listing_metadata_after_namespace_barrier(&disks, bucket, object); - tokio::pin!(read); - assert!( - tokio::time::timeout(std::time::Duration::from_millis(50), &mut read) - .await - .is_err(), - "LIST reconciliation must wait for the publishing writer" - ); - drop(writer_guard); - - let (entries, confirmed_absent) = tokio::time::timeout(std::time::Duration::from_secs(30), read) - .await - .expect("LIST reconciliation should resume after writer release") - .expect("LIST reconciliation should read after the namespace barrier"); - assert_eq!(entries.0.len(), disks.len()); - assert!( - entries.0.iter().all(|entry| entry - .as_ref() - .is_some_and(|entry| entry.name == object && !entry.metadata.is_empty())), - "LIST reconciliation must read encoded metadata and retain the logical key" - ); - assert_eq!(confirmed_absent, 0); - } - #[tokio::test] async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() { let set_disks = make_local_bucket_test_set_disks().await; diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index bb82343a0..34bdf8b69 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -28,11 +28,11 @@ use crate::bucket::metadata_sys::{ use crate::bucket::utils::{check_list_objs_args, has_bad_path_component}; use crate::bucket::versioning::VersioningApi; use crate::cache_value::metacache_set::{ - FallbackClaimTracker, ListPathRawOptions, ListPathRawOutcome, list_path_raw_with_claim_tracker, - list_path_raw_with_partial_result, + FallbackClaimTracker, ListPathRawOptions, ListPathRawOutcome, list_path_raw, list_path_raw_with_claim_tracker, + list_path_raw_with_outcome, }; use crate::disk::error::DiskError; -use crate::disk::{DiskAPI, DiskInfo, DiskStore, RUSTFS_META_BUCKET, WalkDirOptions}; +use crate::disk::{DiskInfo, DiskStore, RUSTFS_META_BUCKET}; use crate::error::{ Error, Result, StorageError, is_all_disk_not_found, is_all_not_found, is_all_volume_not_found, to_object_err, }; @@ -47,9 +47,7 @@ use crate::store::utils::is_reserved_or_invalid_bucket; use futures::future::join_all; use rand::seq::SliceRandom; use rustfs_filemeta::{ - FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, - MetaCacheEntryResolution as ListingEntryResolution, MetacacheReader, MetadataResolutionParams, is_io_eof, - resolve_agreed_entry as resolve_agreed_listing_entry, + FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetadataResolutionParams, }; use rustfs_io_metrics::{ LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, LIST_OBJECTS_SOURCE_WALKER, @@ -64,8 +62,6 @@ use std::sync::{ atomic::{AtomicU64, Ordering}, }; use std::time::Duration; -use tokio::io::duplex; -use tokio::sync::OnceCell; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; @@ -80,10 +76,6 @@ const MAX_OBJECT_LIST: i32 = 1000; const METACACHE_SHARE_PREFIX: bool = false; -fn duration_millis(duration: Duration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} - type ListObjectsInfo = StorageListObjectsInfo; type ListObjectsV2Info = StorageListObjectsV2Info; type ListObjectVersionsInfo = StorageListObjectVersionsInfo; @@ -125,6 +117,11 @@ struct ListObjectVersionsInput<'a> { include_version_purge: bool, } +const EVENT_LISTING_MERGE_REJECTED: &str = "listing_merge_rejected"; +const EVENT_SET_LIST_PATH: &str = "set_list_path"; +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_LISTING: &str = "listing"; + const LIST_MERGED_INPUT_BUFFER: usize = 1; fn list_merged_entry_channel() -> (Sender, Receiver) { @@ -812,531 +809,14 @@ where (objects, prefixes, is_truncated, next_marker, next_version_idmarker) } -fn list_metadata_resolution_params( - bucket: String, - listing_quorum: usize, - latest_object_quorum: usize, - versioned: bool, - write_quorum_slack: usize, -) -> MetadataResolutionParams { - let quorum = if versioned { - listing_quorum - } else { - latest_object_quorum.max(listing_quorum) - }; - let mut resolver = MetadataResolutionParams { - dir_quorum: quorum, - obj_quorum: quorum, +fn list_metadata_resolution_params(bucket: String, listing_quorum: usize, versioned: bool) -> MetadataResolutionParams { + MetadataResolutionParams { + dir_quorum: listing_quorum, + obj_quorum: listing_quorum, + requested_versions: if versioned { 0 } else { 1 }, bucket, - write_quorum_slack, ..Default::default() - }; - - if !versioned { - resolver.requested_versions = 1; } - - resolver -} - -fn resolve_listing_entries( - entries: MetaCacheEntries, - resolver: MetadataResolutionParams, - enforce_write_quorum: bool, -) -> Option { - if enforce_write_quorum { - entries.resolve_with_write_quorum(resolver) - } else { - entries.resolve(resolver) - } -} - -#[derive(Clone)] -struct FallbackListingEntry { - endpoint: String, - entry: MetaCacheEntry, -} - -type FallbackListingEntries = HashMap>; - -#[derive(Clone)] -struct ListingSupplementOptions { - bucket: String, - path: String, - recursive: bool, - incl_deleted: bool, - skip_hidden_prefix_check: bool, - filter_prefix: Option, - forward_to: Option, - per_disk_limit: i32, - skip_total_timeout: bool, - walkdir_timeout: Option, - walkdir_stall_timeout: Option, -} - -struct ListingReconciler { - set: SetDisks, - primary_disks: Arc>, - fallback_disks: Arc>, -} - -impl ListingReconciler { - async fn read_after_namespace_barrier( - &self, - bucket: &str, - object: &str, - ) -> std::result::Result<(MetaCacheEntries, usize), DiskError> { - let disks = self - .primary_disks - .iter() - .chain(self.fallback_disks.iter()) - .cloned() - .map(Some) - .collect::>(); - - self.set - .read_listing_metadata_after_namespace_barrier(&disks, bucket, object) - .await - .map_err(|_| DiskError::ErasureReadQuorum) - } -} - -struct ListingSupplement { - options: ListingSupplementOptions, - fallback_disks: Arc>, - claim_tracker: FallbackClaimTracker, - entries: Option>, - reconciler: Option, -} - -impl ListingSupplement { - #[cfg(test)] - fn new( - options: ListingSupplementOptions, - fallback_disks: Arc>, - claim_tracker: FallbackClaimTracker, - ) -> Arc { - Self::new_inner(options, fallback_disks, claim_tracker, None) - } - - fn new_with_reconciler( - options: ListingSupplementOptions, - fallback_disks: Arc>, - claim_tracker: FallbackClaimTracker, - set: SetDisks, - primary_disks: Arc>, - ) -> Arc { - let reconciler = ListingReconciler { - set, - primary_disks, - fallback_disks: fallback_disks.clone(), - }; - - Self::new_inner(options, fallback_disks, claim_tracker, Some(reconciler)) - } - - fn new_inner( - options: ListingSupplementOptions, - fallback_disks: Arc>, - claim_tracker: FallbackClaimTracker, - reconciler: Option, - ) -> Arc { - let entries = if options.per_disk_limit > 0 && !fallback_disks.is_empty() { - Some(OnceCell::new()) - } else { - None - }; - - Arc::new(Self { - options, - fallback_disks, - claim_tracker, - entries, - reconciler, - }) - } - - fn is_empty(&self) -> bool { - self.fallback_disks.is_empty() - } - - async fn reconcile_object( - &self, - object: &str, - resolver: MetadataResolutionParams, - enforce_write_quorum: bool, - ) -> std::result::Result, DiskError> { - let Some(reconciler) = &self.reconciler else { - return Ok(None); - }; - let (entries, confirmed_absent) = reconciler.read_after_namespace_barrier(&self.options.bucket, object).await?; - if let Some(entry) = resolve_listing_entries(entries, resolver.clone(), enforce_write_quorum) { - return Ok(Some(entry)); - } - if confirmed_absent >= resolver.obj_quorum { - return Ok(None); - } - - Err(DiskError::ErasureReadQuorum) - } - - async fn entries_for(&self, object: &str) -> Vec> { - if self.fallback_disks.is_empty() { - return Vec::new(); - } - - let claimed_keys = self.claim_tracker.claimed_keys().await; - let Some(entries) = &self.entries else { - return self.read_object_entries(object, &claimed_keys).await; - }; - - let entries = entries.get_or_init(|| self.load_entries()).await; - let claimed_keys = self.claim_tracker.claimed_keys().await; - fallback_entries_for_object(entries, object, &claimed_keys) - } - - async fn load_entries(&self) -> FallbackListingEntries { - let claimed_keys = self.claim_tracker.claimed_keys().await; - let futures = self.fallback_disks.iter().filter_map(|disk| { - let endpoint = disk.endpoint().to_string(); - if claimed_keys.contains(&endpoint) { - return None; - } - - Some(read_fallback_listing_disk(disk.clone(), endpoint, self.options.clone())) - }); - - let per_disk_entries = join_all(futures).await; - let entry_count = per_disk_entries.iter().map(Vec::len).sum(); - let mut entries = FallbackListingEntries::with_capacity(entry_count); - for disk_entries in per_disk_entries { - for entry in disk_entries { - entries.entry(entry.entry.name.clone()).or_default().push(entry); - } - } - - entries - } - - async fn read_object_entries(&self, object: &str, claimed_keys: &HashSet) -> Vec> { - let futures = self.fallback_disks.iter().filter_map(|disk| { - let endpoint = disk.endpoint().to_string(); - if claimed_keys.contains(&endpoint) { - return None; - } - - Some(read_fallback_object_disk( - disk.clone(), - endpoint, - self.options.bucket.clone(), - object.to_owned(), - )) - }); - - let entries = join_all(futures).await; - let claimed_keys = self.claim_tracker.claimed_keys().await; - entries - .into_iter() - .flatten() - .filter(|entry| !claimed_keys.contains(&entry.endpoint)) - .map(|entry| Some(entry.entry)) - .collect() - } -} - -fn fallback_entries_for_object( - entries: &FallbackListingEntries, - object: &str, - claimed_keys: &HashSet, -) -> Vec> { - entries - .get(object) - .into_iter() - .flat_map(|entries| entries.iter()) - .filter(|entry| !claimed_keys.contains(&entry.endpoint)) - .map(|entry| Some(entry.entry.clone())) - .collect() -} - -async fn read_fallback_listing_disk( - disk: DiskStore, - endpoint: String, - options: ListingSupplementOptions, -) -> Vec { - let (rd, mut wr) = duplex(64 * 1024); - let walk_endpoint = endpoint.clone(); - let mut walk_jobs = JoinSet::new(); - walk_jobs.spawn(async move { - disk.walk_dir( - WalkDirOptions { - bucket: options.bucket, - base_dir: options.path, - recursive: options.recursive, - incl_deleted: options.incl_deleted, - skip_hidden_prefix_check: options.skip_hidden_prefix_check, - report_notfound: false, - filter_prefix: options.filter_prefix, - forward_to: options.forward_to, - limit: options.per_disk_limit, - skip_total_timeout: options.skip_total_timeout, - timeout_ms: options.walkdir_timeout.map(duration_millis), - stall_timeout_ms: options.walkdir_stall_timeout.map(duration_millis), - ..Default::default() - }, - &mut wr, - ) - .await - }); - - let mut reader = MetacacheReader::new(rd); - let mut entries = Vec::new(); - let mut stream_ok = true; - loop { - match reader.peek().await { - Ok(Some(entry)) => { - entries.push(FallbackListingEntry { - endpoint: endpoint.clone(), - entry, - }); - if let Err(err) = reader.skip(1).await { - debug!( - endpoint = %endpoint, - error = ?err, - "fallback listing supplement stream failed while advancing reader" - ); - stream_ok = false; - break; - } - } - Ok(None) => break, - Err(err) if err == rustfs_filemeta::Error::Unexpected || is_io_eof(&err) => break, - Err(err) => { - debug!( - endpoint = %endpoint, - error = ?err, - "fallback listing supplement stream failed while reading entry" - ); - stream_ok = false; - break; - } - } - } - drop(reader); - - match walk_jobs.join_next().await { - Some(Ok(Ok(()))) if stream_ok => entries, - Some(Ok(Ok(()))) => Vec::new(), - Some(Ok(Err(err))) => { - debug!( - endpoint = %walk_endpoint, - error = ?err, - "fallback listing supplement walk_dir failed" - ); - Vec::new() - } - Some(Err(err)) => { - debug!( - endpoint = %walk_endpoint, - error = ?err, - "fallback listing supplement task failed" - ); - Vec::new() - } - None => Vec::new(), - } -} - -async fn read_fallback_object_disk( - disk: DiskStore, - endpoint: String, - bucket: String, - object: String, -) -> Option { - match disk.read_xl(&bucket, &object, false).await { - Ok(raw) if raw.buf.is_empty() => None, - Ok(raw) => Some(FallbackListingEntry { - endpoint, - entry: MetaCacheEntry { - name: object, - metadata: raw.buf, - cached: None, - reusable: false, - }, - }), - Err(err) if DiskError::is_err_object_not_found(&err) || DiskError::is_err_version_not_found(&err) => None, - Err(err) => { - debug!( - endpoint = %endpoint, - bucket = %bucket, - object = %object, - error = ?err, - "fallback listing supplement read_xl failed" - ); - None - } - } -} - -#[derive(Debug, PartialEq, Eq)] -enum ListingSupplementTarget { - Directory(String), - Object(String), -} - -impl ListingSupplementTarget { - fn name(&self) -> &str { - match self { - Self::Directory(name) | Self::Object(name) => name, - } - } -} - -fn listing_entries_supplement_target( - entries: &MetaCacheEntries, - resolver: &MetadataResolutionParams, - enforce_write_quorum: bool, -) -> Option { - if !enforce_write_quorum { - return None; - } - - if let Some(directory) = entries.0.iter().flatten().find(|entry| entry.is_dir()) { - let directory_copies = entries - .0 - .iter() - .flatten() - .filter(|entry| entry.is_dir() && entry.name == directory.name) - .count(); - // A committed child may have some of its directory copies only on - // fallback disks, just like object metadata in a partial primary sample. - if directory_copies < resolver.dir_quorum { - return Some(ListingSupplementTarget::Directory(directory.name.clone())); - } - } - - for (idx, entry) in entries.0.iter().enumerate() { - let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else { - continue; - }; - if entries.0[..idx] - .iter() - .flatten() - .any(|previous| previous.name == entry.name && previous.is_object()) - { - continue; - } - - let reader_disks = entries - .0 - .iter() - .filter(|candidate| { - candidate - .as_ref() - .is_some_and(|candidate| candidate.name == entry.name && candidate.is_object()) - }) - .count(); - let entries_disagree = !entry.headers_agree( - entries - .0 - .iter() - .flatten() - .filter(|candidate| candidate.name == entry.name && candidate.is_object()), - resolver.strict, - ); - // A split primary sample can fall back to an older version even when a - // newer version reaches write quorum only after the fallback disks join. - if entries_disagree { - return Some(ListingSupplementTarget::Object(entry.name.clone())); - } - - if entry.needs_quorum_supplement(reader_disks, resolver).unwrap_or(false) { - return Some(ListingSupplementTarget::Object(entry.name.clone())); - } - } - - None -} - -async fn resolve_listing_entries_with_supplement( - entries: MetaCacheEntries, - resolver: MetadataResolutionParams, - enforce_write_quorum: bool, - supplement: Arc, -) -> std::result::Result, DiskError> { - if let Some(target) = listing_entries_supplement_target(&entries, &resolver, enforce_write_quorum) { - let mut candidates = entries.0; - if !supplement.is_empty() { - candidates.extend(supplement.entries_for(target.name()).await); - } - if let Some(entry) = resolve_listing_entries(MetaCacheEntries(candidates), resolver.clone(), enforce_write_quorum) { - return Ok(Some(entry)); - } - return match target { - ListingSupplementTarget::Directory(_) => Ok(None), - ListingSupplementTarget::Object(object) => supplement.reconcile_object(&object, resolver, enforce_write_quorum).await, - }; - } - - Ok(resolve_listing_entries(entries, resolver, enforce_write_quorum)) -} - -async fn resolve_agreed_listing_entry_with_supplement( - entry: MetaCacheEntry, - reader_disks: usize, - resolver: MetadataResolutionParams, - supplement: Arc, -) -> std::result::Result, DiskError> { - let fallback = supplement.entries_for(&entry.name).await; - let mut candidates = Vec::with_capacity(reader_disks + fallback.len()); - candidates.resize(reader_disks, Some(&entry)); - candidates.extend(fallback.iter().map(Option::as_ref)); - if let Some(resolved) = MetaCacheEntries::resolve_borrowed_with_write_quorum(&candidates, resolver.clone()) { - return Ok(Some(resolved)); - } - if entry.is_dir() { - return Ok(None); - } - supplement.reconcile_object(&entry.name, resolver, true).await -} - -/// The walk must discover every supported payload layout, including objects -/// written before a storage-class change. Per-version checks enforce the actual -/// data/write quorum; the topology default cannot stand in for object metadata. -fn latest_listing_object_quorum( - listing_quorum: usize, - drive_count: usize, - enforce_write_quorum: bool, - unreachable_disks: usize, -) -> usize { - if !enforce_write_quorum { - return listing_quorum; - } - let read_quorum = drive_count.div_ceil(2); - write_quorum_for_drive_count(drive_count, drive_count / 2) - .saturating_sub(unreachable_disks) - .max(read_quorum) - .max(listing_quorum) -} - -fn latest_listing_write_quorum_slack(enforce_write_quorum: bool, drive_count: usize, online_disks: usize) -> usize { - if !enforce_write_quorum { - return 0; - } - - drive_count.saturating_sub(online_disks) -} - -fn enforce_latest_listing_write_quorum(strict_latest: bool, ask_disks: &str) -> bool { - strict_latest && !matches!(normalize_list_quorum(ask_disks), "disk" | "reduced") -} - -#[cfg(test)] -fn latest_listing_allow_agreed_objects(enforce_write_quorum: bool, reader_disks: usize, object_quorum: usize) -> bool { - !enforce_write_quorum || reader_disks >= object_quorum -} - -fn latest_listing_raw_min_disks(listing_quorum: usize, object_quorum: usize, enforce_write_quorum: bool) -> usize { - if enforce_write_quorum { object_quorum } else { listing_quorum } } fn positive_ask_disks(ask_disks: i32) -> Option { @@ -1367,42 +847,6 @@ fn clamp_ask_disks_to_available(ask_disks: i32, available_disks: usize) -> i32 { ask_disks.min(bounded_usize_to_i32(available_disks)) } -// Keep the usual payload layout on the primary readers without making that -// layout a visibility requirement for older or differently configured objects. -fn preferred_listing_quorum( - metadata_quorum: usize, - drive_count: usize, - parity_count: usize, - enforce_write_quorum: bool, -) -> usize { - if enforce_write_quorum { - metadata_quorum.max(write_quorum_for_drive_count(drive_count, parity_count)) - } else { - metadata_quorum - } -} - -fn expand_ask_disks_for_object_quorum(ask_disks: i32, available_disks: usize, object_quorum: usize) -> i32 { - let Some(asked_disks) = positive_ask_disks(ask_disks) else { - return ask_disks; - }; - - bounded_usize_to_i32(asked_disks.max(object_quorum).min(available_disks)) -} - -fn write_quorum_for_drive_count(drive_count: usize, parity_count: usize) -> usize { - if drive_count == 0 { - return 0; - } - - let data_drives = drive_count.saturating_sub(parity_count); - if data_drives == parity_count { - data_drives.saturating_add(1) - } else { - data_drives - } -} - fn parse_version_marker(marker: String) -> Result { Ok(VersionMarker::parse(marker)?) } @@ -2010,9 +1454,6 @@ impl ECStore { for eset in self.pools.iter() { for set in eset.disk_set.iter() { let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await; - // Captured before any quorum-based filtering: only genuinely - // unreachable drives may relax the write-quorum requirement. - let online_disks = disks.len(); let opts = opts.clone(); let (sender, list_out_rx) = mpsc::channel::(1); @@ -2036,25 +1477,6 @@ impl ECStore { } let listing_quorum = listing_quorum_from_ask_disks(ask_disks); - let enforce_write_quorum = enforce_latest_listing_write_quorum(opts.latest_only, &opts.ask_disks); - let write_quorum_slack = - latest_listing_write_quorum_slack(enforce_write_quorum, set.set_drive_count, online_disks); - let required_obj_quorum = latest_listing_object_quorum( - listing_quorum, - set.set_drive_count, - enforce_write_quorum, - write_quorum_slack, - ); - ask_disks = expand_ask_disks_for_object_quorum( - ask_disks, - disks.len(), - preferred_listing_quorum( - required_obj_quorum, - set.set_drive_count, - set.default_parity_count, - enforce_write_quorum, - ), - ); let fallback_disks = { if let Some(asked_disks) = positive_ask_disks(ask_disks) && disks.len() > asked_disks @@ -2066,23 +1488,8 @@ impl ECStore { Vec::new() } }; - let fallback_disks = Arc::new(fallback_disks); - let disks = Arc::new(disks); - let claim_tracker = FallbackClaimTracker::default(); - let obj_quorum = required_obj_quorum; - let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, obj_quorum, enforce_write_quorum); - - let resolver = list_metadata_resolution_params( - bucket.to_owned(), - listing_quorum, - obj_quorum, - !opts.latest_only, - write_quorum_slack, - ); - let agreed_resolver = resolver.clone(); - let partial_resolver = resolver.clone(); - let reader_disks = disks.len(); + let resolver = list_metadata_resolution_params(bucket.to_owned(), listing_quorum, !opts.latest_only); let path = base_dir_from_prefix(prefix); ensure_non_empty_listing_disks(bucket, &path, &disks).map_err(to_filemeta_err)?; @@ -2099,44 +1506,21 @@ impl ECStore { filter_prefix = "".to_owned(); } - // let (sender, rx1) = mpsc::channel(100); - let tx1 = sender.clone(); let tx2 = sender.clone(); - let supplement = ListingSupplement::new_with_reconciler( - ListingSupplementOptions { - bucket: bucket.to_owned(), - path: path.clone(), - recursive: true, - incl_deleted: !opts.latest_only, - skip_hidden_prefix_check: false, - filter_prefix: Some(filter_prefix.clone()), - forward_to: opts.marker.clone(), - per_disk_limit: bounded_usize_to_i32(opts.limit), - skip_total_timeout: opts.walkdir_timeout.is_none(), - walkdir_timeout: opts.walkdir_timeout, - walkdir_stall_timeout: opts.walkdir_stall_timeout, - }, - fallback_disks.clone(), - claim_tracker.clone(), - set.as_ref().clone(), - disks.clone(), - ); - let agreed_supplement = supplement.clone(); - let partial_supplement = supplement; - list_path_raw_with_partial_result( + list_path_raw( rx_clone, ListPathRawOptions { - disks: disks.iter().cloned().map(Some).collect(), - fallback_disks: fallback_disks.iter().cloned().map(Some).collect(), + disks: disks.into_iter().map(Some).collect(), + fallback_disks: fallback_disks.into_iter().map(Some).collect(), bucket: bucket.to_owned(), path, recursive: true, incl_deleted: !opts.latest_only, filter_prefix: Some(filter_prefix), forward_to: opts.marker.clone(), - min_disks: raw_min_disks, + min_disks: listing_quorum, per_disk_limit: bounded_usize_to_i32(opts.limit), // Skip the total walkdir timeout for listing operations. // Large buckets (millions of objects) can take longer than @@ -2149,31 +1533,7 @@ impl ECStore { agreed: Some(Box::new(move |entry: MetaCacheEntry| { Box::pin({ let value = tx1.clone(); - let resolver = agreed_resolver.clone(); - let supplement = agreed_supplement.clone(); async move { - let entry = match resolve_agreed_listing_entry( - entry, - reader_disks, - &resolver, - enforce_write_quorum, - ) { - ListingEntryResolution::Resolved(entry) => entry, - ListingEntryResolution::NeedsSupplement(entry) => { - let Some(entry) = resolve_agreed_listing_entry_with_supplement( - entry, - reader_disks, - resolver, - supplement, - ) - .await? - else { - return Ok(()); - }; - entry - } - ListingEntryResolution::Rejected => return Ok(()), - }; if entry.is_dir() { return Ok(()); } @@ -2184,31 +1544,22 @@ impl ECStore { } }) })), + partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { + Box::pin({ + let value = tx2.clone(); + let resolver = resolver.clone(); + async move { + if let Some(entry) = entries.resolve(resolver) + && let Err(err) = value.send(entry).await + { + error!("list_path send fail {:?}", err); + } + } + }) + })), finished: None, ..Default::default() }, - claim_tracker, - Box::new(move |entries: MetaCacheEntries, _: &[Option]| { - Box::pin({ - let value = tx2.clone(); - let resolver = partial_resolver.clone(); - let supplement = partial_supplement.clone(); - async move { - if let Some(entry) = resolve_listing_entries_with_supplement( - entries, - resolver, - enforce_write_quorum, - supplement, - ) - .await? - && let Err(err) = value.send(entry).await - { - error!("list_path send fail {:?}", err); - } - Ok(()) - } - }) - }), ) .await }); @@ -2760,9 +2111,6 @@ async fn send_or_cancel(rx: &CancellationToken, out_channel: &Sender } fn listing_merge_corruption(object: &str, version_id: Option, validation: &'static str) -> Error { - const EVENT_LISTING_MERGE_REJECTED: &str = "listing_merge_rejected"; - const LOG_COMPONENT_ECSTORE: &str = "ecstore"; - const LOG_SUBSYSTEM_LISTING: &str = "listing"; error!( event = EVENT_LISTING_MERGE_REJECTED, component = LOG_COMPONENT_ECSTORE, @@ -3295,6 +2643,12 @@ impl SetDisks { /// Streams the merged raw entries of this set into `sender` and reports /// how the producers stopped. + /// + /// Latest-version listing follows MinIO's `listPath`: an entry is visible + /// when about half of the asked drives carry it, a split sample resolves + /// to the newest version that reaches that quorum, and an entry that + /// reaches no quorum is omitted instead of failing the page. Whether the + /// listed version is readable remains the read path's decision. pub async fn list_path( &self, rx: CancellationToken, @@ -3304,14 +2658,24 @@ impl SetDisks { let list_path_started = std::time::Instant::now(); let (mut disks, infos, _) = self.get_online_disks_with_healing_and_info(true).await; - // Captured before any quorum-based filtering: only genuinely - // unreachable drives may relax the write-quorum requirement. let online_disks = disks.len(); - // Version pages resolve every version at listing quorum and omit the - // rest, as MinIO's listPath does with requestedVersions = 0. Unlike - // MinIO they still require a read quorum of online drives. - if opts.versioned && online_disks < set_listing_read_floor(self.set_drive_count, self.default_parity_count) { - return Err(DiskError::ErasureReadQuorum.into()); + // Unlike MinIO, a set with fewer than half of its drives online does + // not list from that minority: the whole page is unavailable. + let set_read_quorum = self.set_drive_count.div_ceil(2); + if self.set_drive_count == 0 || online_disks < set_read_quorum { + debug!( + event = EVENT_SET_LIST_PATH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LISTING, + state = "below_set_read_quorum", + bucket = %opts.bucket, + prefix = %opts.base_dir, + set_drive_count = self.set_drive_count, + online_disks, + set_read_quorum, + "Set listing is below read quorum" + ); + return Err(StorageError::ErasureReadQuorum); } let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32); @@ -3332,60 +2696,24 @@ impl SetDisks { } let listing_quorum = listing_quorum_from_ask_disks(ask_disks); - ensure_non_empty_listing_disks(&opts.bucket, &opts.base_dir, &disks)?; - - let enforce_write_quorum = enforce_latest_listing_write_quorum(!opts.versioned, &opts.ask_disks); - let write_quorum_slack = latest_listing_write_quorum_slack(enforce_write_quorum, self.set_drive_count, online_disks); - let required_obj_quorum = - latest_listing_object_quorum(listing_quorum, self.set_drive_count, enforce_write_quorum, write_quorum_slack); - ask_disks = expand_ask_disks_for_object_quorum( - ask_disks, - disks.len(), - preferred_listing_quorum(required_obj_quorum, self.set_drive_count, self.default_parity_count, enforce_write_quorum), - ); let mut fallback_disks = Vec::new(); - if let Some(asked_disks) = positive_ask_disks(ask_disks) && disks.len() > asked_disks { let mut rand = rand::rng(); disks.shuffle(&mut rand); - + // A fallback drive only replaces a primary reader that fails; it + // never adds a vote to an entry the primary readers already saw. fallback_disks = disks.split_off(asked_disks); } - let fallback_disks = Arc::new(fallback_disks); - let disks = Arc::new(disks); - let claim_tracker = FallbackClaimTracker::default(); + let reader_disks = disks.len(); + let fallback_count = fallback_disks.len(); let bucket = opts.bucket.clone(); let base_dir = opts.base_dir.clone(); - let latest_object_quorum = required_obj_quorum; - let raw_min_disks = latest_listing_raw_min_disks(listing_quorum, latest_object_quorum, enforce_write_quorum); - let resolver = list_metadata_resolution_params( - bucket.clone(), - listing_quorum, - latest_object_quorum, - opts.versioned, - write_quorum_slack, - ); - let agreed_resolver = resolver.clone(); - let partial_resolver = resolver.clone(); - let reader_disks = disks.len(); - - debug!( - bucket = %bucket, - prefix = %base_dir, - set_drive_count = self.set_drive_count, - asked_disks = ask_disks, - listing_quorum = listing_quorum, - latest_object_quorum = latest_object_quorum, - write_quorum_slack = write_quorum_slack, - raw_min_disks = raw_min_disks, - fallback_disks = fallback_disks.len(), - limit = opts.limit, - stop_disk_at_limit = opts.stop_disk_at_limit, - "set_disks list_path selected listing quorum and fallback disks" - ); + // Version pages keep every version that reaches listing quorum + // (requestedVersions = 0); object pages keep only the latest. + let resolver = list_metadata_resolution_params(bucket.clone(), listing_quorum, opts.versioned); let limit = { // A raw scan budget can be consumed entirely by keys folding into @@ -3397,37 +2725,33 @@ impl SetDisks { } }; - let tx1 = sender.clone(); - let tx2 = sender.clone(); - let cancel_for_send1 = rx.clone(); - let cancel_for_send2 = rx.clone(); - let supplement = ListingSupplement::new_with_reconciler( - ListingSupplementOptions { - bucket: bucket.clone(), - path: opts.base_dir.clone(), - recursive: opts.recursive, - incl_deleted: opts.incl_deleted, - skip_hidden_prefix_check: opts.skip_hidden_prefix_check, - filter_prefix: opts.filter_prefix.clone(), - forward_to: opts.marker.clone(), - per_disk_limit: limit, - skip_total_timeout: true, - walkdir_timeout: opts.walkdir_timeout, - walkdir_stall_timeout: opts.walkdir_stall_timeout, - }, - fallback_disks.clone(), - claim_tracker.clone(), - self.clone(), - disks.clone(), + debug!( + event = EVENT_SET_LIST_PATH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LISTING, + state = "quorum_selected", + bucket = %bucket, + prefix = %base_dir, + set_drive_count = self.set_drive_count, + online_disks, + asked_disks = ask_disks, + listing_quorum, + fallback_disks = fallback_count, + limit = opts.limit, + per_disk_limit = limit, + "Set list_path selected listing quorum" ); - let agreed_supplement = supplement.clone(); - let partial_supplement = supplement; - let result = list_path_raw_with_partial_result( + let agreed_sender = sender.clone(); + let agreed_cancel = rx.clone(); + let partial_sender = sender; + let partial_cancel = rx.clone(); + + let result = list_path_raw_with_outcome( rx, ListPathRawOptions { - disks: disks.iter().cloned().map(Some).collect(), - fallback_disks: fallback_disks.iter().cloned().map(Some).collect(), + disks: disks.into_iter().map(Some).collect(), + fallback_disks: fallback_disks.into_iter().map(Some).collect(), bucket: opts.bucket, path: opts.base_dir, recursive: opts.recursive, @@ -3435,7 +2759,7 @@ impl SetDisks { skip_hidden_prefix_check: opts.skip_hidden_prefix_check, filter_prefix: opts.filter_prefix, forward_to: opts.marker, - min_disks: raw_min_disks, + min_disks: listing_quorum, per_disk_limit: limit, // A foreground listing is bounded by lack of drive progress (the walk // stall timeout) and by the page limit, never by how long a healthy @@ -3444,106 +2768,64 @@ impl SetDisks { walkdir_timeout: opts.walkdir_timeout, walkdir_stall_timeout: opts.walkdir_stall_timeout, agreed: Some(Box::new(move |entry: MetaCacheEntry| { - Box::pin({ - let value = tx1.clone(); - let cancel_token = cancel_for_send1.clone(); - let resolver = agreed_resolver.clone(); - let supplement = agreed_supplement.clone(); - async move { - if cancel_token.is_cancelled() { - return Ok(()); - } - - let entry = match resolve_agreed_listing_entry(entry, reader_disks, &resolver, enforce_write_quorum) { - ListingEntryResolution::Resolved(entry) => entry, - ListingEntryResolution::NeedsSupplement(entry) => { - if cancel_token.is_cancelled() { - return Ok(()); - } - - let Some(entry) = - resolve_agreed_listing_entry_with_supplement(entry, reader_disks, resolver, supplement) - .await? - else { - return Ok(()); - }; - entry - } - ListingEntryResolution::Rejected => return Ok(()), - }; - - if let Err(err) = send_or_cancel(&cancel_token, &value, entry).await - && !cancel_token.is_cancelled() - { - error!("list_path send fail {:?}", err); - } - Ok(()) + let sender = agreed_sender.clone(); + let cancel = agreed_cancel.clone(); + Box::pin(async move { + let _ = send_or_cancel(&cancel, &sender, entry).await; + Ok(()) + }) + })), + partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { + let sender = partial_sender.clone(); + let cancel = partial_cancel.clone(); + let resolver = resolver.clone(); + Box::pin(async move { + if cancel.is_cancelled() { + return; + } + if let Some(entry) = entries.resolve(resolver) { + let _ = send_or_cancel(&cancel, &sender, entry).await; } }) })), finished: None, ..Default::default() }, - claim_tracker, - Box::new(move |entries: MetaCacheEntries, _: &[Option]| { - Box::pin({ - let value = tx2.clone(); - let resolver = partial_resolver.clone(); - let cancel_token = cancel_for_send2.clone(); - let supplement = partial_supplement.clone(); - async move { - if cancel_token.is_cancelled() { - return Ok(()); - } - - if let Some(entry) = - resolve_listing_entries_with_supplement(entries, resolver, enforce_write_quorum, supplement).await? - && let Err(err) = send_or_cancel(&cancel_token, &value, entry).await - && !cancel_token.is_cancelled() - { - error!("list_path send fail {:?}", err); - } - Ok(()) - } - }) - }), ) .await; - rustfs_io_metrics::record_stage_duration( - "set_disks_list_objects_list_path", - list_path_started.elapsed().as_secs_f64() * 1000.0, - ); + let elapsed_ms = list_path_started.elapsed().as_secs_f64() * 1000.0; + rustfs_io_metrics::record_stage_duration("set_disks_list_objects_list_path", elapsed_ms); - if let Err(ref err) = result { - debug!( + match &result { + Ok(outcome) => debug!( + event = EVENT_SET_LIST_PATH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LISTING, + result = "ok", bucket = %bucket, - path = %base_dir, - disk_count = disks.len(), - fallback_disks = fallback_disks.len(), - asked_disks = ask_disks, - listing_quorum = listing_quorum, - stop_disk_at_limit = opts.stop_disk_at_limit, - limit = opts.limit, - per_disk_limit = limit, - elapsed_ms = list_path_started.elapsed().as_secs_f64() * 1000.0, - list_error = %err, - "set_disks list_path finished with error" - ); - } else { - debug!( + prefix = %base_dir, + reader_disks, + fallback_disks = fallback_count, + listing_quorum, + limit_reached = outcome.limit_reached, + elapsed_ms, + "Set list_path finished" + ), + Err(err) => debug!( + event = EVENT_SET_LIST_PATH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LISTING, + result = "error", bucket = %bucket, - path = %base_dir, - disk_count = disks.len(), - fallback_disks = fallback_disks.len(), - asked_disks = ask_disks, - listing_quorum = listing_quorum, - stop_disk_at_limit = opts.stop_disk_at_limit, - limit = opts.limit, - per_disk_limit = limit, - elapsed_ms = list_path_started.elapsed().as_secs_f64() * 1000.0, - "set_disks list_path finished" - ); + prefix = %base_dir, + reader_disks, + fallback_disks = fallback_count, + listing_quorum, + elapsed_ms, + error = %err, + "Set list_path finished" + ), } result.map_err(Error::from) @@ -3618,32 +2900,32 @@ mod lifecycle_exact_tests; #[cfg(test)] mod test { use super::{ - ENV_API_LIST_OBJECTS_QUORUM, FallbackListingEntries, FallbackListingEntry, GatherResultsState, ListObjectsInfo, - ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution, ListingSupplement, ListingSupplementOptions, - ListingSupplementTarget, MAX_OBJECT_LIST, RUSTFS_META_BUCKET, VersionMarker, enforce_latest_listing_write_quorum, - expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, - latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_write_quorum_slack, list_marker_key, - list_merged_entry_channel, list_metadata_resolution_params, list_objects_paginate, list_objects_quorum_from_env, - listing_entries_supplement_target, max_keys_plus_one, merge_entry_channels, normalize_list_quorum, - observe_scanner_namespace_mutations, parse_version_marker, recursive_walk_list_path_options, - reset_scanner_namespace_mutation_generation_for_test, resolve_agreed_listing_entry, resolve_listing_entries, - resolve_listing_entries_with_supplement, scanner_namespace_mutation_generation, send_or_cancel, - should_purge_empty_directory_listing, version_marker_for_entries, walk_result_from_set_errors, + ENV_API_LIST_OBJECTS_QUORUM, GatherResultsState, ListObjectsInfo, ListPathOptions, ListSourceMode, MAX_OBJECT_LIST, + RUSTFS_META_BUCKET, VersionMarker, gather_results, list_marker_key, list_merged_entry_channel, + list_metadata_resolution_params, list_objects_paginate, list_objects_quorum_from_env, max_keys_plus_one, + merge_entry_channels, normalize_list_quorum, observe_scanner_namespace_mutations, parse_version_marker, + recursive_walk_list_path_options, reset_scanner_namespace_mutation_generation_for_test, + scanner_namespace_mutation_generation, send_or_cancel, should_purge_empty_directory_listing, version_marker_for_entries, + walk_result_from_set_errors, }; use crate::bucket::replication::ReplicationState; - use crate::cache_value::metacache_set::{FallbackClaimTracker, ListPathRawOutcome, TestReaderBehavior, list_path_raw}; - use crate::disk::{DeleteOptions, DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, error::DiskError, new_disk}; + use crate::cache_value::metacache_set::ListPathRawOutcome; + use crate::disk::{DeleteOptions, DiskAPI, STORAGE_FORMAT_FILE, endpoint::Endpoint, error::DiskError}; use crate::error::{Result, StorageError}; - use crate::object_api::ObjectInfo; + use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader}; use crate::set_disk::{SetDisks, hermetic_set_disks_isolated}; + use crate::storage_api_contracts::namespace::NamespaceLocking as _; + use crate::storage_api_contracts::object::ObjectIO as _; use bytes::Bytes; + use http::HeaderMap; use rustfs_filemeta::{ FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker, ObjectPartInfo, VersionType, }; - use std::collections::{HashMap, HashSet}; + use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::Duration; + use tokio::io::AsyncReadExt as _; use tokio::sync::mpsc; use tokio::time::timeout; use tokio_util::sync::CancellationToken; @@ -3932,9 +3214,20 @@ mod test { mod_time: time::OffsetDateTime, etag: &str, data_dir: Uuid, + ) -> MetaCacheEntry { + test_unversioned_object_meta_entry_with_erasure(name, mod_time, etag, data_dir, 2, 2) + } + + fn test_unversioned_object_meta_entry_with_erasure( + name: &str, + mod_time: time::OffsetDateTime, + etag: &str, + data_dir: Uuid, + data_blocks: usize, + parity_blocks: usize, ) -> MetaCacheEntry { let mut meta = FileMeta::new(); - let mut fi = FileInfo::new(name, 2, 2); + let mut fi = FileInfo::new(name, data_blocks, parity_blocks); fi.erasure.index = 1; fi.data_dir = Some(data_dir); fi.volume = "bucket".to_owned(); @@ -4037,142 +3330,6 @@ mod test { } } - #[test] - fn fallback_entries_for_object_filters_claimed_physical_disks() { - let mut entries = FallbackListingEntries::new(); - entries.insert( - "object".to_string(), - vec![ - FallbackListingEntry { - endpoint: "disk-a".to_string(), - entry: test_meta_entry("object"), - }, - FallbackListingEntry { - endpoint: "disk-b".to_string(), - entry: test_meta_entry("object"), - }, - ], - ); - let claimed = HashSet::from(["disk-a".to_string()]); - - let candidates = fallback_entries_for_object(&entries, "object", &claimed); - - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].as_ref().map(|entry| entry.name.as_str()), Some("object")); - } - - #[tokio::test] - async fn listing_supplement_uses_page_cache_only_for_bounded_walks() { - let (_dirs, disks, _set) = hermetic_set_disks_isolated(4).await; - let fallback = Arc::new(vec![disks[0].clone()]); - let bounded = ListingSupplement::new( - ListingSupplementOptions { - bucket: "bucket".to_string(), - path: String::new(), - recursive: true, - incl_deleted: false, - skip_hidden_prefix_check: false, - filter_prefix: None, - forward_to: None, - per_disk_limit: 100, - skip_total_timeout: true, - walkdir_timeout: None, - walkdir_stall_timeout: None, - }, - fallback.clone(), - FallbackClaimTracker::default(), - ); - let unbounded = ListingSupplement::new( - ListingSupplementOptions { - bucket: "bucket".to_string(), - path: String::new(), - recursive: true, - incl_deleted: false, - skip_hidden_prefix_check: false, - filter_prefix: None, - forward_to: None, - per_disk_limit: 0, - skip_total_timeout: false, - walkdir_timeout: None, - walkdir_stall_timeout: None, - }, - fallback, - FallbackClaimTracker::default(), - ); - - let empty = ListingSupplement::new(bounded.options.clone(), Arc::new(Vec::new()), FallbackClaimTracker::default()); - assert!(empty.entries.is_none(), "no fallback readers need no page cache"); - assert!(bounded.entries.is_some()); - assert!(bounded.options.skip_total_timeout); - assert!(unbounded.entries.is_none()); - assert!(!unbounded.options.skip_total_timeout); - } - - #[tokio::test] - async fn listing_supplement_unbounded_reads_object_metadata_from_fallback_disk() { - let tempdir = tempfile::tempdir().expect("tempdir should be created"); - let endpoint = - Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("local disk should be created"); - disk.make_volume("bucket").await.expect("test bucket should be created"); - let mut fi = FileInfo::new("object", 1, 1); - fi.volume = "bucket".to_owned(); - fi.name = "object".to_owned(); - fi.size = 1; - fi.parts = vec![ObjectPartInfo { - number: 1, - size: 1, - actual_size: 1, - ..Default::default() - }]; - fi.fresh = true; - fi.erasure.index = 1; - fi.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")); - fi.metadata.insert("etag".to_owned(), "object-etag".to_owned()); - disk.write_metadata("", "bucket", "object", fi) - .await - .expect("test metadata should be written"); - - let supplement = ListingSupplement::new( - ListingSupplementOptions { - bucket: "bucket".to_string(), - path: String::new(), - recursive: true, - incl_deleted: false, - skip_hidden_prefix_check: false, - filter_prefix: None, - forward_to: None, - per_disk_limit: 0, - skip_total_timeout: false, - walkdir_timeout: None, - walkdir_stall_timeout: None, - }, - Arc::new(vec![disk]), - FallbackClaimTracker::default(), - ); - - let entries = supplement.entries_for("object").await; - - assert!(supplement.entries.is_none()); - assert_eq!(entries.len(), 1); - let entry = entries - .into_iter() - .next() - .and_then(|entry| entry) - .expect("fallback object metadata should be returned"); - let info = entry.to_fileinfo("bucket").expect("fallback metadata should decode"); - assert_eq!(entry.name, "object"); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("object-etag")); - } - fn test_object_meta_entry_with_erasure_versions( name: &str, versions: &[(time::OffsetDateTime, &str, usize, usize)], @@ -6349,7 +5506,7 @@ mod test { #[test] fn list_metadata_resolution_params_limits_plain_listing_to_latest_version() { - let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 3, false, 0); + let resolver = list_metadata_resolution_params("bucket".to_string(), 3, false); assert_eq!(resolver.dir_quorum, 3); assert_eq!(resolver.obj_quorum, 3); @@ -6646,7 +5803,7 @@ mod test { #[test] fn list_metadata_resolution_params_keeps_all_versions_for_version_listing() { - let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, true, 0); + let resolver = list_metadata_resolution_params("bucket".to_string(), 3, true); assert_eq!(resolver.dir_quorum, 3); assert_eq!(resolver.obj_quorum, 3); @@ -6654,6 +5811,45 @@ mod test { assert_eq!(resolver.requested_versions, 0); } + /// Lists `bucket` on one set and returns each emitted object with the + /// etag of the version it selected as latest (or `delete-marker`). + async fn collect_listing_etags( + set: &SetDisks, + bucket: &str, + mode: &str, + ) -> (Result, Vec<(String, String)>) { + let (sender, mut receiver) = mpsc::channel(1); + let walk = set.list_path( + CancellationToken::new(), + ListPathOptions { + bucket: bucket.to_owned(), + ask_disks: mode.to_owned(), + recursive: true, + limit: 100, + ..Default::default() + }, + sender, + ); + let drain = async { + let mut listed = Vec::new(); + while let Some(entry) = receiver.recv().await { + if entry.is_dir() { + continue; + } + let latest = match entry.to_fileinfo(bucket) { + Ok(info) if info.deleted => "delete-marker".to_owned(), + Ok(info) => info.metadata.get("etag").cloned().unwrap_or_default(), + Err(err) => format!("undecodable: {err}"), + }; + listed.push((entry.name, latest)); + } + listed + }; + timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) }) + .await + .expect("native listing must terminate") + } + async fn single_set_store(drive_count: usize) -> (Vec, Arc) { let mut dirs = Vec::with_capacity(drive_count); let mut endpoints = Vec::with_capacity(drive_count); @@ -6871,7 +6067,14 @@ mod test { } for mode in ["optimal", "strict"] { let (result, names) = collect_native_listing(&set, bucket, mode).await; - if online >= 8 { + if online == 16 && mode == "optimal" { + // `optimal` asks a random half of the drives and needs four of + // the nine copies in that sample, as MinIO's optimal mode does; + // about one sample in six holds fewer, so the key may be + // omitted. It must still never fail the page. + result.expect("a sampled listing must not fail the page"); + assert!(names.is_empty() || names == ["readable.bin"], "online {online}, mode {mode}: {names:?}"); + } else if online >= 8 { result.expect("EC8 committed metadata must remain listable"); assert_eq!(names, ["readable.bin"], "online {online}, mode {mode}"); } else { @@ -6883,8 +6086,19 @@ mod test { } #[tokio::test] - async fn issue_8828_rejects_unreadable_and_stale_agreed_versions() { - for (data, parity, copies, leading_free) in [(12, 4, 8, false), (8, 8, 7, false), (12, 4, 8, true)] { + async fn issue_8828_lists_at_listing_quorum_without_failing_the_page() { + // Eight of sixteen drives are online, so every mode asks those eight + // and needs four copies. Readability is the read path's decision: + // a key below its data quorum still lists, a stale minority stays + // hidden, and neither outcome fails the page for its sibling. + for (data, parity, copies, leading_free, listed) in [ + (12, 4, 8, false, true), + (8, 8, 7, false, true), + (12, 4, 8, true, true), + (8, 8, 4, false, true), + (8, 8, 3, false, false), + (12, 4, 1, false, false), + ] { let bucket = "issue-8828-unreadable"; let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks(16, 4).await; let mut entry = test_object_meta_entry_with_erasure_versions( @@ -6906,9 +6120,25 @@ mod test { entry.metadata = metadata.marshal_msg().expect("free and object metadata"); entry.cached = None; } + let sibling = test_object_meta_entry_with_erasure_versions( + "sibling.bin", + &[( + time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture time"), + "sibling", + 12, + 4, + )], + ); for (slot, dir) in dirs.iter().enumerate() { let path = dir.path().join(bucket); tokio::fs::create_dir_all(&path).await.expect("create bucket directory"); + let sibling_path = path.join(&sibling.name); + tokio::fs::create_dir_all(&sibling_path) + .await + .expect("create sibling directory"); + tokio::fs::write(sibling_path.join(STORAGE_FORMAT_FILE), &sibling.metadata) + .await + .expect("write sibling metadata"); if slot < copies { let path = path.join(&entry.name); tokio::fs::create_dir_all(&path).await.expect("create object directory"); @@ -6925,11 +6155,14 @@ mod test { } for mode in ["optimal", "strict"] { let (result, names) = collect_native_listing(&set, bucket, mode).await; - assert!( - matches!(result, Err(StorageError::ErasureReadQuorum)), - "layout {data}+{parity}, mode {mode}: {result:?}" - ); - assert!(names.is_empty(), "unreadable/stale metadata must not publish an object"); + let context = format!("layout {data}+{parity}, copies {copies}, free {leading_free}, mode {mode}"); + result.unwrap_or_else(|err| panic!("{context}: one object must never fail the page: {err:?}")); + let expected: &[&str] = if listed { + &["sibling.bin", "unreadable.bin"] + } else { + &["sibling.bin"] + }; + assert_eq!(names, expected, "{context}"); } } } @@ -7017,363 +6250,8 @@ mod test { } } - #[test] - fn agreed_filtered_history_preserves_selected_bodies_and_inline_data() { - for warm in [false, true] { - let mut entry = test_transitioned_meta_entry("key", "remote-key", true); - let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("older version"); - let history = test_object_meta_entry_with_erasure_versions( - "key", - &[(time, "visible", 8, 8), (time - time::Duration::seconds(1), "older", 8, 8)], - ); - let cached = entry.cached.as_mut().expect("free record fixture"); - cached.versions.extend(history.cached.expect("user history").versions); - let mut inline = vec![1]; - rmp::encode::write_map_len(&mut inline, 1).expect("inline map"); - rmp::encode::write_str(&mut inline, &cached.versions[1].header.version_id.expect("visible id").to_string()) - .expect("inline key"); - rmp::encode::write_bin(&mut inline, b"inline-payload").expect("inline value"); - cached.data.update(&inline); - let body = cached.versions[1].meta.as_ptr(); - entry.metadata = cached.marshal_msg().expect("complete history"); - if !warm { - entry.cached = None; - } - let resolver = list_metadata_resolution_params("bucket".to_owned(), 4, 8, false, 8); - let ListingEntryResolution::Resolved(entry) = resolve_agreed_listing_entry(entry, 8, &resolver, true) else { - panic!("readable free-headed history must resolve"); - }; - let info = entry.to_fileinfo("bucket").expect("filtered visible version"); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("visible")); - let cached = entry.cached.expect("filtered cache"); - assert_eq!(cached.versions.len(), 2, "one free record and latest user version"); - if warm { - assert_eq!(cached.versions[1].meta.as_ptr(), body, "selected cached body must move"); - } - assert_eq!(cached.data.as_slice(), inline.as_slice(), "inline payload is preserved"); - let decoded = FileMeta::load(&entry.metadata).expect("filtered serialized history"); - assert_eq!(decoded.versions, cached.versions); - assert_eq!(decoded.data.as_slice(), inline.as_slice()); - } - } - - #[test] - fn latest_listing_primary_sizing_preserves_default_layout_and_sampling_modes() { - assert_eq!(super::preferred_listing_quorum(9, 16, 4, true), 12); - assert_eq!(super::preferred_listing_quorum(8, 16, 4, true), 12); - assert_eq!( - expand_ask_disks_for_object_quorum(8, 8, super::preferred_listing_quorum(8, 16, 4, true)), - 8 - ); - assert_eq!(super::preferred_listing_quorum(1, 16, 4, false), 1); - assert_eq!(super::preferred_listing_quorum(2, 16, 4, false), 2); - } - - #[test] - fn latest_listing_object_quorum_uses_write_quorum_for_strict_latest_listing() { - let required_quorum = latest_listing_object_quorum(2, 8, true, 0); - let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum); - - assert_eq!(required_quorum, 5); - assert_eq!(ask_disks, 5); - assert_eq!(latest_listing_object_quorum(2, 8, true, 0), 5); - } - - #[test] - fn latest_listing_object_quorum_is_independent_of_payload_storage_class() { - let required_quorum = latest_listing_object_quorum(2, 8, true, 0); - let ask_disks = expand_ask_disks_for_object_quorum(4, 8, required_quorum); - - assert_eq!(required_quorum, 5); - assert_eq!(ask_disks, 5); - assert_eq!(latest_listing_object_quorum(2, 8, true, 0), 5); - } - - #[test] - fn latest_listing_object_quorum_keeps_reduced_and_versioned_listing_quorum() { - assert!(!enforce_latest_listing_write_quorum(true, "reduced")); - assert!(!enforce_latest_listing_write_quorum(true, "disk")); - assert!(enforce_latest_listing_write_quorum(true, "optimal")); - assert!(!enforce_latest_listing_write_quorum(false, "optimal")); - assert_eq!(latest_listing_object_quorum(1, 4, false, 0), 1); - assert_eq!(latest_listing_object_quorum(1, 4, false, 0), 1); - assert_eq!(latest_listing_object_quorum(2, 4, false, 0), 2); - assert_eq!(latest_listing_object_quorum(2, 4, false, 0), 2); - assert_eq!(expand_ask_disks_for_object_quorum(2, 4, 2), 2); - } - - #[test] - fn latest_listing_object_quorum_relaxes_write_quorum_by_unreachable_drives() { - // 4-drive set, EC 2+2: with one drive offline an object committed at - // write quorum 3 can only ever show 2 metadata copies to the listing. - let slack = latest_listing_write_quorum_slack(true, 4, 3); - assert_eq!(slack, 1); - assert_eq!(latest_listing_object_quorum(2, 4, true, slack), 2); - assert_eq!(latest_listing_object_quorum(2, 4, true, 0), 3); - - // The relaxed quorum never drops below the erasure read quorum, so a - // quorum-deleted object (at most one stale copy) stays hidden even - // with half the set unreachable. - let slack = latest_listing_write_quorum_slack(true, 4, 2); - assert_eq!(slack, 2); - assert_eq!(latest_listing_object_quorum(1, 4, true, slack), 2); - - assert_eq!(latest_listing_write_quorum_slack(false, 4, 2), 0); - } - - #[test] - fn latest_listing_resolves_degraded_object_when_a_set_drive_is_unreachable() { - let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]); - let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]); - - let slack = latest_listing_write_quorum_slack(true, 4, 3); - let obj_quorum = latest_listing_object_quorum(2, 4, true, slack); - assert_eq!(obj_quorum, 2); - let resolver = list_metadata_resolution_params("bucket".to_string(), 2, obj_quorum, false, slack); - let resolved = resolve_listing_entries(entries(), resolver, true) - .expect("object committed at write quorum should stay listable with one holder drive offline"); - assert_eq!(resolved.name, "object"); - - // Without the unreachable-drive slack the same sample is dropped even - // though the object still satisfies read quorum for GET. - let strict_quorum = latest_listing_object_quorum(2, 4, true, 0); - let strict_resolver = list_metadata_resolution_params("bucket".to_string(), 2, strict_quorum, false, 0); - assert!(resolve_listing_entries(entries(), strict_resolver, true).is_none()); - } - - #[test] - fn agreed_listing_entry_relaxes_write_quorum_by_unreachable_drives() { - let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]); - let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1); - - assert!(matches!( - resolve_agreed_listing_entry(entry.clone(), 2, &resolver, true), - ListingEntryResolution::Resolved(_) - )); - - resolver.write_quorum_slack = 0; - assert!(matches!( - resolve_agreed_listing_entry(entry, 2, &resolver, true), - ListingEntryResolution::NeedsSupplement(_) - )); - } - - #[test] - fn agreed_listing_entry_covers_quorum_corrupt_and_free_version_branches() { - let t1 = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("t1"); - let t2 = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("t2"); - let obj_entry = test_object_meta_entry_with_erasure_versions("object", &[(t1, "etag", 2, 2)]); - - // 1. enforce_write_quorum = false immediately resolves without inspecting quorum or metadata. - let strict_resolver = list_metadata_resolution_params("bucket".to_string(), 4, 4, false, 0); - assert!(matches!( - resolve_agreed_listing_entry(obj_entry.clone(), 1, &strict_resolver, false), - ListingEntryResolution::Resolved(_) - )); - - // 2. Directory entry resolves when reader_disks >= dir_quorum and rejects below dir_quorum. - let dir_entry = test_dir_meta_entry("prefix/"); - assert!(matches!( - resolve_agreed_listing_entry(dir_entry.clone(), 4, &strict_resolver, true), - ListingEntryResolution::Resolved(_) - )); - assert!(matches!( - resolve_agreed_listing_entry(dir_entry, 3, &strict_resolver, true), - ListingEntryResolution::Rejected - )); - - // 3. reader_disks < obj_quorum rejects before decoding xl.meta. - assert!(matches!( - resolve_agreed_listing_entry(obj_entry, 3, &strict_resolver, true), - ListingEntryResolution::Rejected - )); - - // 4. Corrupt xl.meta resolves when obj_quorum <= 1 and rejects when obj_quorum > 1. - let corrupt_entry = MetaCacheEntry { - name: "corrupt".to_string(), - metadata: b"not-valid-xl-meta".to_vec(), - cached: None, - reusable: false, - }; - let q1_resolver = list_metadata_resolution_params("bucket".to_string(), 1, 1, false, 0); - assert!(matches!( - resolve_agreed_listing_entry(corrupt_entry.clone(), 1, &q1_resolver, true), - ListingEntryResolution::Resolved(_) - )); - let q2_resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 0); - assert!(matches!( - resolve_agreed_listing_entry(corrupt_entry, 2, &q2_resolver, true), - ListingEntryResolution::Rejected - )); - - // 5. Latest version is a tier free-version record (`first.header.free_version() == true`), - // followed by a committed object version: the fast-path at index 0 is skipped because - // index 0 is a free version, and the loop selects the free version + first object version. - let mut transitioned = test_transitioned_meta_entry("transitioned-obj", "remote-key", true); - let mut fm = transitioned.cached.take().expect("cached transitioned meta"); - let older_obj = test_object_meta_entry_with_erasure_versions( - "transitioned-obj", - &[(t1 - time::Duration::seconds(10), "older-etag", 2, 2)], - ); - fm.versions.extend(older_obj.cached.expect("older cached").versions); - // Ensure free-version record is at index 0 (newest). - fm.versions[0].header.mod_time = Some(t2); - assert!(fm.versions[0].header.free_version()); - transitioned.metadata = fm.marshal_msg().expect("marshal free+object versions"); - transitioned.cached = None; - - // With reader_disks = 2, the index-0 free version satisfies quorum 2, so without the - // `!first.header.free_version()` guard the fast-path would falsely resolve; instead - // the loop inspects the 2+2 object version (write quorum 3) and requests a supplement. - assert!(matches!( - resolve_agreed_listing_entry(transitioned.clone(), 2, &q2_resolver, true), - ListingEntryResolution::NeedsSupplement(_) - )); - - let ListingEntryResolution::Resolved(resolved) = resolve_agreed_listing_entry(transitioned, 3, &q2_resolver, true) else { - panic!("expected Resolved for free-version + committed object"); - }; - let info = resolved.to_fileinfo("bucket").expect("decode resolved entry"); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("older-etag")); - - // 6. Only a free-version record exists (`selected_object_versions == 0` when `requested_versions == 1`): - // `selected_versions` has length == `cached.versions.len()`, so `resolve_agreed_listing_entry` returns Resolved. - let mut free_only = test_transitioned_meta_entry("free-only-obj", "remote-key", true); - free_only.cached = None; - assert!(matches!( - resolve_agreed_listing_entry(free_only, 3, &q2_resolver, true), - ListingEntryResolution::Resolved(_) - )); - } - - #[test] - fn cached_entry_supplement_check_honors_write_quorum_slack() { - let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let mut entry = test_object_meta_entry_with_erasure_versions("object", &[(mod_time, "etag", 2, 2)]); - entry.xl_meta_ref().expect("test entry should decode"); - let mut resolver = list_metadata_resolution_params("bucket".to_string(), 2, 2, false, 1); - - assert!(!entry.needs_quorum_supplement(2, &resolver).expect("valid metadata")); - - resolver.write_quorum_slack = 0; - assert!(entry.needs_quorum_supplement(2, &resolver).expect("valid metadata")); - } - - #[test] - fn latest_listing_object_quorum_requires_write_quorum_when_degraded_cannot_satisfy_it() { - let required_quorum = latest_listing_object_quorum(2, 8, true, 0); - let ask_disks = expand_ask_disks_for_object_quorum(4, 4, required_quorum); - - assert_eq!(required_quorum, 5); - assert_eq!(ask_disks, 4); - assert_eq!(latest_listing_object_quorum(2, 8, true, 0), 5); - } - #[tokio::test] - async fn latest_listing_supplements_a_split_delete_marker_sample() { - let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]); - let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time); - let entries = MetaCacheEntries(vec![ - Some(stale.clone()), - Some(stale.clone()), - Some(deleted.clone()), - Some(deleted.clone()), - ]); - let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0); - - assert_eq!( - listing_entries_supplement_target(&entries, &resolver, true), - Some(ListingSupplementTarget::Object("object".to_string())) - ); - assert_eq!(listing_entries_supplement_target(&entries, &resolver, false), None); - - let mut primary = resolve_listing_entries(MetaCacheEntries(entries.0.clone()), resolver.clone(), true) - .expect("the partial sample should fall back to the stale object version"); - assert!(!primary.is_latest_delete_marker()); - - let mut fallback_disks = Vec::new(); - let mut fallback_tempdirs = Vec::new(); - for _ in 0..2 { - let tempdir = tempfile::tempdir().expect("fallback tempdir should be created"); - let endpoint = - Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8")).expect("valid endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("fallback disk should be created"); - disk.make_volume("bucket").await.expect("fallback bucket should be created"); - disk.write_all( - "bucket", - &format!("object/{STORAGE_FORMAT_FILE}"), - bytes::Bytes::copy_from_slice(&deleted.metadata), - ) - .await - .expect("fallback metadata should be written"); - fallback_disks.push(disk); - fallback_tempdirs.push(tempdir); - } - let supplement = ListingSupplement::new( - ListingSupplementOptions { - bucket: "bucket".to_string(), - path: String::new(), - recursive: true, - incl_deleted: false, - skip_hidden_prefix_check: false, - filter_prefix: None, - forward_to: None, - per_disk_limit: 100, - skip_total_timeout: true, - walkdir_timeout: None, - walkdir_stall_timeout: None, - }, - Arc::new(fallback_disks), - FallbackClaimTracker::default(), - ); - - let mut supplemented = resolve_listing_entries_with_supplement(entries, resolver, true, supplement) - .await - .expect("supplemented metadata should not remain ambiguous") - .expect("the supplemented sample should resolve the committed delete marker"); - assert!(supplemented.is_latest_delete_marker()); - } - - #[tokio::test] - async fn latest_listing_fails_closed_for_unversioned_two_by_two_metadata_split() { - async fn collect_listing(set: &Arc, bucket: &str) -> (crate::error::Result, Vec) { - let (sender, mut receiver) = mpsc::channel(4); - let walk = set.list_path( - CancellationToken::new(), - ListPathOptions { - bucket: bucket.to_string(), - recursive: true, - ask_disks: "optimal".to_string(), - limit: 100, - ..Default::default() - }, - sender, - ); - let drain = async { - let mut names = Vec::new(); - while let Some(entry) = receiver.recv().await { - if !entry.is_dir() { - names.push(entry.name); - } - } - names - }; - - tokio::join!(walk, drain) - } - + async fn latest_listing_resolves_unversioned_two_by_two_split_to_the_newer_version() { let bucket = "list-two-by-two-overwrite"; let object = "object"; let old_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); @@ -7389,20 +6267,23 @@ mod test { .expect("split metadata should be written"); } - let (split_result, split_names) = collect_listing(&set, bucket).await; - assert!( - matches!(split_result, Err(StorageError::ErasureReadQuorum)), - "an unresolved 2+2 generation must fail the listing, got {split_result:?}" - ); - assert!(split_names.is_empty(), "an unresolved key must not be emitted from either generation"); + for mode in ["optimal", "strict"] { + let (split_result, split_entries) = collect_listing_etags(&set, bucket, mode).await; + split_result.expect("a 2+2 split must not fail the listing"); + assert_eq!( + split_entries, + [(object.to_string(), "new-etag".to_string())], + "{mode}: a tie lists the newer generation, as GET serves it" + ); + } disks[0] .write_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"), Bytes::copy_from_slice(&new.metadata)) .await .expect("third new-generation copy should be written"); - let (committed_result, committed_names) = collect_listing(&set, bucket).await; + let (committed_result, committed_entries) = collect_listing_etags(&set, bucket, "optimal").await; committed_result.expect("a 3+1 committed generation should list successfully"); - assert_eq!(committed_names, [object.to_string()]); + assert_eq!(committed_entries, [(object.to_string(), "new-etag".to_string())]); for disk in disks.iter().skip(1) { disk.delete( @@ -7417,375 +6298,335 @@ mod test { .await .expect("quorum deletion should remove the object copy"); } - let (deleted_result, deleted_names) = collect_listing(&set, bucket).await; + let (deleted_result, deleted_entries) = collect_listing_etags(&set, bucket, "optimal").await; deleted_result.expect("a quorum-deleted object should remain a successful empty listing"); - assert!(deleted_names.is_empty(), "one stale copy must not resurrect a quorum-deleted object"); + assert!(deleted_entries.is_empty(), "one stale copy must not resurrect a quorum-deleted object"); + } + + async fn write_listing_metadata( + dirs: &[tempfile::TempDir], + bucket: &str, + entry: &MetaCacheEntry, + slots: impl IntoIterator, + ) { + for slot in slots { + let path = dirs[slot].path().join(bucket).join(&entry.name); + tokio::fs::create_dir_all(&path).await.expect("create object directory"); + tokio::fs::write(path.join(STORAGE_FORMAT_FILE), &entry.metadata) + .await + .expect("write object metadata"); + } } #[tokio::test] - async fn latest_listing_supplement_checks_fallback_disks_for_common_prefix_quorum() { - let mut fallback_disks = Vec::new(); - let mut fallback_tempdirs = Vec::new(); - for index in 0..4 { - let tempdir = tempfile::tempdir().expect("fallback tempdir should be created"); - let endpoint = Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8")) - .expect("fallback endpoint should parse"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("fallback disk should be created"); - disk.make_volume("bucket").await.expect("fallback bucket should be created"); - for copies in [3, 4] { - if index < copies { - let object = format!("quux-{copies}/thud"); - let entry = test_object_meta_entry(&object); - disk.write_all("bucket", &format!("{object}/{STORAGE_FORMAT_FILE}"), bytes::Bytes::from(entry.metadata)) - .await - .expect("fallback child metadata should be written"); + async fn latest_listing_torn_write_visibility_follows_listing_quorum() { + // A torn write leaves the new generation on `copies` drives. The newest + // version carried by `ceil(asked/2)` asked drives is listed; otherwise a + // previous generation that reaches the quorum stays latest, and a key + // that reaches no quorum is omitted. The object's own erasure geometry + // never changes the rule, and no outcome fails the page. + let bucket = "torn-write-visibility"; + let old_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let new_time = old_time + time::Duration::seconds(100); + for (drives, parity) in [(4_usize, 2_usize), (16, 8), (16, 4)] { + let data = drives - parity; + let write_quorum = if data == parity { data + 1 } else { data }; + let half = drives.div_ceil(2); + let mut copy_counts = vec![write_quorum - 1, write_quorum, half - 1, half]; + copy_counts.sort_unstable(); + copy_counts.dedup(); + + let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks(drives, parity).await; + for dir in &dirs { + tokio::fs::create_dir_all(dir.path().join(bucket)) + .await + .expect("create bucket directory"); + } + let mut cases = Vec::new(); + for &copies in ©_counts { + for overwrite in [false, true] { + let name = format!("{}-{copies:02}", if overwrite { "overwrite" } else { "new" }); + let new = + test_unversioned_object_meta_entry_with_erasure(&name, new_time, "new", Uuid::from_u128(2), data, parity); + write_listing_metadata(&dirs, bucket, &new, 0..copies).await; + if overwrite { + let old = test_unversioned_object_meta_entry_with_erasure( + &name, + old_time, + "old", + Uuid::from_u128(1), + data, + parity, + ); + write_listing_metadata(&dirs, bucket, &old, copies..drives).await; + } + cases.push((name, copies, overwrite)); + } + } + cases.sort(); + + for online in [drives, half] { + let offline: Vec<_> = { + let mut disks = set.disks.write().await; + disks.iter_mut().skip(online).map(Option::take).collect() + }; + let quorum = online.div_ceil(2); + let expected: Vec<(String, String)> = cases + .iter() + .filter_map(|(name, copies, overwrite)| { + let new_copies = (*copies).min(online); + let old_copies = if *overwrite { online - new_copies } else { 0 }; + if new_copies >= quorum { + Some((name.clone(), "new".to_owned())) + } else if old_copies >= quorum { + Some((name.clone(), "old".to_owned())) + } else { + None + } + }) + .collect(); + // A sampling mode is deterministic only when it asks every online drive. + let modes: &[&str] = if drives == 4 || online == half { + &["strict", "optimal"] + } else { + &["strict"] + }; + for mode in modes { + let context = format!("{drives} drives EC {data}+{parity}, {online} online, {mode}"); + let (result, listed) = collect_listing_etags(&set, bucket, mode).await; + result.unwrap_or_else(|err| panic!("{context}: a torn write must not fail the page: {err:?}")); + assert_eq!(listed, expected, "{context}"); + } + let mut disks = set.disks.write().await; + for (slot, disk) in disks.iter_mut().skip(online).zip(offline) { + *slot = disk; } } - fallback_disks.push(disk); - fallback_tempdirs.push(tempdir); } - let supplement = ListingSupplement::new( - ListingSupplementOptions { - bucket: "bucket".to_owned(), - path: String::new(), - recursive: false, - incl_deleted: false, - skip_hidden_prefix_check: false, - filter_prefix: None, - forward_to: None, - per_disk_limit: 100, - skip_total_timeout: true, - walkdir_timeout: None, - walkdir_stall_timeout: None, - }, - Arc::new(fallback_disks), - FallbackClaimTracker::default(), - ); - // A 16-drive EC:4 set asks 12 primary disks. A committed write may - // exist on eight primary disks and all four remaining fallback disks. - let resolver = list_metadata_resolution_params("bucket".to_owned(), 4, 12, false, 0); - for fallback_copies in [3, 4] { - let prefix = format!("quux-{fallback_copies}/"); - let mut primary = vec![Some(test_dir_meta_entry(&prefix)); 8]; - primary.extend([None, None, None, None]); - let entry = - resolve_listing_entries_with_supplement(MetaCacheEntries(primary), resolver.clone(), true, supplement.clone()) + } + + #[tokio::test] + async fn latest_listing_and_get_agree_on_hermetic_set_states() { + #[derive(Clone, Copy)] + enum Generation { + Old, + New, + Corrupt, + } + + async fn put_generation( + set: &SetDisks, + dirs: &[tempfile::TempDir], + bucket: &str, + object: &str, + body: &[u8], + ) -> (String, Vec>) { + let mut reader = PutObjReader::from_vec(body.to_vec()); + let info = set + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("generation PUT should commit on every drive"); + let mut metadata = Vec::with_capacity(dirs.len()); + for dir in dirs { + metadata.push( + tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("generation metadata should carry the inline shard"), + ); + } + (info.etag.expect("PUT should report an etag"), metadata) + } + + let bucket = "list-get-consistency"; + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + let committed = "a-committed"; + let (old_etag, old) = put_generation(&set, &dirs, bucket, committed, b"old generation").await; + let (new_etag, new) = put_generation(&set, &dirs, bucket, committed, b"new generation").await; + assert_ne!(old_etag, new_etag); + + let states = [ + ( + "b-copied", + [ + Some(Generation::New), + Some(Generation::New), + Some(Generation::New), + Some(Generation::New), + ], + ), + ( + "c-split", + [ + Some(Generation::Old), + Some(Generation::Old), + Some(Generation::New), + Some(Generation::New), + ], + ), + ("d-torn-two", [Some(Generation::New), Some(Generation::New), None, None]), + ("e-torn-one", [Some(Generation::New), None, None, None]), + ("f-stale-one", [None, None, None, Some(Generation::Old)]), + ( + "g-overwrite-torn-one", + [ + Some(Generation::New), + Some(Generation::Old), + Some(Generation::Old), + Some(Generation::Old), + ], + ), + ( + "h-corrupt-majority", + [ + Some(Generation::New), + Some(Generation::Corrupt), + Some(Generation::Corrupt), + Some(Generation::Corrupt), + ], + ), + ]; + for (name, copies) in &states { + for (slot, copy) in copies.iter().enumerate() { + let Some(copy) = copy else { + continue; + }; + let path = dirs[slot].path().join(bucket).join(name); + tokio::fs::create_dir_all(&path).await.expect("create state directory"); + let bytes: &[u8] = match copy { + Generation::Old => &old[slot], + Generation::New => &new[slot], + Generation::Corrupt => b"corrupt xl.meta", + }; + tokio::fs::write(path.join(STORAGE_FORMAT_FILE), bytes) .await - .expect("directory supplementation should not fail"); - assert_eq!( - entry.map(|entry| entry.name), - (fallback_copies == 4).then_some(prefix), - "the common prefix needs all twelve copies, including fallback disks" + .expect("write state metadata"); + } + } + + let (result, listed) = collect_listing_etags(&set, bucket, "strict").await; + result.expect("LIST must not fail because one object is unresolved or corrupt"); + let expected: Vec<(String, String)> = [ + ("a-committed", &new_etag), + ("b-copied", &new_etag), + ("c-split", &new_etag), + ("d-torn-two", &new_etag), + ("g-overwrite-torn-one", &old_etag), + ] + .into_iter() + .map(|(name, etag)| (name.to_owned(), etag.clone())) + .collect(); + assert_eq!(listed, expected); + + for name in std::iter::once(committed).chain(states.iter().map(|(name, _)| *name)) { + let listed_etag = listed.iter().find(|(listed, _)| listed == name).map(|(_, etag)| etag); + let get = set + .get_object_reader(bucket, name, None, HeaderMap::new(), &ObjectOptions::default()) + .await; + match (listed_etag, get) { + (Some(listed_etag), Ok(mut reader)) => { + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("listed object should stream"); + assert_eq!( + reader.object_info.etag.as_ref(), + Some(listed_etag), + "{name}: GET must serve the version LIST selected" + ); + } + // Known divergence: when every drive answers, GET of the latest + // version demands write quorum (three of four), so a 2+2 split + // answers 503 although LIST shows the newer generation, as + // MinIO's listing does. F.0's dangling classification does not + // cover a split; a GET change is a separate decision. + (Some(_), Err(StorageError::InsufficientReadQuorum(..))) if name == "c-split" => {} + (Some(_), Err(err)) => assert!( + matches!( + err, + StorageError::ObjectNotFound(..) | StorageError::FileNotFound | StorageError::VersionNotFound(..) + ), + "{name}: a listed key may only be 200 or 404, got {err:?}" + ), + (None, Ok(reader)) => panic!("{name}: GET serves {:?} that LIST omitted", reader.object_info.etag), + // Majority-corrupt metadata surfaces as `FileCorrupt`, an + // unavailability error like the read-quorum variants. + (None, Err(err)) => assert!( + matches!( + err, + StorageError::ObjectNotFound(..) + | StorageError::FileNotFound + | StorageError::ErasureReadQuorum + | StorageError::InsufficientReadQuorum(..) + | StorageError::FileCorrupt + ), + "{name}: an omitted key must be 404 or unavailable, got {err:?}" + ), + } + } + } + + #[tokio::test] + async fn latest_listing_resolves_concurrent_overwrite_split_without_object_lock() { + let bucket = "concurrent-overwrite-split"; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await; + let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(16, 8, ctx).await; + for dir in &dirs { + tokio::fs::create_dir_all(dir.path().join(bucket)) + .await + .expect("create bucket directory"); + } + let old_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let new_time = old_time + time::Duration::seconds(1); + // old|new copies of an overwrite that is still publishing. + let mut expected = Vec::new(); + for (old_copies, latest) in [(5, "new"), (8, "new"), (11, "old")] { + let name = format!("split-{old_copies:02}-{:02}", 16 - old_copies); + let old = test_unversioned_object_meta_entry_with_erasure(&name, old_time, "old", Uuid::from_u128(1), 8, 8); + let new = test_unversioned_object_meta_entry_with_erasure(&name, new_time, "new", Uuid::from_u128(2), 8, 8); + write_listing_metadata(&dirs, bucket, &old, 0..old_copies).await; + write_listing_metadata(&dirs, bucket, &new, old_copies..16).await; + expected.push((name, latest.to_owned())); + } + // A delete marker that half the set already carries hides the key. + let tombstone = "tombstone-08-08"; + let object = test_object_meta_entry_with_erasure_versions(tombstone, &[(old_time, "object-etag", 4, 2)]); + let marked = test_object_with_delete_marker_meta_entry(tombstone, old_time, new_time); + write_listing_metadata(&dirs, bucket, &object, 0..8).await; + write_listing_metadata(&dirs, bucket, &marked, 8..16).await; + expected.push((tombstone.to_owned(), "delete-marker".to_owned())); + + // Hold every key's write lock the way an in-flight overwrite does. + let mut guards = Vec::new(); + for (name, _) in &expected { + guards.push( + set.new_ns_lock(bucket, name) + .await + .expect("namespace lock should be created") + .get_write_lock(Duration::from_secs(5)) + .await + .expect("writer lock should be acquired"), ); } - } - - #[test] - fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() { - let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let delete_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let stale = test_object_meta_entry_with_erasure_versions("object", &[(object_mod_time, "object-etag", 4, 2)]); - let deleted = test_object_with_delete_marker_meta_entry("object", object_mod_time, delete_mod_time); - let resolver = list_metadata_resolution_params("bucket".to_string(), 2, 4, false, 0); - let entries = MetaCacheEntries(vec![ - Some(stale.clone()), - Some(stale.clone()), - Some(stale.clone()), - Some(deleted.clone()), - ]); - - assert_eq!( - listing_entries_supplement_target(&entries, &resolver, true), - Some(ListingSupplementTarget::Object("object".to_string())) - ); - - let mut resolved = resolve_listing_entries( - MetaCacheEntries(vec![ - Some(stale.clone()), - Some(stale.clone()), - Some(stale), - Some(deleted.clone()), - Some(deleted.clone()), - Some(deleted), - ]), - resolver, - true, - ) - .expect("the previous object version should remain visible below delete-marker write quorum"); - - assert!(!resolved.is_latest_delete_marker()); - } - - #[tokio::test] - async fn latest_listing_agreed_path_requires_enough_readers_for_write_quorum() { - let entry = test_object_meta_entry("object"); - let seen = Arc::new(Mutex::new(Vec::new())); - let seen_clone = seen.clone(); - let allow_agreed_objects = latest_listing_allow_agreed_objects(true, 4, 5); - let raw_min_disks = latest_listing_raw_min_disks(2, 5, true); - - assert!(!allow_agreed_objects); - assert_eq!(raw_min_disks, 5); - let err = list_path_raw( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None, None, None], - min_disks: raw_min_disks, - test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry.clone()]); 4], - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let seen = seen_clone.clone(); - Box::pin(async move { - if !allow_agreed_objects && !entry.is_dir() { - return Ok(()); - } - seen.lock().expect("seen mutex poisoned").push(entry.name); - Ok(()) - }) - })), - ..Default::default() - }, - ) - .await - .expect_err("latest listing should fail when selected readers cannot satisfy write quorum"); - - assert_eq!(err, DiskError::ErasureReadQuorum); - assert!(seen.lock().expect("seen mutex poisoned").is_empty()); - - let seen = Arc::new(Mutex::new(Vec::new())); - let seen_clone = seen.clone(); - let allow_agreed_objects = latest_listing_allow_agreed_objects(true, 5, 5); - let raw_min_disks = latest_listing_raw_min_disks(3, 5, true); - - assert!(allow_agreed_objects); - assert_eq!(raw_min_disks, 5); - list_path_raw( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None, None, None, None], - min_disks: raw_min_disks, - test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry]); 5], - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let seen = seen_clone.clone(); - Box::pin(async move { - if !allow_agreed_objects && !entry.is_dir() { - return Ok(()); - } - seen.lock().expect("seen mutex poisoned").push(entry.name); - Ok(()) - }) - })), - ..Default::default() - }, - ) - .await - .expect("write-quorum all-agree listing should complete"); - - assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["object".to_string()]); - } - - #[tokio::test] - async fn latest_listing_agreed_path_reconciles_low_parity_partial_latest() { - let old_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let new_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let entry = test_object_meta_entry_with_erasure_versions( - "object", - &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)], - ); - let fallback_old_entry = test_object_meta_entry_with_erasure_versions("object", &[(old_mod_time, "old-etag", 4, 4)]); - let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0); - let seen = Arc::new(Mutex::new(Vec::new())); - let seen_clone = seen.clone(); - - list_path_raw( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None, None, None, None], - min_disks: 5, - test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry]); 5], - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let seen = seen_clone.clone(); - let resolver = resolver.clone(); - let fallback_old_entry = fallback_old_entry.clone(); - Box::pin(async move { - let entry = match resolve_agreed_listing_entry(entry, 5, &resolver, true) { - ListingEntryResolution::Resolved(entry) => entry, - ListingEntryResolution::NeedsSupplement(entry) => { - let mut candidates = vec![Some(entry); 5]; - candidates.extend(vec![Some(fallback_old_entry); 2]); - let Some(entry) = resolve_listing_entries(MetaCacheEntries(candidates), resolver, true) else { - return Ok(()); - }; - entry - } - ListingEntryResolution::Rejected => return Ok(()), - }; - let info = entry.to_fileinfo("bucket").expect("resolved entry should decode"); - seen.lock().expect("seen mutex poisoned").push(( - entry.name, - info.mod_time, - info.metadata.get("etag").cloned(), - )); - Ok(()) - }) - })), - ..Default::default() - }, - ) - .await - .expect("all-agree listing should complete"); - - assert_eq!( - seen.lock().expect("seen mutex poisoned").as_slice(), - &[("object".to_string(), Some(old_mod_time), Some("old-etag".to_string()))] - ); - } - - #[tokio::test] - async fn latest_listing_agreed_path_falls_back_to_previous_version_without_supplement() { - let old_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let new_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let entry = test_object_meta_entry_with_erasure_versions( - "object", - &[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)], - ); - let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0); - let seen = Arc::new(Mutex::new(Vec::new())); - let seen_clone = seen.clone(); - - list_path_raw( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None, None, None, None], - min_disks: 5, - test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry]); 5], - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let seen = seen_clone.clone(); - let resolver = resolver.clone(); - Box::pin(async move { - let entry = match resolve_agreed_listing_entry(entry, 5, &resolver, true) { - ListingEntryResolution::Resolved(entry) => entry, - ListingEntryResolution::NeedsSupplement(entry) => { - let supplement = ListingSupplement::new( - ListingSupplementOptions { - bucket: "bucket".to_string(), - path: String::new(), - recursive: true, - incl_deleted: false, - skip_hidden_prefix_check: false, - filter_prefix: None, - forward_to: None, - per_disk_limit: 0, - skip_total_timeout: true, - walkdir_timeout: None, - walkdir_stall_timeout: None, - }, - Arc::new(Vec::new()), - FallbackClaimTracker::default(), - ); - super::resolve_agreed_listing_entry_with_supplement(entry, 5, resolver, supplement) - .await? - .expect("the previous committed version must resolve without extra disks") - } - ListingEntryResolution::Rejected => return Ok(()), - }; - let info = entry.to_fileinfo("bucket").expect("resolved entry should decode"); - seen.lock().expect("seen mutex poisoned").push(( - entry.name, - info.mod_time, - info.metadata.get("etag").cloned(), - )); - Ok(()) - }) - })), - ..Default::default() - }, - ) - .await - .expect("all-agree listing should complete"); - - assert_eq!( - seen.lock().expect("seen mutex poisoned").as_slice(), - &[("object".to_string(), Some(old_mod_time), Some("old-etag".to_string()))] - ); - } - - #[tokio::test] - async fn latest_listing_agreed_path_supplements_committed_low_parity_latest() { - let new_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let entry = test_object_meta_entry_with_erasure_versions("object", &[(new_mod_time, "new-etag", 7, 1)]); - let fallback_entry = entry.clone(); - let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 5, false, 0); - let seen = Arc::new(Mutex::new(Vec::new())); - let seen_clone = seen.clone(); - - list_path_raw( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None, None, None, None], - min_disks: 5, - test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry]); 5], - agreed: Some(Box::new(move |entry: MetaCacheEntry| { - let seen = seen_clone.clone(); - let resolver = resolver.clone(); - let fallback_entry = fallback_entry.clone(); - Box::pin(async move { - let ListingEntryResolution::NeedsSupplement(entry) = - resolve_agreed_listing_entry(entry, 5, &resolver, true) - else { - return Ok(()); - }; - let mut candidates = vec![Some(entry); 5]; - candidates.extend(vec![Some(fallback_entry); 2]); - let resolved = resolve_listing_entries(MetaCacheEntries(candidates), resolver, true) - .expect("supplemented low-parity latest should satisfy object write quorum"); - let info = resolved.to_fileinfo("bucket").expect("resolved entry should decode"); - seen.lock().expect("seen mutex poisoned").push(( - resolved.name, - info.mod_time, - info.metadata.get("etag").cloned(), - )); - Ok(()) - }) - })), - ..Default::default() - }, - ) - .await - .expect("all-agree listing should complete"); - - assert_eq!( - seen.lock().expect("seen mutex poisoned").as_slice(), - &[("object".to_string(), Some(new_mod_time), Some("new-etag".to_string()))] - ); - } - - #[tokio::test] - async fn latest_listing_agreed_directory_requires_write_quorum() { - let entry = test_dir_meta_entry("prefix/"); - let raw_min_disks = latest_listing_raw_min_disks(2, 5, true); - - let err = list_path_raw( - CancellationToken::new(), - ListPathRawOptions { - disks: vec![None, None, None, None], - min_disks: raw_min_disks, - test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![entry]); 4], - agreed: Some(Box::new(move |_| Box::pin(async { Ok(()) }))), - ..Default::default() - }, - ) - .await - .expect_err("latest listing should not expose prefixes below write quorum"); - - assert_eq!(err, DiskError::ErasureReadQuorum); + let (result, listed) = timeout(Duration::from_secs(5), collect_listing_etags(&set, bucket, "strict")) + .await + .expect("LIST must not wait for an object lock"); + drop(guards); + result.expect("a split generation must not fail the listing"); + assert_eq!(listed, expected); } #[test] @@ -8463,11 +7304,12 @@ mod test { let marked = test_object_with_delete_marker_meta_entry("key", time, time + time::Duration::seconds(1)); let mut inputs = Vec::new(); for marker_copies in [1, 2] { - let resolver = list_metadata_resolution_params("bucket".to_string(), 3, 3, true, 0); + let resolver = list_metadata_resolution_params("bucket".to_string(), 3, true); let copies = (0..3) .map(|index| Some(if index < marker_copies { marked.clone() } else { old.clone() })) .collect(); - let entry = resolve_listing_entries(MetaCacheEntries(copies), resolver, false) + let entry = MetaCacheEntries(copies) + .resolve(resolver) .expect("each set independently retains its quorum-backed history"); inputs.push(entry); } diff --git a/crates/ecstore/src/store/list_objects/differential_tests.rs b/crates/ecstore/src/store/list_objects/differential_tests.rs index 55ad393b0..11085b633 100644 --- a/crates/ecstore/src/store/list_objects/differential_tests.rs +++ b/crates/ecstore/src/store/list_objects/differential_tests.rs @@ -14,14 +14,14 @@ use super::*; use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; -use crate::disk::STORAGE_FORMAT_FILE; +use crate::disk::{DiskAPI, STORAGE_FORMAT_FILE, WalkDirOptions}; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; use bytes::Bytes; -use rustfs_filemeta::{FileInfo, ObjectPartInfo}; +use rustfs_filemeta::{FileInfo, MetacacheReader, ObjectPartInfo}; use std::collections::BTreeMap; use std::pin::Pin; use std::task::{Context, Poll}; -use tokio::io::{AsyncWrite, DuplexStream}; +use tokio::io::{AsyncWrite, DuplexStream, duplex}; use tokio::sync::Notify; use tokio::time::timeout; @@ -468,6 +468,50 @@ async fn list_objects_shared_corpus_offline_corrupt_and_minority_stale() { fixture.assert_walk("", None, None, 3).await; } +#[tokio::test] +#[serial_test::serial] +async fn list_objects_shared_corpus_quorum_delete_leaves_no_resurrectable_stale_copy() { + let mut fixture = Fixture::new("list-corpus-stale-copies", namespace(&shared_keys()), true).await; + // A delete that reached write quorum (three of four drives) leaves one + // stale copy, the most any quorum delete can leave on this set. + let deleted = ["a0", "z"]; + for key in deleted { + let physical_key = rustfs_utils::path::encode_dir_object(key); + for dir in fixture.dirs.iter().take(3) { + tokio::fs::remove_dir_all(dir.path().join(&fixture.bucket).join(&physical_key)) + .await + .expect("quorum delete fixture should remove the object directory"); + } + } + fixture.objects.retain(|object| !deleted.contains(&object.key.as_str())); + + temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("strict"))], async { + for max in [1, 3, 1000] { + fixture.assert_walk("", None, None, max).await; + fixture.assert_walk("", Some("/"), None, max).await; + } + }) + .await; + + // A sampling mode promises only that no committed key goes missing. + temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("optimal"))], async { + let mut listed = Vec::new(); + let mut marker = None; + for _ in 0..=fixture.objects.len() + deleted.len() { + let page = fixture.page("", marker.clone(), None, 3).await; + listed.extend(page.objects.into_iter().map(|object| object.name)); + if !page.is_truncated { + break; + } + marker = page.next_marker; + } + for object in &fixture.objects { + assert!(listed.contains(&object.key), "optimal LIST lost committed key {}", object.key); + } + }) + .await; +} + #[tokio::test] async fn list_objects_shared_corpus_versions_delete_markers_and_free_versions() { let mut objects = vec![ diff --git a/crates/filemeta/benches/xl_meta_bench.rs b/crates/filemeta/benches/xl_meta_bench.rs index bb2aced39..fdcf04c74 100644 --- a/crates/filemeta/benches/xl_meta_bench.rs +++ b/crates/filemeta/benches/xl_meta_bench.rs @@ -446,7 +446,6 @@ fn bench_metacache_resolve(c: &mut Criterion) { requested_versions: 1, bucket: "bucket".to_string(), strict: false, - write_quorum_slack: 0, candidates: Vec::new(), }); black_box(resolved); @@ -463,24 +462,6 @@ fn bench_metacache_resolve(c: &mut Criterion) { requested_versions: 1, bucket: "bucket".to_string(), strict: false, - write_quorum_slack: 0, - candidates: Vec::new(), - }); - black_box(resolved); - } - }); - }); - - group.bench_function("8_disks_write_quorum/1000_objects", |b| { - b.iter(|| { - for entries in &sets_8_agree { - let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams { - dir_quorum: 4, - obj_quorum: 4, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }); black_box(resolved); @@ -491,13 +472,12 @@ fn bench_metacache_resolve(c: &mut Criterion) { group.bench_function("8_disks_disagree_merge/1000_objects", |b| { b.iter(|| { for entries in &sets_8_disagree { - let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams { + let resolved = entries.resolve(MetadataResolutionParams { dir_quorum: 4, obj_quorum: 4, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }); black_box(resolved); @@ -508,13 +488,12 @@ fn bench_metacache_resolve(c: &mut Criterion) { group.bench_function("8_disks_agree_13_versions/1000_objects", |b| { b.iter(|| { for entries in &sets_8_agree_13v { - let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams { + let resolved = entries.resolve(MetadataResolutionParams { dir_quorum: 4, obj_quorum: 4, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }); black_box(resolved); @@ -525,13 +504,12 @@ fn bench_metacache_resolve(c: &mut Criterion) { group.bench_function("8_disks_disagree_13_versions/1000_objects", |b| { b.iter(|| { for entries in &sets_8_disagree_13v { - let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams { + let resolved = entries.resolve(MetadataResolutionParams { dir_quorum: 4, obj_quorum: 4, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }); black_box(resolved); @@ -596,191 +574,6 @@ fn bench_metacache_resolve(c: &mut Criterion) { group.finish(); } -fn listing_history(versions: usize, cached: bool) -> MetaCacheEntry { - let mut metadata = FileMeta::new(); - for index in 0..versions { - let (data, parity) = if index + 1 == versions { (12, 4) } else { (8, 8) }; - let mut info = FileInfo::new("object", data, parity); - info.volume = "bucket".to_owned(); - info.erasure.index = 1; - info.data_dir = Some(Uuid::from_u128(0x1000 + u128::try_from(index).expect("data directory index"))); - info.versioned = true; - info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("version index"))); - info.mod_time = Some( - OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("timestamp")).expect("fixture time"), - ); - info.size = 1; - info.parts = vec![ObjectPartInfo { - number: 1, - size: 1, - actual_size: 1, - ..Default::default() - }]; - info.metadata.insert("etag".to_owned(), format!("etag-{index}")); - info.metadata.insert("x-amz-meta-description".to_owned(), "m".repeat(1024)); - metadata.add_version(info).expect("valid version history"); - } - MetaCacheEntry { - name: "object".to_owned(), - metadata: metadata.marshal_msg().expect("serialize history"), - cached: cached.then_some(metadata), - reusable: false, - } -} - -#[inline] -fn owned_agreed_latest_reference( - mut entry: MetaCacheEntry, - replicas: usize, - params: &rustfs_filemeta::MetadataResolutionParams, - enforce_write_quorum: bool, -) -> rustfs_filemeta::MetaCacheEntryResolution { - use rustfs_filemeta::MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved}; - if !enforce_write_quorum { - return Resolved(entry); - } - if entry.is_dir() { - return if replicas >= params.dir_quorum { - Resolved(entry) - } else { - Rejected - }; - } - if replicas < params.obj_quorum { - return Rejected; - } - let metadata = match entry.xl_meta_ref() { - Ok(metadata) => metadata, - Err(_) if params.obj_quorum <= 1 => return Resolved(entry), - Err(_) => return Rejected, - }; - if params.requested_versions == 1 - && let Some(first) = metadata.versions.first() - && !first.header.free_version() - && replicas >= first.listing_quorum(params.obj_quorum, params.write_quorum_slack) - { - return Resolved(entry); - } - // These fixtures cover the fast prefix, never older-version reconciliation. - NeedsSupplement(entry) -} - -fn bench_listing_borrowed_metadata(c: &mut Criterion) { - use rustfs_filemeta::{FileMetaVersion, MetaCacheEntries, MetadataResolutionParams}; - let mut group = c.benchmark_group("listing_borrowed_metadata"); - group - .sample_size(20) - .warm_up_time(Duration::from_millis(150)) - .measurement_time(Duration::from_millis(700)); - for versions in [2, 16, 256] { - for cached in [false, true] { - let entry = listing_history(versions, cached); - let params = MetadataResolutionParams { - dir_quorum: 8, - obj_quorum: 8, - requested_versions: 1, - write_quorum_slack: 8, - bucket: "bucket".to_owned(), - ..Default::default() - }; - let slots = vec![Some(&entry); 8]; - let owned = MetaCacheEntries(vec![Some(entry.clone()); 8]) - .resolve_with_write_quorum(params.clone()) - .expect("previous committed version"); - let borrowed = - MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()).expect("borrowed previous version"); - assert_eq!(owned.metadata, borrowed.metadata); - let id = format!("{versions}_versions/{}", if cached { "cached" } else { "cold" }); - group.throughput(Throughput::Bytes(u64::try_from(entry.metadata.len()).expect("metadata size"))); - // ABBA order permits an explicit baseline-drift check for each fixture. - for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] { - group.bench_function(BenchmarkId::new("replicas", format!("{id}/{phase}")), |b| { - b.iter(|| { - if phase.starts_with("owned") { - black_box( - MetaCacheEntries(vec![Some(black_box(&entry).clone()); 8]) - .resolve_with_write_quorum(params.clone()), - ) - } else { - let slots = vec![Some(black_box(&entry)); 8]; - black_box(MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone())) - } - }); - }); - } - let mut agreed_metadata = entry - .cached - .clone() - .unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("agreed fixture")); - let mut latest = FileMetaVersion::try_from(agreed_metadata.versions[0].meta.as_slice()).expect("latest body"); - let object = latest.object.as_mut().expect("latest object"); - object.erasure_m = 8; - object.erasure_n = 8; - agreed_metadata.versions[0] = latest.try_into().expect("balanced latest header"); - let agreed = MetaCacheEntry { - name: entry.name.clone(), - metadata: agreed_metadata.marshal_msg().expect("agreed history"), - cached: cached.then_some(agreed_metadata), - reusable: false, - }; - for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] { - group.bench_function(BenchmarkId::new("agreed", format!("{id}/{phase}")), |b| { - b.iter_batched( - || agreed.clone(), - |candidate| { - let resolved = if phase.starts_with("owned") { - owned_agreed_latest_reference(candidate, 8, ¶ms, true) - } else { - rustfs_filemeta::resolve_agreed_entry(candidate, 8, ¶ms, true) - }; - let rustfs_filemeta::MetaCacheEntryResolution::Resolved(candidate) = resolved else { - panic!("balanced latest must use agreed fast prefix"); - }; - black_box(candidate) - }, - BatchSize::SmallInput, - ); - }); - } - let replicas: Vec<_> = (1..=8) - .map(|disk| { - let mut metadata = entry - .cached - .clone() - .unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("decode fixture")); - for version in &mut metadata.versions { - let mut payload = FileMetaVersion::try_from(version.meta.as_slice()).expect("version body"); - payload.object.as_mut().expect("object body").erasure_index = disk; - version.meta = payload.marshal_msg().expect("per-disk body"); - } - MetaCacheEntry { - name: entry.name.clone(), - metadata: metadata.marshal_msg().expect("per-disk metadata"), - cached: cached.then_some(metadata), - reusable: false, - } - }) - .collect(); - assert!(replicas[0].headers_agree(&replicas, false)); - for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] { - group.bench_function(BenchmarkId::new("headers", format!("{id}/{phase}")), |b| { - b.iter(|| { - let agreed = if phase.starts_with("owned") { - replicas - .iter() - .all(|candidate| replicas[0].matches(Some(black_box(candidate)), false).1) - } else { - replicas[0].headers_agree(black_box(&replicas), false) - }; - black_box(agreed) - }); - }); - } - } - } - group.finish(); -} - criterion_group!( benches, bench_create_real_xlmeta, @@ -797,8 +590,7 @@ criterion_group!( bench_list_page_to_fileinfo, bench_delete_inline_versions, bench_metacache_stream_page, - bench_metacache_resolve, - bench_listing_borrowed_metadata + bench_metacache_resolve ); criterion_main!(benches); diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 117673e9e..7ffbad04e 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -155,7 +155,6 @@ pub fn is_skip_meta_key(key: &str) -> bool { } mod codec; -pub(crate) use codec::{BorrowedFileMeta, reconcile_headers}; mod inline_data; pub(crate) mod msgp_decode; mod validation; diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index fd66a5f07..49af79a94 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -622,18 +622,6 @@ pub struct FileMetaShallowVersion { pub meta: Vec, // FileMetaVersion.marshal_msg } -fn write_quorum_from_erasure(data_blocks: usize, parity_blocks: usize) -> Option { - if data_blocks == 0 { - return None; - } - - Some(if data_blocks == parity_blocks { - data_blocks.saturating_add(1) - } else { - data_blocks - }) -} - impl FileMetaShallowVersion { /// Parse version meta with legacy format compatibility. /// Use this instead of `FileMetaVersion::default()` + `unmarshal_msg()` to handle old-version xl.meta. @@ -641,60 +629,11 @@ impl FileMetaShallowVersion { FileMetaVersion::try_from(self.meta.as_slice()) } - pub fn write_quorum(&self, fallback_quorum: usize) -> usize { - write_quorum_from_header_and_meta(&self.header, self.meta.as_slice(), fallback_quorum) - } - - /// Quorum for listing this version without accepting an unreadable payload. - pub fn listing_quorum(&self, fallback_quorum: usize, unreachable_disks: usize) -> usize { - listing_quorum_from_header_and_meta(&self.header, &self.meta, fallback_quorum, unreachable_disks) - } - pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result { FileMetaVersion::decode_fileinfo_from_slice(self.meta.as_slice(), volume, path, all_parts, true) } } -#[inline] -pub(crate) fn write_quorum_from_header_and_meta(header: &FileMetaVersionHeader, meta: &[u8], fallback_quorum: usize) -> usize { - let header_quorum = header.write_quorum(fallback_quorum); - if header.has_ec() || !matches!(header.version_type, VersionType::Object | VersionType::Legacy) { - return header_quorum; - } - - FileMetaVersion::try_from(meta) - .ok() - .and_then(|version| version.erasure_write_quorum()) - .unwrap_or(header_quorum) -} - -pub(crate) fn listing_quorum_from_header_and_meta( - header: &FileMetaVersionHeader, - meta: &[u8], - fallback_quorum: usize, - unreachable_disks: usize, -) -> usize { - let geometry = if header.version_type == VersionType::Object && header.has_ec() { - Some((usize::from(header.ec_m), usize::from(header.ec_n))) - } else if matches!(header.version_type, VersionType::Object | VersionType::Legacy) { - FileMetaVersion::try_from(meta) - .ok() - .and_then(|version| version.erasure_geometry()) - } else { - None - }; - let Some((data_blocks, parity_blocks)) = geometry else { - return fallback_quorum; - }; - let Some(write_quorum) = write_quorum_from_erasure(data_blocks, parity_blocks) else { - return fallback_quorum; - }; - write_quorum - .saturating_sub(unreachable_disks) - .max(data_blocks) - .max(fallback_quorum) -} - impl TryFrom for FileMetaShallowVersion { type Error = Error; @@ -737,26 +676,6 @@ pub(super) struct V2ObjectSliceSummary { } impl FileMetaVersion { - fn erasure_geometry(&self) -> Option<(usize, usize)> { - Some(match self.version_type { - VersionType::Object | VersionType::Legacy => { - if let Some(object) = &self.object { - (object.erasure_m, object.erasure_n) - } else if let Some(object) = &self.legacy_object { - (object.erasure.data_blocks, object.erasure.parity_blocks) - } else { - return None; - } - } - _ => return None, - }) - } - - fn erasure_write_quorum(&self) -> Option { - let (data_blocks, parity_blocks) = self.erasure_geometry()?; - write_quorum_from_erasure(data_blocks, parity_blocks) - } - /// Fast borrowed scanner for `V2Obj` slices. Returning `Ok` guarantees /// `FileMetaVersion::try_from` would succeed with the identical summary fields; /// any non-`V2Obj` or invalid input returns `Err` so callers fall back to `try_from`. @@ -1775,14 +1694,6 @@ impl FileMetaVersionHeader { self.ec_m > 0 && self.ec_n > 0 } - pub fn write_quorum(&self, fallback_quorum: usize) -> usize { - if self.version_type != VersionType::Object || !self.has_ec() { - return fallback_quorum; - } - - write_quorum_from_erasure(usize::from(self.ec_m), usize::from(self.ec_n)).unwrap_or(fallback_quorum) - } - pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool { let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o); // Disk-loaded headers keep the null version as Some(nil), not None: all null @@ -4167,15 +4078,49 @@ pub fn merge_file_meta_versions( .iter() .map(|v| v.iter().map(|ver| (&ver.header, ver.meta.as_slice())).collect()) .collect(); - merge_file_meta_borrowed_streams(quorum, strict, requested_versions, false, 0, streams) + merge_file_meta_borrowed_streams(quorum, strict, requested_versions, streams) +} + +/// Picks the most common header among the tops that share `latest`'s version +/// id; a tie keeps the header that sorts first (the newest). +fn most_common_header_for_version_id<'a>( + tops: &[(&'a FileMetaVersionHeader, &'a [u8])], + mut latest: (&'a FileMetaVersionHeader, &'a [u8]), + strict: bool, +) -> ((&'a FileMetaVersionHeader, &'a [u8]), usize) { + let version_id = latest.0.version_id; + let comparable = |header: &FileMetaVersionHeader| { + let mut header = header.clone(); + if !strict { + header.signature = [0; 4]; + } + header + }; + let mut counts: HashMap = HashMap::new(); + for &top in tops { + if top.0.version_id == version_id { + *counts.entry(comparable(top.0)).or_insert(0) += 1; + } + } + let mut latest_count = 0; + for (header, count) in &counts { + if *count < latest_count || (*count == latest_count && latest.0.sorts_before(header)) { + continue; + } + for &top in tops { + if comparable(top.0) == *header { + latest = top; + } + } + latest_count = *count; + } + (latest, latest_count) } pub(crate) fn merge_file_meta_borrowed_streams( mut quorum: usize, mut strict: bool, requested_versions: usize, - enforce_write_quorum: bool, - write_quorum_slack: usize, mut streams: Vec>, ) -> Vec { let to_shallow = |(header, meta): (&FileMetaVersionHeader, &[u8])| FileMetaShallowVersion { @@ -4192,32 +4137,13 @@ pub(crate) fn merge_file_meta_borrowed_streams( } if streams.len() == 1 { - if !enforce_write_quorum { - return streams[0].iter().copied().map(to_shallow).collect(); - } - - let required_quorum = streams[0] - .first() - .map(|&(hdr, meta)| listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack)) - .unwrap_or(quorum); - if streams.len() >= required_quorum { - return streams[0].iter().copied().map(to_shallow).collect(); - } - return Vec::new(); + return streams[0].iter().copied().map(to_shallow).collect(); } if quorum == 1 { strict = true; } - let required_quorum = |(hdr, meta): (&FileMetaVersionHeader, &[u8])| { - if enforce_write_quorum { - listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack) - } else { - quorum - } - }; - let mut tops: Vec<(&FileMetaVersionHeader, &[u8])> = Vec::with_capacity(streams.len()); let mut n_versions = 0; @@ -4249,7 +4175,7 @@ pub(crate) fn merge_file_meta_borrowed_streams( let latest: (&FileMetaVersionHeader, &[u8]); if consistent { latest = tops[0]; - if tops.len() >= required_quorum(latest) { + if tops.len() >= quorum { merged.push(to_shallow(latest)); if !latest.0.free_version() { n_versions += 1; @@ -4258,6 +4184,7 @@ pub(crate) fn merge_file_meta_borrowed_streams( } else { let mut latest_ref = tops[0]; let mut latest_count = 1; + let mut settled_by_version_id = false; for &ver in &tops[1..] { if ver.0 == latest_ref.0 { latest_count += 1; @@ -4281,39 +4208,26 @@ pub(crate) fn merge_file_meta_borrowed_streams( } if ver.0.version_id == latest_ref.0.version_id { - let mut x: HashMap = HashMap::new(); - for &a in &tops { - if a.0.version_id != ver.0.version_id { - continue; - } - let mut hdr = a.0.clone(); - if !strict { - hdr.signature = [0; 4]; - } - *x.entry(hdr).or_insert(0) += 1; - } - latest_count = 0; - for (k, v) in x.iter() { - if *v < latest_count || (*v == latest_count && latest_ref.0.sorts_before(k)) { - continue; - } - for &a in &tops { - let mut hdr = a.0.clone(); - if !strict { - hdr.signature = [0; 4]; - } - if hdr == *k { - latest_ref = a; - } - } - - latest_count = *v; - } + (latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict); + settled_by_version_id = true; break; } } + // Readers return their heads in any order. When every older header + // sharing the newest header's version id (an interrupted null-version + // overwrite) came first, the scan above never compared them, and the + // stream trim below would then drop that older committed generation. + if !settled_by_version_id + && tops.iter().any(|ver| { + ver.0 != latest_ref.0 + && ver.0.version_id == latest_ref.0.version_id + && (strict || !ver.0.matches_not_strict(latest_ref.0)) + }) + { + (latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict); + } latest = latest_ref; - if latest_count >= required_quorum(latest) { + if latest_count >= quorum { if !latest.0.free_version() { n_versions += 1; } @@ -5102,39 +5016,6 @@ mod tests { wr } - fn encode_legacy_v2_object_body(data_blocks: usize, parity_blocks: usize) -> Vec { - let drive_count = data_blocks + parity_blocks; - let payload = LegacyObjectVersionFixture { - version_type: LegacyObjectVersionTypeFixture::Object, - object: Some(LegacyObjectFixture { - version_id: Some(sample_version_id().as_bytes().to_vec()), - data_dir: Some(Uuid::from_u128(42).as_bytes().to_vec()), - erasure_algorithm: "ReedSolomon".to_string(), - erasure_m: data_blocks, - erasure_n: parity_blocks, - erasure_block_size: 1_048_576, - erasure_index: 1, - erasure_dist: (1..=drive_count) - .map(|idx| u8::try_from(idx).expect("test drive index should fit u8")) - .collect(), - bitrot_checksum_algo: "HighwayHash".to_string(), - part_numbers: vec![1], - part_etags: vec!["etag-1".to_string()], - part_sizes: vec![11], - part_actual_sizes: vec![11], - part_indices: vec![Vec::new()], - size: 11, - mod_time: Some(sample_mod_time()), - meta_sys: HashMap::new(), - meta_user: HashMap::from([("content-type".to_string(), "text/plain".to_string())]), - }), - delete_marker: None, - write_version: 3, - }; - - rmp_serde::to_vec_named(&payload).expect("legacy object payload should marshal") - } - #[test] fn version_header_unmarshal_v1_uses_legacy_layout_defaults() { let expected = sample_header(); @@ -5169,80 +5050,6 @@ mod tests { assert_eq!(decoded.ec_m, 0); } - #[test] - fn shallow_version_write_quorum_uses_legacy_object_payload_when_header_lacks_ec() { - let expected = sample_header(); - let encoded = encode_v2_header(&expected); - let mut header = FileMetaVersionHeader::default(); - header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode"); - - let version = FileMetaShallowVersion { - header, - meta: encode_legacy_v2_object_body(7, 1), - }; - - assert_eq!(version.header.write_quorum(5), 5); - assert_eq!(version.write_quorum(5), 7); - } - - #[test] - fn listing_quorum_uses_each_versions_data_floor() { - for (data, parity, offline, fallback, expected) in - [(8, 8, 0, 9, 9), (8, 8, 8, 8, 8), (12, 4, 8, 8, 12), (12, 4, 0, 9, 12)] - { - let version = FileMetaShallowVersion { - header: FileMetaVersionHeader { - version_type: VersionType::Object, - ec_m: data, - ec_n: parity, - ..Default::default() - }, - // Modern headers must not decode the payload to decide quorum. - meta: vec![0xff], - }; - assert_eq!(version.listing_quorum(fallback, offline), expected); - } - let marker = FileMetaShallowVersion { - header: FileMetaVersionHeader { - version_type: VersionType::Delete, - ..Default::default() - }, - meta: Vec::new(), - }; - assert_eq!(marker.listing_quorum(9, 0), 9, "healthy marker needs majority metadata"); - assert_eq!(marker.listing_quorum(8, 8), 8, "degraded marker retains metadata floor"); - } - - #[test] - fn listing_quorum_preserves_legacy_and_zero_parity_data_floor() { - for (data, parity, fallback, expected) in [(12, 4, 8, 12), (8, 8, 8, 8), (4, 0, 2, 4)] { - let version = FileMetaShallowVersion { - header: FileMetaVersionHeader { - version_type: VersionType::Object, - ..Default::default() - }, - meta: encode_legacy_v2_object_body(data, parity), - }; - assert_eq!(version.listing_quorum(fallback, 8), expected); - } - } - - #[test] - fn shallow_version_write_quorum_uses_zero_parity_object_payload() { - let expected = sample_header(); - let encoded = encode_v2_header(&expected); - let mut header = FileMetaVersionHeader::default(); - header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode"); - - let version = FileMetaShallowVersion { - header, - meta: encode_legacy_v2_object_body(4, 0), - }; - - assert_eq!(version.header.write_quorum(2), 2); - assert_eq!(version.write_quorum(2), 4); - } - #[test] fn version_header_unmarshal_v3_round_trips_current_layout() { let expected = sample_header(); diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index 0205e20c0..bf70fad5f 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -51,12 +51,6 @@ pub struct MetadataResolutionParams { pub requested_versions: usize, pub bucket: String, pub strict: bool, - /// Number of set drives that were unreachable when the listing snapshot was - /// taken. Write-quorum enforcement relaxes each version's required quorum by - /// this amount, never below the version's data quorum or `obj_quorum`. A - /// version legally committed at write quorum can have metadata on drives no - /// reader could consult, and must not be dropped for it. - pub write_quorum_slack: usize, pub candidates: Vec>, } @@ -276,88 +270,6 @@ impl MetaCacheEntry { FileMeta::decode_file_info_versions_from_buf(&self.metadata, bucket, self.name.as_str(), false) } - fn borrow_metadata(&self) -> Result> { - if self.is_dir() { - return Err(Error::FileNotFound); - } - match &self.cached { - Some(metadata) => Ok(metadata.borrow_cached()), - None => FileMeta::decode_borrowed(&self.metadata), - } - } - - /// Compare a replica group after validating this history once. - pub fn headers_agree<'a>(&'a self, replicas: impl IntoIterator, strict: bool) -> bool { - if self.is_dir() { - return replicas.into_iter().all(|other| self.name == other.name && other.is_dir()); - } - let Ok(metadata) = self.borrow_metadata() else { - return false; - }; - replicas.into_iter().all(|other| { - if self.name != other.name || other.is_dir() { - return false; - } - if std::ptr::eq(self, other) { - return true; - } - if let Some(other) = &other.cached { - crate::filemeta::reconcile_headers( - other.versions.iter().map(|version| &version.header), - metadata.versions.as_slice().iter().map(|(header, _, _)| header), - true, - strict, - ) - .is_some() - } else { - FileMeta::decode_borrowed_with_hint(&other.metadata, Some(&metadata)).is_ok_and(|candidate| { - crate::filemeta::reconcile_headers( - candidate.versions.as_slice().iter().map(|(header, _, _)| header), - metadata.versions.as_slice().iter().map(|(header, _, _)| header), - true, - strict, - ) - .is_some() - }) - } - }) - } - - fn cached_latest_listing_quorum(&self, params: &MetadataResolutionParams) -> Option { - if params.requested_versions != 1 { - return None; - } - self.cached - .as_ref()? - .versions - .first() - .filter(|version| !version.header.free_version()) - .map(|version| version.listing_quorum(params.obj_quorum, params.write_quorum_slack)) - } - - /// Whether the inspected versions need more independent replica votes. - pub fn needs_quorum_supplement(&self, replicas: usize, params: &MetadataResolutionParams) -> Result { - if let Some(required) = self.cached_latest_listing_quorum(params) { - return Ok(replicas < required); - } - let metadata = self.borrow_metadata()?; - let mut selected = 0; - for (header, _, meta) in metadata.versions.as_slice() { - let required = - crate::filemeta::listing_quorum_from_header_and_meta(header, meta, params.obj_quorum, params.write_quorum_slack); - if replicas < required && (params.requested_versions == 0 || selected < params.requested_versions) { - return Ok(true); - } - if !header.free_version() { - selected += 1; - } - if params.requested_versions > 0 && selected == params.requested_versions { - break; - } - } - Ok(false) - } - pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option, bool) { if other.is_none() { return (None, false); @@ -489,125 +401,6 @@ impl MetaCacheEntry { } } -/// Outcome of resolving one history observed identically by independent readers. -pub enum MetaCacheEntryResolution { - Resolved(MetaCacheEntry), - NeedsSupplement(MetaCacheEntry), - Rejected, -} - -/// Apply per-version listing quorum without materializing unchanged histories. -#[inline] -pub fn resolve_agreed_entry( - mut entry: MetaCacheEntry, - replicas: usize, - params: &MetadataResolutionParams, - enforce_write_quorum: bool, -) -> MetaCacheEntryResolution { - use MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved}; - if !enforce_write_quorum { - return Resolved(entry); - } - if entry.is_dir() { - return if replicas >= params.dir_quorum { - Resolved(entry) - } else { - Rejected - }; - } - if replicas < params.obj_quorum { - return Rejected; - } - if let Some(required) = entry.cached_latest_listing_quorum(params) { - return if replicas >= required { - Resolved(entry) - } else { - NeedsSupplement(entry) - }; - } - let metadata = match entry.borrow_metadata() { - Ok(metadata) => metadata, - Err(_) if params.obj_quorum <= 1 => return Resolved(entry), - Err(_) => return Rejected, - }; - let versions = metadata.versions.as_slice(); - let mut selected = Vec::new(); - let mut selected_objects = 0; - let mut needs_supplement = false; - for (index, (header, _, meta)) in versions.iter().enumerate() { - let required = - crate::filemeta::listing_quorum_from_header_and_meta(header, meta, params.obj_quorum, params.write_quorum_slack); - if replicas < required { - needs_supplement |= params.requested_versions == 0 || selected_objects < params.requested_versions; - continue; - } - // The common latest-user-version path keeps the original buffer and cache. - if index == 0 && params.requested_versions == 1 && !header.free_version() { - return Resolved(entry); - } - selected.push(index); - if !header.free_version() { - selected_objects += 1; - } - if params.requested_versions > 0 && selected_objects == params.requested_versions { - break; - } - } - if needs_supplement { - return NeedsSupplement(entry); - } - if selected.is_empty() { - return Rejected; - } - if selected.len() == versions.len() { - return Resolved(entry); - } - if entry.cached.is_some() { - drop(metadata); - let Some(cached) = entry.cached.as_mut() else { - return Rejected; - }; - let mut selected = selected.into_iter().peekable(); - let mut index = 0; - cached.versions.retain(|_| { - let keep = selected.peek() == Some(&index); - if keep { - selected.next(); - } - index += 1; - keep - }); - let Ok(metadata) = cached.marshal_msg() else { - return Rejected; - }; - entry.metadata = metadata; - } else { - let mut filtered = FileMeta { - meta_ver: metadata.meta_ver, - ..Default::default() - }; - if !metadata.inline_buf.is_empty() { - filtered.data.update(metadata.inline_buf); - } - filtered.versions = selected - .into_iter() - .map(|index| { - let (header, _, meta) = &versions[index]; - FileMetaShallowVersion { - header: header.clone(), - meta: meta.to_vec(), - } - }) - .collect(); - let Ok(serialized) = filtered.marshal_msg() else { - return Rejected; - }; - entry.metadata = serialized; - entry.cached = Some(filtered); - } - Resolved(entry) -} - #[derive(Debug, Default)] pub struct MetaCacheEntries(pub Vec>); @@ -618,23 +411,7 @@ impl MetaCacheEntries { } pub fn resolve(&self, params: MetadataResolutionParams) -> Option { - Self::resolve_inner(&self.0, params, false) - } - - pub fn resolve_with_write_quorum(&self, params: MetadataResolutionParams) -> Option { - Self::resolve_inner(&self.0, params, true) - } - - /// Resolve independently observed replicas without copying their metadata. - /// - /// Each occupied slot represents one distinct reader. Repeated references - /// are valid only when the caller already established agreement among those - /// readers; pointer equality alone is not evidence of extra replicas. - pub fn resolve_borrowed_with_write_quorum( - entries: &[Option<&MetaCacheEntry>], - params: MetadataResolutionParams, - ) -> Option { - Self::resolve_inner(entries, params, true) + Self::resolve_inner(&self.0, params) } /// Resolve the cross-disk UNION of every version present on ANY disk slot. @@ -659,7 +436,6 @@ impl MetaCacheEntries { requested_versions: 0, bucket: bucket.to_string(), strict: false, - write_quorum_slack: 0, candidates: Vec::new(), }) } @@ -843,11 +619,7 @@ impl MetaCacheEntries { discovery } - fn resolve_inner>( - entries: &[Option], - mut params: MetadataResolutionParams, - enforce_write_quorum: bool, - ) -> Option { + fn resolve_inner(entries: &[Option], mut params: MetadataResolutionParams) -> Option { if entries.is_empty() { debug!( bucket = %params.bucket, @@ -868,7 +640,7 @@ impl MetaCacheEntries { let mut objs_agree = 0; let mut objs_valid = 0; - for (entry_idx, entry) in entries.iter().flatten().map(std::borrow::Borrow::borrow).enumerate() { + for (entry_idx, entry) in entries.iter().flatten().enumerate() { debug!(entry = %entry.name, "metacache resolve examining candidate entry"); if entry.name.is_empty() { continue; @@ -971,56 +743,29 @@ impl MetaCacheEntries { return directory.cloned(); } - if objs_agree == objs_valid - && (!enforce_write_quorum - || (params.requested_versions == 1 - && selected_borrowed - .versions - .as_slice() - .first() - .is_some_and(|(header, _, _)| !header.free_version()))) - { - let required_quorum = if enforce_write_quorum { - selected_borrowed - .versions - .as_slice() - .first() - .map(|(hdr, _, meta)| { - crate::filemeta::listing_quorum_from_header_and_meta( - hdr, - meta, - params.obj_quorum, - params.write_quorum_slack, - ) - }) - .unwrap_or(params.obj_quorum) - } else { - params.obj_quorum - }; - - if objs_agree >= required_quorum { - debug!( - selected = %selected_ref.name, - objs_agree, - objs_valid, - "metacache resolve reused selected candidate because all valid object entries agreed" - ); - let selected_fm = selected_ref - .cached - .clone() - .unwrap_or_else(|| selected_borrowed.into_file_meta()); - return Some(MetaCacheEntry { - name: selected_ref.name.clone(), - metadata: selected_ref.metadata.clone(), - cached: Some(selected_fm), - reusable: selected_ref.reusable, - }); - } + // Every valid copy agrees and `objs_valid` already reached the quorum. + if objs_agree == objs_valid { + debug!( + selected = %selected_ref.name, + objs_agree, + objs_valid, + "metacache resolve reused selected candidate because all valid object entries agreed" + ); + let selected_fm = selected_ref + .cached + .clone() + .unwrap_or_else(|| selected_borrowed.into_file_meta()); + return Some(MetaCacheEntry { + name: selected_ref.name.clone(), + metadata: selected_ref.metadata.clone(), + cached: Some(selected_fm), + reusable: selected_ref.reusable, + }); } let selected_meta_ver = selected_borrowed.meta_ver; let mut streams: Vec> = Vec::with_capacity(objs_valid); - for (entry_idx, entry) in entries.iter().flatten().map(std::borrow::Borrow::borrow).enumerate() { + for (entry_idx, entry) in entries.iter().flatten().enumerate() { if entry.name.is_empty() || entry.is_dir() || invalid_entries.contains(&entry_idx) { continue; } @@ -1045,19 +790,13 @@ impl MetaCacheEntries { } debug_assert_eq!(streams.len(), objs_valid); - let had_user_versions = enforce_write_quorum - && streams - .iter() - .any(|stream| stream.iter().any(|(header, _)| !header.free_version())); let versions = crate::filemeta::merge_file_meta_borrowed_streams( params.obj_quorum, params.strict, params.requested_versions, - enforce_write_quorum, - params.write_quorum_slack, streams, ); - if versions.is_empty() || (had_user_versions && versions.iter().all(|version| version.header.free_version())) { + if versions.is_empty() { debug!( selected = %selected_ref.name, requested_versions = params.requested_versions, @@ -2701,17 +2440,6 @@ mod tests { } } - fn metacache_entry_without_header_ec(mut entry: MetaCacheEntry) -> MetaCacheEntry { - let mut cached = entry.cached.take().expect("test entry should have cached metadata"); - for version in cached.versions.iter_mut() { - version.header.ec_m = 0; - version.header.ec_n = 0; - } - entry.metadata = cached.marshal_msg().expect("test file metadata should marshal"); - entry.cached = Some(cached); - entry - } - fn metacache_dir_entry(name: &str) -> MetaCacheEntry { MetaCacheEntry { name: name.to_string(), @@ -3213,410 +2941,74 @@ mod tests { } #[test] - fn resolve_rejects_low_parity_partial_latest_below_required_object_quorum() { + fn resolve_interrupted_null_overwrite_is_independent_of_reader_order() { let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 7, 1); - let new_entry = metacache_entry_with_erasure(new_mod_time, "new-etag", 7, 1); - - let resolved = MetaCacheEntries(vec![ - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry), - Some(old_entry.clone()), - Some(old_entry), - ]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 5, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }); - - assert!(resolved.is_none()); - } - - #[test] - fn resolve_rejects_zero_parity_partial_latest_below_required_object_quorum() { - let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let new_entry = metacache_entry_with_erasure(new_mod_time, "new-etag", 7, 0); - - let resolved = MetaCacheEntries(vec![ - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry), - ]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 5, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }); - - assert!(resolved.is_none()); - } - - #[test] - fn resolve_accepts_low_parity_latest_at_required_object_quorum() { - let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let new_entry = metacache_entry_with_erasure(new_mod_time, "new-etag", 7, 1); - - let resolved = MetaCacheEntries(vec![ - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry.clone()), - Some(new_entry), - ]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 5, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }) - .expect("latest metadata should resolve after satisfying its own write quorum"); - - let info = resolved - .to_fileinfo("bucket") - .expect("resolved committed metadata should decode as file info"); - assert_eq!(info.mod_time, Some(new_mod_time)); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new-etag")); - } - - #[test] - fn resolve_skips_low_parity_partial_latest_and_returns_committed_previous_version() { - let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 4, 4); - let new_and_old_entry = - metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)]); - - let resolved = MetaCacheEntries(vec![ - Some(new_and_old_entry.clone()), - Some(new_and_old_entry.clone()), - Some(new_and_old_entry.clone()), - Some(new_and_old_entry.clone()), - Some(new_and_old_entry), - Some(old_entry.clone()), - Some(old_entry), - ]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 5, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }) - .expect("previous committed metadata should resolve after rejecting partial latest"); - - let info = resolved - .to_fileinfo("bucket") - .expect("resolved committed metadata should decode as file info"); - assert_eq!(info.mod_time, Some(old_mod_time)); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag")); - } - - #[test] - fn resolve_skips_legacy_header_low_parity_partial_latest_using_payload_quorum() { - let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 4, 4); - let new_and_old_entry = metacache_entry_without_header_ec(metacache_entry_with_erasure_versions(&[ - (old_mod_time, "old-etag", 4, 4), - (new_mod_time, "new-etag", 7, 1), - ])); - - let resolved = MetaCacheEntries(vec![ - Some(new_and_old_entry.clone()), - Some(new_and_old_entry.clone()), - Some(new_and_old_entry.clone()), - Some(new_and_old_entry.clone()), - Some(new_and_old_entry), - Some(old_entry.clone()), - Some(old_entry), - ]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 5, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }) - .expect("previous committed metadata should resolve after rejecting legacy-header partial latest"); - - let info = resolved - .to_fileinfo("bucket") - .expect("resolved committed metadata should decode as file info"); - assert_eq!(info.mod_time, Some(old_mod_time)); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag")); - } - - #[test] - fn borrowed_resolution_counts_reader_slots_and_preserves_owned_semantics() { - let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time"); - for cached in [false, true] { - for legacy in [false, true] { - let mut entry = metacache_entry_with_erasure(time, "etag", 8, 8); - if legacy { - entry = metacache_entry_without_header_ec(entry); - } - if !cached { - entry.cached = None; - } - for replicas in [0, 1, 7, 8, 9] { - let mut slots = vec![Some(&entry); replicas]; - slots.extend([None, None]); - let params = MetadataResolutionParams { - dir_quorum: 8, - obj_quorum: 8, - requested_versions: 1, - write_quorum_slack: 8, - bucket: "bucket".to_owned(), - ..Default::default() - }; - let borrowed = MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()); - let owned = MetaCacheEntries(slots.iter().map(|slot| slot.map(Clone::clone)).collect()) - .resolve_with_write_quorum(params); - assert_eq!(borrowed.is_some(), replicas >= 8, "replicas={replicas}, cached={cached}, legacy={legacy}"); - assert_eq!( - borrowed.map(|entry| (entry.name, entry.metadata)), - owned.map(|entry| (entry.name, entry.metadata)) - ); - } - } - } - } - - #[test] - fn borrowed_header_agreement_matches_existing_strict_and_relaxed_contracts() { - let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time"); - let current = metacache_entry_with_erasure(time, "current", 8, 8); - let older = metacache_entry_with_erasure(time - time::Duration::seconds(1), "older", 8, 8); - let legacy = metacache_entry_without_header_ec(current.clone()); - let mut changed_signature = current.clone(); - changed_signature.cached.as_mut().expect("cached fixture").versions[0] - .header - .signature[0] ^= 1; - changed_signature.metadata = changed_signature - .cached - .as_ref() - .expect("cached fixture") - .marshal_msg() - .expect("changed header"); - let corrupt = MetaCacheEntry { - name: "object".to_owned(), - metadata: vec![0xff], - ..Default::default() - }; - let directory = metacache_dir_entry("object"); - let cases = [current, older, legacy, changed_signature, corrupt, directory]; - for strict in [false, true] { - for cached in [false, true] { - for left in &cases { - for right in &cases { - let mut left = left.clone(); - let mut right = right.clone(); - if !cached { - left.cached = None; - right.cached = None; - } - assert_eq!( - left.headers_agree([&right], strict), - left.matches(Some(&right), strict).1, - "strict={strict}, cached={cached}" - ); - } - } - } - } - } - - #[test] - fn header_groups_keep_the_original_geometry_baseline_across_legacy_wildcards() { - let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time"); - let original = metacache_entry_with_erasure(time, "etag", 8, 8); - let legacy = metacache_entry_without_header_ec(original.clone()); - let mut conflict = original.clone(); - let metadata = conflict.cached.as_mut().expect("fixture cache"); - metadata.versions[0].header.ec_m = 12; - metadata.versions[0].header.ec_n = 4; - conflict.metadata = metadata.marshal_msg().expect("conflicting geometry header"); - for strict in [false, true] { - for cache_mask in 0..8 { - let mut replicas = [original.clone(), legacy.clone(), conflict.clone()]; - for (index, entry) in replicas.iter_mut().enumerate() { - if cache_mask & (1 << index) == 0 { - entry.cached = None; - } - } - for base in &replicas { - for order in [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] { - let group = order.map(|index| &replicas[index]); - let expected = group.iter().all(|candidate| base.matches(Some(candidate), strict).1); - assert_eq!( - base.headers_agree(group, strict), - expected, - "strict={strict}, mask={cache_mask}, order={order:?}" - ); - } - } - } - } - } - - #[test] - fn agreed_resolution_preserves_buffers_without_materializing_unchanged_metadata() { - let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time"); - for cached in [false, true] { - let mut entry = metacache_entry_with_erasure(time, "etag", 8, 8); - if !cached { - entry.cached = None; - } - let metadata = entry.metadata.as_ptr(); - let cached_version = entry.cached.as_ref().map(|meta| meta.versions[0].meta.as_ptr()); - let MetaCacheEntryResolution::Resolved(entry) = resolve_agreed_entry( - entry, - 8, - &MetadataResolutionParams { - dir_quorum: 8, - obj_quorum: 8, + let old_entry = metacache_entry_with_mod_time(old_mod_time, "old-etag"); + let new_entry = metacache_entry_with_mod_time(new_mod_time, "new-etag"); + // Every placement of the new generation across four readers. Both + // generations share the null version id, so the most common header + // wins and a tie keeps the newer one. + for placement in 0_u32..16 { + let new_copies = placement.count_ones(); + let entries = (0..4) + .map(|slot| { + Some(if placement & (1 << slot) != 0 { + new_entry.clone() + } else { + old_entry.clone() + }) + }) + .collect(); + let resolved = MetaCacheEntries(entries) + .resolve(MetadataResolutionParams { + dir_quorum: 2, + obj_quorum: 2, requested_versions: 1, - write_quorum_slack: 8, + bucket: "bucket".to_string(), ..Default::default() - }, - true, - ) else { - panic!("eight independent EC8 readers must resolve"); - }; - assert_eq!(entry.metadata.as_ptr(), metadata, "unchanged serialized metadata must move"); + }) + .unwrap_or_else(|| panic!("placement {placement:04b}: one generation always reaches quorum")); + let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode"); + let expected = if new_copies >= 2 { "new-etag" } else { "old-etag" }; + assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected), "placement {placement:04b}"); + } + } + + #[test] + fn resolve_merge_keeps_the_newest_version_at_listing_quorum() { + let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); + // The merge counts metadata copies only; the EC geometry of a version + // does not raise the quorum a listing needs (MinIO mergeXLV2Versions). + let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2); + let new_and_old_entry = + metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]); + for (new_copies, expected) in [(2, (new_mod_time, "new-etag")), (1, (old_mod_time, "old-etag"))] { + let mut entries = vec![Some(new_and_old_entry.clone()); new_copies]; + entries.extend(vec![Some(old_entry.clone()); 3 - new_copies]); + let resolved = MetaCacheEntries(entries) + .resolve(MetadataResolutionParams { + dir_quorum: 2, + obj_quorum: 2, + requested_versions: 1, + bucket: "bucket".to_string(), + strict: true, + ..Default::default() + }) + .expect("a version at listing quorum must resolve"); + let info = resolved + .to_fileinfo("bucket") + .expect("resolved metadata should decode as file info"); + assert_eq!(info.mod_time, Some(expected.0), "new copies: {new_copies}"); assert_eq!( - entry.cached.as_ref().map(|meta| meta.versions[0].meta.as_ptr()), - cached_version, - "cold metadata stays borrowed and existing decoded buffers move" + info.metadata.get("etag").map(String::as_str), + Some(expected.1), + "new copies: {new_copies}" ); } } - #[test] - fn listing_quorum_never_relaxes_a_versions_read_quorum() { - let time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("fixture time"); - for (data, parity, copies, expected) in [(8, 8, 8, true), (12, 4, 8, false), (12, 4, 11, false), (12, 4, 12, true)] { - let entry = metacache_entry_with_erasure(time, "etag", data, parity); - let entries = MetaCacheEntries(vec![Some(entry); copies]); - let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 8, - dir_quorum: 8, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - write_quorum_slack: 16 - copies, - ..Default::default() - }); - assert_eq!(resolved.is_some(), expected, "layout {data}+{parity}, {copies} copies"); - } - } - - #[test] - fn resolve_with_write_quorum_relaxes_requirement_by_unreachable_drives() { - let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - // EC 2+2 version, write quorum 3 of 4: committed with one holder drive - // now offline, so only 2 of the 3 reachable drives carry the metadata. - let entry = metacache_entry_with_erasure(mod_time, "etag", 2, 2); - let entries = || MetaCacheEntries(vec![Some(entry.clone()), Some(entry.clone()), None]); - - let resolved = entries() - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 2, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - write_quorum_slack: 1, - ..Default::default() - }) - .expect("committed version should resolve when the missing copy sits on an unreachable drive"); - let info = resolved - .to_fileinfo("bucket") - .expect("resolved committed metadata should decode as file info"); - assert_eq!(info.mod_time, Some(mod_time)); - - // Without the slack the same sample is rejected outright. - let rejected = entries().resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 3, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }); - assert!(rejected.is_none()); - } - - #[test] - fn resolve_with_write_quorum_slack_accepts_committed_latest_during_merge() { - let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - // The newer EC 2+2 version committed at write quorum 3 of 4; one holder - // drive is offline, and a third reachable drive still carries only the - // previous version. The candidates disagree, so the merge path (not the - // all-agree branch) must honor the slack. - let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2); - let new_and_old_entry = - metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]); - - let resolved = MetaCacheEntries(vec![Some(new_and_old_entry.clone()), Some(new_and_old_entry), Some(old_entry)]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 2, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - write_quorum_slack: 1, - ..Default::default() - }) - .expect("committed latest should survive the merge when its missing copy is on an unreachable drive"); - let info = resolved - .to_fileinfo("bucket") - .expect("resolved committed metadata should decode as file info"); - assert_eq!(info.mod_time, Some(new_mod_time)); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new-etag")); - } - - #[test] - fn resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge() { - let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); - let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp"); - // The newer EC 2+2 version (write quorum 3) reached only ONE drive: even - // with one drive unreachable (slack 1) it cannot prove the relaxed - // quorum of 2, so the committed previous version must win the merge. - let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 2, 2); - let new_and_old_entry = - metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 2, 2), (new_mod_time, "new-etag", 2, 2)]); - - let resolved = MetaCacheEntries(vec![Some(new_and_old_entry), Some(old_entry.clone()), Some(old_entry)]) - .resolve_with_write_quorum(MetadataResolutionParams { - obj_quorum: 2, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - write_quorum_slack: 1, - ..Default::default() - }) - .expect("committed previous version should resolve after rejecting the partial latest"); - let info = resolved - .to_fileinfo("bucket") - .expect("resolved committed metadata should decode as file info"); - assert_eq!(info.mod_time, Some(old_mod_time)); - assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag")); - } - #[test] fn resolve_rejects_partial_directory_below_dir_quorum() { let partial_dir = metacache_dir_entry("prefix/"); @@ -3638,46 +3030,36 @@ mod tests { let mut object = metacache_entry_with_erasure(modified, "marker-etag", 2, 2); object.name = "folder/".to_string(); let directory = metacache_dir_entry("folder/"); - for enforce_write_quorum in [false, true] { - for (object_copies, directory_copies) in [(3, 1), (3, 3), (2, 3), (0, 3), (2, 2)] { - for reverse in [false, true] { - let mut entries = vec![Some(object.clone()); object_copies]; - entries.extend(vec![Some(directory.clone()); directory_copies]); - if reverse { - entries.reverse(); - } - let entries = MetaCacheEntries(entries); - let params = MetadataResolutionParams { - dir_quorum: 3, - obj_quorum: 3, - requested_versions: 1, - bucket: "bucket".to_string(), - strict: true, - ..Default::default() - }; - let resolved = if enforce_write_quorum { - entries.resolve_with_write_quorum(params) - } else { - entries.resolve(params) - }; - let context = format!( - "objects={object_copies}, directories={directory_copies}, reverse={reverse}, write={enforce_write_quorum}" + for (object_copies, directory_copies) in [(3, 1), (3, 3), (2, 3), (0, 3), (2, 2)] { + for reverse in [false, true] { + let mut entries = vec![Some(object.clone()); object_copies]; + entries.extend(vec![Some(directory.clone()); directory_copies]); + if reverse { + entries.reverse(); + } + let resolved = MetaCacheEntries(entries).resolve(MetadataResolutionParams { + dir_quorum: 3, + obj_quorum: 3, + requested_versions: 1, + bucket: "bucket".to_string(), + strict: true, + ..Default::default() + }); + let context = format!("objects={object_copies}, directories={directory_copies}, reverse={reverse}"); + if object_copies >= 3 { + let resolved = resolved.expect("metadata at quorum must shadow a same-named prefix"); + assert!(resolved.is_object(), "{context}"); + assert_eq!( + resolved + .to_fileinfo("bucket") + .expect("marker metadata should decode") + .mod_time, + Some(modified) ); - if object_copies >= 3 { - let resolved = resolved.expect("metadata at quorum must shadow a same-named prefix"); - assert!(resolved.is_object(), "{context}"); - assert_eq!( - resolved - .to_fileinfo("bucket") - .expect("marker metadata should decode") - .mod_time, - Some(modified) - ); - } else if directory_copies >= 3 { - assert!(resolved.expect("prefix at quorum must remain a fallback").is_dir(), "{context}"); - } else { - assert!(resolved.is_none(), "{context}"); - } + } else if directory_copies >= 3 { + assert!(resolved.expect("prefix at quorum must remain a fallback").is_dir(), "{context}"); + } else { + assert!(resolved.is_none(), "{context}"); } } } @@ -4491,13 +3873,12 @@ mod tests { })); } let resolved_ec = MetaCacheEntries(distinct_ec_entries) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 2, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve distinct ec"); @@ -4545,7 +3926,6 @@ mod tests { requested_versions: 0, bucket: "bucket".to_string(), strict: false, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve keep_first"); @@ -4572,7 +3952,6 @@ mod tests { requested_versions: 0, bucket: "bucket".to_string(), strict: false, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve prefer_second"); @@ -4683,24 +4062,22 @@ mod tests { })); let res_uncached = MetaCacheEntries(uncached_disagree) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 3, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve uncached disagree"); let res_cached = MetaCacheEntries(cached_disagree) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 3, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve cached disagree"); @@ -4729,13 +4106,12 @@ mod tests { reusable: false, })); let res_agree_corrupt = MetaCacheEntries(agree_with_corrupt) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 3, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve agree with corrupt"); @@ -4762,13 +4138,12 @@ mod tests { })); } let res_supplement = MetaCacheEntries(supplement_entries) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 3, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve supplement"); @@ -4803,13 +4178,12 @@ mod tests { })); } let res_outlier_last = MetaCacheEntries(outlier_last) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 3, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve outlier_last"); @@ -4872,13 +4246,12 @@ mod tests { reusable: false, })); let res_mixed_empty = MetaCacheEntries(mixed_empty_entries) - .resolve_with_write_quorum(MetadataResolutionParams { + .resolve(MetadataResolutionParams { dir_quorum: 2, obj_quorum: 4, requested_versions: 1, bucket: "bucket".to_string(), strict: true, - write_quorum_slack: 0, candidates: Vec::new(), }) .expect("resolve mixed empty"); diff --git a/docs/testing/ecstore-core-invariants.md b/docs/testing/ecstore-core-invariants.md index d679d2fa5..bb86ab1e4 100644 --- a/docs/testing/ecstore-core-invariants.md +++ b/docs/testing/ecstore-core-invariants.md @@ -44,7 +44,7 @@ shown. The manifest contains their complete module paths. | MPU delete quorum | `abort_enforces_delete_write_quorum_boundary` | Abort requires the existing delete write quorum. | | MPU physical publication | `complete_multipart_timeout_keeps_namespace_owner_until_physical_publication` | Timeout cannot retire an active physical publication. | | Restore failures | `multipart_restore_aborts_every_post_create_failure` | Every injected post-create failure aborts the remote upload. | -| Partial LIST metadata | `rustfs-filemeta`: `resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge` | A partial newer version remains hidden. | +| Partial LIST metadata | `rustfs-filemeta`: `resolve_merge_keeps_the_newest_version_at_listing_quorum` | A newer version below the listing quorum remains hidden behind the committed version. | | Metadata observations | `metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction` and its companion tests | Pending, absent, corrupt, offline and successful disk slots retain the existing quorum decisions across all four-slot arrival orders. | | Metadata cache bodies | `metadata_cache_real_get_and_range_preserve_body_across_read_modes` | Real cache misses and hits return identical full-body and non-zero Range bytes for verified inline/non-inline objects, with early-stop enabled/disabled and one offline disk. Hits share a complete four-slot snapshot containing all available disks and issue no new metadata reads. | | Late metadata identity | `late_metadata_refresh_rejects_changed_generation_and_mapped_shards` | Metadata from a real four-disk PUT still reaches selection quorum, but changed generation or incorrect per-slot shard mapping fails the disk-backed late-refresh boundary with an object-scoped read-quorum error. | @@ -100,10 +100,13 @@ Removing only the two old #7049 limit guards still passes because the recursive child's stop result provides another protection. Removing those guards and ignoring that result in the same stack-draining loop makes raw limit 1 return `a/first` and `a0` instead of only `a/first`. This tests the historical invariant, -not a literal revert of #7049. Setting the current offline write-quorum -allowance to zero makes the #7010 corpus return `InsufficientReadQuorum` where +not a literal revert of #7049. At that main, setting the offline write-quorum +allowance to zero made the #7010 corpus return `InsufficientReadQuorum` where the independent oracle requires visible objects. Restoring both production -files passes both tests. These local controls do not establish a concurrent +files passes both tests. The listing now takes its quorum as `ceil(asked/2)` +of the online drives it asked (MinIO's rule), so the #7010 boundary needs two +copies among three online drives; a write-quorum requirement of three would +hide those objects again. These local controls do not establish a concurrent cross-page snapshot or process-crash safety. ## Write owner and completion routes