fix(ecstore): require commit quorum for latest metadata (#4117)

This commit is contained in:
GatewayJ
2026-07-02 00:41:11 +08:00
committed by GitHub
parent 572a001f93
commit 7728c9f203
7 changed files with 2558 additions and 128 deletions
@@ -19,7 +19,7 @@ use futures::future::join_all;
use metrics::counter;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
use std::{
collections::VecDeque,
collections::{HashSet, VecDeque},
future::Future,
pin::Pin,
sync::{Arc, OnceLock},
@@ -41,6 +41,33 @@ pub type PartialFn =
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
#[derive(Clone, Default)]
pub(crate) struct FallbackClaimTracker {
claimed: Arc<TokioMutex<HashSet<String>>>,
}
impl FallbackClaimTracker {
pub(crate) async fn claim_disk(&self, disk: &DiskStore) {
self.claimed.lock().await.insert(disk.endpoint().to_string());
}
pub(crate) async fn claimed_keys(&self) -> HashSet<String> {
self.claimed.lock().await.clone()
}
#[cfg(test)]
pub(crate) async fn claim_test_fallback(&self) {
let mut claimed = self.claimed.lock().await;
let key = format!("test-fallback-{}", claimed.len());
claimed.insert(key);
}
#[cfg(test)]
pub(crate) async fn contains_key(&self, key: &str) -> bool {
self.claimed.lock().await.contains(key)
}
}
#[derive(Debug)]
enum PeekOutcome {
Ready(Option<MetaCacheEntry>),
@@ -129,9 +156,28 @@ impl Clone for ListPathRawOptions {
}
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
list_path_raw_inner(rx, opts, None).await
}
pub(crate) async fn list_path_raw_with_claim_tracker(
rx: CancellationToken,
opts: ListPathRawOptions,
claim_tracker: FallbackClaimTracker,
) -> disk::error::Result<()> {
list_path_raw_inner(rx, opts, Some(claim_tracker)).await
}
async fn list_path_raw_inner(
rx: CancellationToken,
opts: ListPathRawOptions,
fallback_claim_tracker: Option<FallbackClaimTracker>,
) -> disk::error::Result<()> {
if opts.disks.is_empty() {
return Err(DiskError::ErasureReadQuorum);
}
if opts.min_disks > opts.disks.len() {
return Err(DiskError::ErasureReadQuorum);
}
let log_bucket = opts.bucket.clone();
let log_path = opts.path.clone();
@@ -151,6 +197,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
for (disk_idx, disk) in opts.disks.iter().enumerate() {
let opdisk = disk.clone();
let opts_clone = opts.clone();
let fallback_claim_tracker = fallback_claim_tracker.clone();
let fds_clone = fds.clone();
#[cfg(test)]
let test_fallbacks_clone = test_fallbacks.clone();
@@ -276,6 +323,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
while need_fallback {
#[cfg(test)]
if let Some(behavior) = take_fallback_candidate(&test_fallbacks_clone).await {
if let Some(claim_tracker) = fallback_claim_tracker.as_ref() {
claim_tracker.claim_test_fallback().await;
}
match behavior {
TestReaderBehavior::Eof => {
need_fallback = false;
@@ -341,6 +391,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
record_producer_error(&producer_errs_clone, disk_idx, &err);
return Err(err);
};
if let Some(claim_tracker) = fallback_claim_tracker.as_ref() {
claim_tracker.claim_disk(&disk).await;
}
let fallback_walk_started = std::time::Instant::now();
match disk
@@ -797,6 +850,22 @@ mod tests {
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[tokio::test]
async fn list_path_raw_rejects_impossible_min_disks() {
let err = list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None],
min_disks: 3,
..Default::default()
},
)
.await
.expect_err("impossible listing quorum should fail before producing partial results");
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[test]
fn missing_path_error_classification_excludes_actionable_failures() {
assert!(is_missing_path_error(&DiskError::FileNotFound));
@@ -889,6 +958,27 @@ mod tests {
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &[2]);
}
#[tokio::test]
async fn list_path_raw_records_claimed_fallback_candidates() {
let claim_tracker = FallbackClaimTracker::default();
list_path_raw_with_claim_tracker(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::PrimaryErrorThenFallback(DiskError::DiskNotFound)],
test_fallback_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
..Default::default()
},
claim_tracker.clone(),
)
.await
.expect("fallback producer should restore the single logical reader");
assert!(claim_tracker.contains_key("test-fallback-0").await);
}
#[tokio::test]
async fn list_path_raw_returns_timeout_when_reader_stalls_before_completion() {
let err = list_path_raw(
+340 -34
View File
@@ -13,6 +13,14 @@
// limitations under the License.
use super::*;
use rustfs_utils::http;
#[derive(Clone, Copy)]
struct FileInfoIdentityGroup {
hash: [u8; 32],
count: usize,
mod_time: Option<OffsetDateTime>,
}
impl SetDisks {
pub(super) fn all_not_found_metadata(errs: &[Option<DiskError>]) -> bool {
@@ -338,6 +346,90 @@ impl SetDisks {
(new_disk, mod_time, None)
}
fn usable_fileinfo_count(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> (usize, bool) {
let mut has_read_error = false;
let mut usable_metadata = 0;
for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
if err.is_some() {
has_read_error = true;
continue;
}
if meta.is_valid() {
usable_metadata += 1;
}
}
(usable_metadata, has_read_error)
}
pub(super) fn latest_fileinfo_selection_quorum(
version_id: &str,
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
read_quorum: usize,
write_quorum: usize,
) -> usize {
if !version_id.is_empty() || write_quorum <= read_quorum {
return read_quorum;
}
let (usable_metadata, has_read_error) = Self::usable_fileinfo_count(parts_metadata, errs);
if usable_metadata < write_quorum {
return read_quorum;
}
if !has_read_error {
return write_quorum;
}
let mut identity_counts = HashMap::with_capacity(usable_metadata);
for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
if err.is_some() || !meta.is_valid() {
continue;
}
let key = Self::file_info_quorum_hash(meta);
let count = identity_counts.entry(key).or_insert(0);
*count += 1;
if *count >= write_quorum {
return write_quorum;
}
}
read_quorum
}
pub(super) fn select_valid_fileinfo(
disks: &[Option<DiskStore>],
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
version_id: &str,
read_quorum: usize,
write_quorum: usize,
) -> disk::error::Result<(Vec<Option<DiskStore>>, FileInfo, usize)> {
let selection_quorum =
Self::latest_fileinfo_selection_quorum(version_id, parts_metadata, errs, read_quorum, write_quorum);
let (usable_metadata, has_read_error) = Self::usable_fileinfo_count(parts_metadata, errs);
if version_id.is_empty()
&& write_quorum > read_quorum
&& has_read_error
&& usable_metadata >= write_quorum
&& selection_quorum == read_quorum
{
let (online_disks, fi) = Self::pick_degraded_latest_fileinfo(disks, parts_metadata, errs, read_quorum, write_quorum)?;
return Ok((online_disks, fi, read_quorum));
}
let (online_disks, mod_time, etag) = Self::list_online_disks(disks, parts_metadata, errs, selection_quorum);
let fi = Self::pick_valid_fileinfo(parts_metadata, mod_time, etag, selection_quorum)?;
Ok((online_disks, fi, selection_quorum))
}
pub(super) fn pick_valid_fileinfo(
metas: &[FileInfo],
mod_time: Option<OffsetDateTime>,
@@ -347,52 +439,271 @@ impl SetDisks {
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
}
fn update_hash_bytes(hasher: &mut Sha256, value: &[u8]) {
hasher.update(value.len().to_le_bytes());
hasher.update(value);
}
fn update_hash_str(hasher: &mut Sha256, value: &str) {
Self::update_hash_bytes(hasher, value.as_bytes());
}
fn update_hash_optional_uuid(hasher: &mut Sha256, value: Option<Uuid>) {
if let Some(value) = value {
hasher.update([1]);
hasher.update(value.as_bytes());
} else {
hasher.update([0]);
}
}
fn update_hash_optional_time(hasher: &mut Sha256, value: Option<OffsetDateTime>) {
if let Some(value) = value {
hasher.update([1]);
hasher.update(value.unix_timestamp_nanos().to_le_bytes());
} else {
hasher.update([0]);
}
}
fn update_hash_optional_u32(hasher: &mut Sha256, value: Option<u32>) {
if let Some(value) = value {
hasher.update([1]);
hasher.update(value.to_le_bytes());
} else {
hasher.update([0]);
}
}
fn update_hash_optional_u64(hasher: &mut Sha256, value: Option<u64>) {
if let Some(value) = value {
hasher.update([1]);
hasher.update(value.to_le_bytes());
} else {
hasher.update([0]);
}
}
fn update_hash_optional_bytes(hasher: &mut Sha256, value: Option<&Bytes>) {
if let Some(value) = value {
hasher.update([1]);
Self::update_hash_bytes(hasher, value);
} else {
hasher.update([0]);
}
}
fn update_hash_optional_str(hasher: &mut Sha256, value: Option<&str>) {
if let Some(value) = value {
hasher.update([1]);
Self::update_hash_str(hasher, value);
} else {
hasher.update([0]);
}
}
fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
value
.get(..prefix.len())
.is_some_and(|value_prefix| value_prefix.eq_ignore_ascii_case(prefix))
}
fn internal_metadata_suffix(name: &str) -> Option<&str> {
name.get(http::RUSTFS_INTERNAL_PREFIX.len()..)
.filter(|_| Self::starts_with_ignore_ascii_case(name, http::RUSTFS_INTERNAL_PREFIX))
.or_else(|| {
name.get(http::MINIO_INTERNAL_PREFIX.len()..)
.filter(|_| Self::starts_with_ignore_ascii_case(name, http::MINIO_INTERNAL_PREFIX))
})
}
fn is_replication_quorum_metadata_key(name: &str) -> bool {
if name.eq_ignore_ascii_case(http::AMZ_BUCKET_REPLICATION_STATUS) {
return true;
}
let Some(suffix) = Self::internal_metadata_suffix(name) else {
return false;
};
suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICA_STATUS)
|| suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICA_TIMESTAMP)
|| suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICATION_STATUS)
|| suffix.eq_ignore_ascii_case(http::SUFFIX_REPLICATION_TIMESTAMP)
|| suffix.eq_ignore_ascii_case(http::SUFFIX_PURGESTATUS)
|| Self::starts_with_ignore_ascii_case(suffix, http::SUFFIX_REPLICATION_RESET_ARN_PREFIX)
}
fn update_hash_quorum_metadata_map(hasher: &mut Sha256, entries: &HashMap<String, String>) {
let mut entries = entries
.iter()
.filter(|(name, _)| !Self::is_replication_quorum_metadata_key(name))
.collect::<Vec<_>>();
entries.sort_by(|left, right| left.0.cmp(right.0));
hasher.update(entries.len().to_le_bytes());
for (name, value) in entries {
Self::update_hash_str(hasher, name);
Self::update_hash_str(hasher, value);
}
}
fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
let mut hasher = Sha256::new();
Self::update_file_info_quorum_hash(&mut hasher, meta);
let digest = hasher.finalize();
let mut key = [0u8; 32];
key.copy_from_slice(digest.as_slice());
key
}
fn update_file_info_quorum_hash(hasher: &mut Sha256, meta: &FileInfo) {
hasher.update(meta.size.to_le_bytes());
hasher.update([meta.deleted as u8, meta.mark_deleted as u8]);
hasher.update([u8::from(meta.deleted), u8::from(meta.mark_deleted)]);
hasher.update([u8::from(meta.expire_restored)]);
Self::update_hash_optional_time(hasher, meta.mod_time);
Self::update_hash_str(hasher, &meta.transition_status);
Self::update_hash_str(hasher, &meta.transition_tier);
Self::update_hash_str(hasher, &meta.transitioned_objname);
Self::update_hash_optional_uuid(hasher, meta.transition_version_id);
Self::update_hash_optional_u32(hasher, meta.mode);
Self::update_hash_optional_u64(hasher, meta.written_by_version);
if let Some(version_id) = meta.version_id {
hasher.update(version_id.as_bytes());
}
Self::update_hash_optional_uuid(hasher, meta.version_id);
Self::update_hash_optional_uuid(hasher, meta.data_dir);
if let Some(data_dir) = meta.data_dir {
hasher.update(data_dir.as_bytes());
}
Self::update_hash_optional_bytes(hasher, meta.checksum.as_ref());
if let Some(checksum) = &meta.checksum {
hasher.update(checksum);
}
Self::update_hash_quorum_metadata_map(hasher, &meta.metadata);
hasher.update(meta.parts.len().to_le_bytes());
for part in meta.parts.iter() {
hasher.update(format!("part.{}", part.number).as_bytes());
hasher.update(format!("part.{}", part.size).as_bytes());
hasher.update(part.number.to_le_bytes());
hasher.update(part.size.to_le_bytes());
hasher.update(part.actual_size.to_le_bytes());
hasher.update(part.etag.as_bytes());
Self::update_hash_str(hasher, &part.etag);
if let Some(mod_time) = part.mod_time {
hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
}
Self::update_hash_optional_time(hasher, part.mod_time);
if let Some(index) = &part.index {
hasher.update(index);
}
Self::update_hash_optional_bytes(hasher, part.index.as_ref());
Self::update_hash_optional_str(hasher, part.error.as_deref());
if let Some(checksums) = &part.checksums {
let mut checksum_entries = checksums.iter().collect::<Vec<_>>();
checksum_entries.sort_by(|left, right| left.0.cmp(right.0));
hasher.update(checksum_entries.len().to_le_bytes());
for (name, value) in checksum_entries {
hasher.update(name.as_bytes());
hasher.update(value.as_bytes());
Self::update_hash_str(hasher, name);
Self::update_hash_str(hasher, value);
}
} else {
hasher.update(0usize.to_le_bytes());
}
}
if !meta.deleted && meta.size != 0 {
hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes());
hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes());
hasher.update(meta.erasure.data_blocks.to_le_bytes());
hasher.update(meta.erasure.parity_blocks.to_le_bytes());
hasher.update(meta.erasure.distribution.len().to_le_bytes());
for disk_index in meta.erasure.distribution.iter() {
hasher.update(disk_index.to_le_bytes());
}
}
}
fn latest_fileinfo_identity_groups(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<FileInfoIdentityGroup> {
let mut groups: Vec<FileInfoIdentityGroup> = Vec::with_capacity(parts_metadata.len());
for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
if err.is_some() || !meta.is_valid() {
continue;
}
let hash = Self::file_info_quorum_hash(meta);
if let Some(group) = groups.iter_mut().find(|group| group.hash == hash) {
group.count += 1;
continue;
}
groups.push(FileInfoIdentityGroup {
hash,
count: 1,
mod_time: meta.mod_time,
});
}
groups
}
fn pick_fileinfo_identity(
disks: &[Option<DiskStore>],
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
hash: [u8; 32],
quorum: usize,
) -> disk::error::Result<(Vec<Option<DiskStore>>, FileInfo)> {
let mut online_disks = vec![None; disks.len()];
let mut selected = None;
let mut count = 0;
for (i, ((meta, err), disk)) in parts_metadata.iter().zip(errs.iter()).zip(disks.iter()).enumerate() {
if err.is_some() || !meta.is_valid() || Self::file_info_quorum_hash(meta) != hash {
continue;
}
count += 1;
online_disks[i].clone_from(disk);
if selected.is_none() {
selected = Some(meta.clone());
}
}
if count < quorum {
return Err(DiskError::ErasureReadQuorum);
}
selected
.map(|mut fi| {
fi.is_latest = fi.successor_mod_time.is_none();
(online_disks, fi)
})
.ok_or(DiskError::ErasureReadQuorum)
}
fn pick_degraded_latest_fileinfo(
disks: &[Option<DiskStore>],
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
read_quorum: usize,
write_quorum: usize,
) -> disk::error::Result<(Vec<Option<DiskStore>>, FileInfo)> {
let mut groups = Self::latest_fileinfo_identity_groups(parts_metadata, errs);
if groups.is_empty() {
return Err(DiskError::ErasureReadQuorum);
}
groups.sort_by(|left, right| right.mod_time.cmp(&left.mod_time).then_with(|| right.count.cmp(&left.count)));
let latest_mod_time = groups[0].mod_time;
let mut older_start = 0;
while older_start < groups.len() && groups[older_start].mod_time == latest_mod_time {
if groups[older_start].count >= write_quorum {
return Self::pick_fileinfo_identity(disks, parts_metadata, errs, groups[older_start].hash, write_quorum);
}
older_start += 1;
}
if older_start > 1 {
return Err(DiskError::ErasureReadQuorum);
}
for group in groups.iter().skip(older_start) {
if group.count >= read_quorum {
return Self::pick_fileinfo_identity(disks, parts_metadata, errs, group.hash, read_quorum);
}
}
Err(DiskError::ErasureReadQuorum)
}
pub(super) fn find_file_info_in_quorum(
metas: &[FileInfo],
mod_time: &Option<OffsetDateTime>,
@@ -405,7 +716,6 @@ impl SetDisks {
}
let mut meta_hashes = vec![None; metas.len()];
let mut hasher = Sha256::new();
for (i, meta) in metas.iter().enumerate() {
if !meta.is_valid() {
@@ -437,8 +747,6 @@ impl SetDisks {
let mod_valid = mod_time == &meta.mod_time;
if etag_only || mod_valid {
Self::update_file_info_quorum_hash(&mut hasher, meta);
if meta.is_remote() {
// TODO:
}
@@ -447,9 +755,7 @@ impl SetDisks {
// TODO: IsCompressed
meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice()));
hasher.reset();
meta_hashes[i] = Some(Self::file_info_quorum_hash(meta));
} else {
debug!(
index = i,
@@ -462,7 +768,7 @@ impl SetDisks {
let mut count_map = HashMap::new();
for hash in meta_hashes.iter().flatten() {
for hash in meta_hashes.iter().flatten().copied() {
*count_map.entry(hash).or_insert(0) += 1;
}
@@ -495,7 +801,7 @@ impl SetDisks {
for (i, op_hash) in meta_hashes.iter().enumerate() {
if let Some(hash) = op_hash
&& let Some(max_hash) = max_val
&& hash == max_hash
&& *hash == max_hash
&& metas[i].is_valid()
{
if !found {
@@ -504,7 +810,7 @@ impl SetDisks {
}
let props = ObjProps {
mod_time: metas[i].mod_time,
successor_mod_time: metas[i].successor_mod_time,
num_versions: metas[i].num_versions,
};
@@ -517,9 +823,9 @@ impl SetDisks {
for (val, &count) in &valid_obj_map {
if count >= quorum {
fi.mod_time = val.mod_time;
fi.successor_mod_time = val.successor_mod_time;
fi.num_versions = val.num_versions;
fi.is_latest = val.mod_time.is_none();
fi.is_latest = val.successor_mod_time.is_none();
break;
}
+363 -14
View File
@@ -97,7 +97,8 @@ use rustfs_common::heal_channel::{
use rustfs_config::MI_B;
use rustfs_filemeta::{
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
RawFileInfo, ReplicateDecision, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw, merge_file_meta_versions,
RawFileInfo, ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw,
merge_file_meta_versions,
};
use rustfs_io_metrics::{
record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
@@ -3475,7 +3476,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
};
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
Ok((r, w)) => (r as usize, w as usize),
Ok((r, w)) => (
usize::try_from(r)
.map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![src_bucket, src_object]))?,
usize::try_from(w)
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![src_bucket, src_object]))?,
),
Err(mut err) => {
if err == DiskError::ErasureReadQuorum
&& !src_bucket.starts_with(RUSTFS_META_BUCKET)
@@ -3494,10 +3500,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
};
let (online_disks, mod_time, etag) = Self::list_online_disks(&disks, &metas, &errs, read_quorum);
let mut fi = Self::pick_valid_fileinfo(&metas, mod_time, etag, read_quorum)
.map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
let src_version_id = src_opts.version_id.as_deref().unwrap_or_default();
let (online_disks, mut fi, _) =
Self::select_valid_fileinfo(&disks, &metas, &errs, src_version_id, read_quorum, write_quorum)
.map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
if fi.deleted {
if src_opts.version_id.is_none() {
@@ -4078,8 +4084,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
};
let read_quorum = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
Ok((res, _)) => res,
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
Ok((read_quorum, write_quorum)) => (read_quorum, write_quorum),
Err(mut err) => {
if err == DiskError::ErasureReadQuorum
&& !bucket.starts_with(RUSTFS_META_BUCKET)
@@ -4098,11 +4104,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
};
let read_quorum = read_quorum as usize;
let read_quorum =
usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
let write_quorum = usize::try_from(write_quorum)
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
let (online_disks, mod_time, etag) = Self::list_online_disks(&disks, &metas, &errs, read_quorum);
let mut fi = Self::pick_valid_fileinfo(&metas, mod_time, etag, read_quorum)
let version_id = opts.version_id.as_deref().unwrap_or_default();
let (online_disks, mut fi, _) = Self::select_valid_fileinfo(&disks, &metas, &errs, version_id, read_quorum, write_quorum)
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
if fi.deleted {
@@ -6045,13 +6053,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
#[derive(Debug, PartialEq, Eq)]
struct ObjProps {
mod_time: Option<OffsetDateTime>,
successor_mod_time: Option<OffsetDateTime>,
num_versions: usize,
}
impl Hash for ObjProps {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.mod_time.hash(state);
self.successor_mod_time.hash(state);
self.num_versions.hash(state);
}
}
@@ -8062,6 +8070,34 @@ mod tests {
}
}
fn decoded_quorum_test_fileinfo_with_metadata(
mod_time: OffsetDateTime,
data_dir: Uuid,
part_etag: &str,
erasure_index: usize,
extra_metadata: &[(&str, &str)],
) -> FileInfo {
let mut fi = quorum_test_fileinfo(mod_time, data_dir, part_etag, erasure_index);
for (name, value) in extra_metadata {
fi.metadata.insert((*name).to_string(), (*value).to_string());
}
let mut meta = FileMeta::new();
meta.add_version(fi).expect("test file metadata should accept object version");
let encoded = meta.marshal_msg().expect("test file metadata should marshal");
rustfs_filemeta::get_file_info(
&encoded,
"bucket",
"object",
"",
rustfs_filemeta::FileInfoOpts {
data: false,
include_free_versions: false,
},
)
.expect("test file metadata should decode as file info")
}
#[test]
fn test_find_file_info_in_quorum_uses_part_identity() {
let mod_time = OffsetDateTime::now_utc();
@@ -8096,6 +8132,319 @@ mod tests {
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[test]
fn test_latest_fileinfo_selection_quorum_requires_write_quorum_when_full_metadata_is_available() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 1),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 2),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-b", 3),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-b", 4),
];
let errs = vec![None, None, None, None];
let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
assert_eq!(quorum, 3);
}
#[test]
fn test_latest_fileinfo_selection_quorum_preserves_read_quorum_for_version_or_degraded_reads() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 1),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 2),
FileInfo::default(),
FileInfo::default(),
];
let degraded_errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
let clean_errs = vec![None, None, None, None];
assert_eq!(SetDisks::latest_fileinfo_selection_quorum("", &metas, &degraded_errs, 2, 3), 2);
assert_eq!(SetDisks::latest_fileinfo_selection_quorum("version-id", &metas, &clean_errs, 2, 3), 2);
}
#[test]
fn test_latest_fileinfo_selection_quorum_keeps_read_quorum_for_partial_overwrite_with_read_error() {
let old_mod_time = OffsetDateTime::now_utc();
let new_mod_time = old_mod_time + time::Duration::seconds(1);
let old_data_dir = Uuid::new_v4();
let new_data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(old_mod_time, old_data_dir, "part-etag-old", 1),
quorum_test_fileinfo(old_mod_time, old_data_dir, "part-etag-old", 2),
quorum_test_fileinfo(new_mod_time, new_data_dir, "part-etag-new", 3),
FileInfo::default(),
];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
let (online_disks, mod_time, etag) = SetDisks::list_online_disks(&vec![None; metas.len()], &metas, &errs, quorum);
let fi = SetDisks::pick_valid_fileinfo(&metas, mod_time, etag, quorum)
.expect("old metadata should remain readable with read quorum");
assert_eq!(quorum, 2);
assert_eq!(online_disks.len(), metas.len());
assert_eq!(fi.data_dir, Some(old_data_dir));
assert_eq!(fi.parts[0].etag, "part-etag-old");
let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("old metadata should remain selectable with read quorum");
assert_eq!(selected_quorum, 2);
assert_eq!(selected.data_dir, Some(old_data_dir));
assert_eq!(selected.parts[0].etag, "part-etag-old");
assert!(selected.is_latest);
}
#[test]
fn test_latest_fileinfo_selection_rejects_partial_latest_read_quorum_with_read_error() {
let old_mod_time = OffsetDateTime::now_utc();
let new_mod_time = old_mod_time + time::Duration::seconds(1);
let old_data_dir = Uuid::new_v4();
let new_data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(new_mod_time, new_data_dir, "part-etag-new", 1),
quorum_test_fileinfo(new_mod_time, new_data_dir, "part-etag-new", 2),
quorum_test_fileinfo(old_mod_time, old_data_dir, "part-etag-old", 3),
FileInfo::default(),
];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
}
#[test]
fn test_latest_fileinfo_selection_preserves_degraded_read_quorum_without_competing_latest() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 1),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-old", 2),
FileInfo::default(),
FileInfo::default(),
];
let errs = vec![None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("read quorum should remain enough when no competing latest is visible");
assert_eq!(selected_quorum, 2);
assert_eq!(selected.data_dir, Some(data_dir));
assert_eq!(selected.parts[0].etag, "part-etag-old");
}
#[test]
fn test_latest_fileinfo_selection_ignores_derived_version_stack_drift() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let mut latest_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
latest_meta.is_latest = true;
latest_meta.num_versions = 1;
let mut stale_stack_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
stale_stack_meta.is_latest = false;
stale_stack_meta.successor_mod_time = Some(mod_time + time::Duration::seconds(1));
stale_stack_meta.num_versions = 2;
let mut newer_stack_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
newer_stack_meta.is_latest = false;
newer_stack_meta.successor_mod_time = Some(mod_time + time::Duration::seconds(2));
newer_stack_meta.num_versions = 3;
let metas = vec![latest_meta, stale_stack_meta, newer_stack_meta, FileInfo::default()];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("same object version should stay readable despite derived version stack drift");
assert_eq!(selected_quorum, 3);
assert_eq!(selected.data_dir, Some(data_dir));
assert_eq!(selected.parts[0].etag, "part-etag");
assert_eq!(selected.mod_time, Some(mod_time));
}
#[test]
fn test_latest_fileinfo_selection_uses_successor_mod_time_quorum_for_latest_flag() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let mut stale_stack_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
stale_stack_meta.is_latest = false;
stale_stack_meta.successor_mod_time = Some(mod_time + time::Duration::seconds(1));
stale_stack_meta.num_versions = 2;
let mut latest_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
latest_meta_a.is_latest = true;
latest_meta_a.num_versions = 1;
let mut latest_meta_b = latest_meta_a.clone();
latest_meta_b.erasure.index = 3;
let metas = vec![stale_stack_meta, latest_meta_a, latest_meta_b];
let selected = SetDisks::find_file_info_in_quorum(&metas, &Some(mod_time), &None, 2)
.expect("latest flag should be derived from successor mod time quorum");
assert!(selected.is_latest);
assert_eq!(selected.successor_mod_time, None);
assert_eq!(selected.num_versions, 1);
assert_eq!(selected.mod_time, Some(mod_time));
}
#[test]
fn test_latest_fileinfo_selection_ignores_replication_state_drift() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let replication_status_key = format!(
"{}{}",
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS
);
let replication_timestamp_key = format!(
"{}{}",
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
);
let replication_reset_key = format!(
"{}{}target-a",
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX
);
let meta_a = decoded_quorum_test_fileinfo_with_metadata(
mod_time,
data_dir,
"part-etag",
1,
&[
(&replication_status_key, "target-a=COMPLETED;"),
(&replication_timestamp_key, "2024-01-01T00:00:00Z"),
(&replication_reset_key, "COMPLETED"),
],
);
let meta_b = decoded_quorum_test_fileinfo_with_metadata(
mod_time,
data_dir,
"part-etag",
2,
&[
(&replication_status_key, "target-a=PENDING;"),
(&replication_timestamp_key, "2024-01-01T00:00:01Z"),
(&replication_reset_key, "PENDING"),
],
);
let meta_c = decoded_quorum_test_fileinfo_with_metadata(
mod_time,
data_dir,
"part-etag",
3,
&[
(&replication_status_key, "target-a=FAILED;"),
(&replication_timestamp_key, "2024-01-01T00:00:02Z"),
(&replication_reset_key, "FAILED"),
],
);
assert!(meta_a.replication_state_internal.is_some());
assert_eq!(
meta_a
.metadata
.get(rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS)
.map(String::as_str),
Some("COMPLETED")
);
let metas = vec![meta_a, meta_b, meta_c, FileInfo::default()];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let (_, selected, selected_quorum) = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3)
.expect("replication status drift should not split readable object identity");
assert_eq!(selected_quorum, 3);
assert_eq!(selected.data_dir, Some(data_dir));
assert_eq!(selected.parts[0].etag, "part-etag");
}
#[test]
fn test_latest_fileinfo_selection_rejects_same_modtime_metadata_split_without_write_quorum() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let mut old_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
let mut old_meta_b = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
let mut partial_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
old_meta_a.metadata.insert("x-amz-meta-color".to_string(), "blue".to_string());
old_meta_b.metadata.insert("x-amz-meta-color".to_string(), "blue".to_string());
partial_meta
.metadata
.insert("x-amz-meta-color".to_string(), "red".to_string());
let metas = vec![old_meta_a, old_meta_b, partial_meta, FileInfo::default()];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
assert_eq!(quorum, 2);
assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
}
#[test]
fn test_latest_fileinfo_selection_rejects_same_modtime_partial_metadata_read_quorum() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let mut old_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
let mut partial_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
let mut partial_meta_b = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
old_meta.metadata.insert("x-amz-meta-color".to_string(), "blue".to_string());
partial_meta_a
.metadata
.insert("x-amz-meta-color".to_string(), "red".to_string());
partial_meta_b
.metadata
.insert("x-amz-meta-color".to_string(), "red".to_string());
let metas = vec![old_meta, partial_meta_a, partial_meta_b, FileInfo::default()];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
}
#[test]
fn test_latest_fileinfo_selection_rejects_same_modtime_transition_split_without_write_quorum() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let old_meta_a = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 1);
let old_meta_b = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 2);
let mut partial_meta = quorum_test_fileinfo(mod_time, data_dir, "part-etag", 3);
partial_meta.transition_status = TRANSITION_COMPLETE.to_string();
partial_meta.transition_tier = "WARM".to_string();
partial_meta.transitioned_objname = "remote/object".to_string();
partial_meta.transition_version_id = Some(Uuid::new_v4());
let metas = vec![old_meta_a, old_meta_b, partial_meta, FileInfo::default()];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
let quorum = SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3);
let result = SetDisks::select_valid_fileinfo(&vec![None; metas.len()], &metas, &errs, "", 2, 3);
assert_eq!(quorum, 2);
assert!(matches!(result, Err(DiskError::ErasureReadQuorum)));
}
#[test]
fn test_latest_fileinfo_selection_quorum_uses_write_quorum_for_degraded_committed_identity() {
let mod_time = OffsetDateTime::now_utc();
let data_dir = Uuid::new_v4();
let metas = vec![
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 1),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 2),
quorum_test_fileinfo(mod_time, data_dir, "part-etag-a", 3),
FileInfo::default(),
];
let errs = vec![None, None, None, Some(DiskError::DiskNotFound)];
assert_eq!(SetDisks::latest_fileinfo_selection_quorum("", &metas, &errs, 2, 3), 3);
}
#[test]
fn test_list_object_parities() {
// Test extracting parity counts from file info
+6 -6
View File
@@ -2362,7 +2362,7 @@ impl SetDisks {
let _min_disks = self.set_drive_count - self.default_parity_count;
let metadata_resolve_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
let (read_quorum, _) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))
{
Ok(v) => v,
@@ -2378,8 +2378,8 @@ impl SetDisks {
};
let read_quorum =
usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, read_quorum);
let write_quorum = usize::try_from(write_quorum)
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
record_get_stage_duration_if_enabled(
@@ -2390,9 +2390,9 @@ impl SetDisks {
return Err(to_object_err(err.into(), vec![bucket, object]));
}
let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum);
let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum)?;
let (op_online_disks, fi, fileinfo_selection_quorum) =
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, fileinfo_selection_quorum);
if errs.iter().any(|err| err.is_some()) {
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
submit_read_repair_heal(
File diff suppressed because it is too large Load Diff
+160 -5
View File
@@ -271,6 +271,18 @@ pub struct FileMetaShallowVersion {
pub meta: Vec<u8>, // FileMetaVersion.marshal_msg
}
fn write_quorum_from_erasure(data_blocks: usize, parity_blocks: usize) -> Option<usize> {
if data_blocks == 0 {
return None;
}
Some(if data_blocks == parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
impl FileMetaShallowVersion {
/// Parse version meta with legacy format compatibility.
/// Use this instead of `FileMetaVersion::default()` + `unmarshal_msg()` to handle old-version xl.meta.
@@ -278,6 +290,18 @@ impl FileMetaShallowVersion {
FileMetaVersion::try_from(self.meta.as_slice())
}
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
let header_quorum = self.header.write_quorum(fallback_quorum);
if self.header.has_ec() || !matches!(self.header.version_type, VersionType::Object | VersionType::Legacy) {
return header_quorum;
}
self.parse_version_meta()
.ok()
.and_then(|version| version.erasure_write_quorum())
.unwrap_or(header_quorum)
}
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
let file_version = self.parse_version_meta()?;
Ok(file_version.into_fileinfo(volume, path, all_parts))
@@ -312,6 +336,23 @@ pub struct FileMetaVersion {
}
impl FileMetaVersion {
fn erasure_write_quorum(&self) -> Option<usize> {
let (data_blocks, parity_blocks) = match self.version_type {
VersionType::Object | VersionType::Legacy => {
if let Some(object) = &self.object {
(object.erasure_m, object.erasure_n)
} else if let Some(object) = &self.legacy_object {
(object.erasure.data_blocks, object.erasure.parity_blocks)
} else {
return None;
}
}
_ => return None,
};
write_quorum_from_erasure(data_blocks, parity_blocks)
}
fn decode_data_dir_from_v2_object(buf: &[u8]) -> Result<Option<Uuid>> {
let mut cur = std::io::Cursor::new(buf);
let mut fields = rmp::decode::read_map_len(&mut cur)?;
@@ -775,6 +816,14 @@ impl FileMetaVersionHeader {
self.ec_m > 0 && self.ec_n > 0
}
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
if self.version_type != VersionType::Object || !self.has_ec() {
return fallback_quorum;
}
write_quorum_from_erasure(usize::from(self.ec_m), usize::from(self.ec_n)).unwrap_or(fallback_quorum)
}
pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool {
let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o);
if self.version_id.is_none() {
@@ -2543,9 +2592,28 @@ pub enum Flags {
// mergeXLV2Versions
pub fn merge_file_meta_versions(
quorum: usize,
strict: bool,
requested_versions: usize,
versions: &[Vec<FileMetaShallowVersion>],
) -> Vec<FileMetaShallowVersion> {
merge_file_meta_versions_inner(quorum, strict, requested_versions, false, versions)
}
pub(crate) fn merge_file_meta_versions_with_write_quorum(
quorum: usize,
strict: bool,
requested_versions: usize,
versions: &[Vec<FileMetaShallowVersion>],
) -> Vec<FileMetaShallowVersion> {
merge_file_meta_versions_inner(quorum, strict, requested_versions, true, versions)
}
fn merge_file_meta_versions_inner(
mut quorum: usize,
mut strict: bool,
requested_versions: usize,
enforce_write_quorum: bool,
versions: &[Vec<FileMetaShallowVersion>],
) -> Vec<FileMetaShallowVersion> {
if quorum == 0 {
@@ -2557,13 +2625,32 @@ pub fn merge_file_meta_versions(
}
if versions.len() == 1 {
return versions[0].clone();
if !enforce_write_quorum {
return versions[0].clone();
}
let required_quorum = versions[0]
.first()
.map(|version| version.write_quorum(quorum).max(quorum))
.unwrap_or(quorum);
if versions.len() >= required_quorum {
return versions[0].clone();
}
return Vec::new();
}
if quorum == 1 {
strict = true;
}
let required_quorum = |version: &FileMetaShallowVersion| {
if enforce_write_quorum {
version.write_quorum(quorum).max(quorum)
} else {
quorum
}
};
let mut versions = versions.to_owned();
let mut n_versions = 0;
@@ -2594,9 +2681,12 @@ pub fn merge_file_meta_versions(
let mut latest = FileMetaShallowVersion::default();
if consistent {
merged.push(tops[0].clone());
if !tops[0].header.free_version() {
n_versions += 1;
latest = tops[0].clone();
if tops.len() >= required_quorum(&latest) {
merged.push(latest.clone());
if !latest.header.free_version() {
n_versions += 1;
}
}
} else {
let mut latest_count = 0;
@@ -2659,7 +2749,7 @@ pub fn merge_file_meta_versions(
break;
}
}
if latest_count >= quorum {
if latest_count >= required_quorum(&latest) {
if !latest.header.free_version() {
n_versions += 1;
}
@@ -3053,6 +3143,39 @@ mod tests {
wr
}
fn encode_legacy_v2_object_body(data_blocks: usize, parity_blocks: usize) -> Vec<u8> {
let drive_count = data_blocks + parity_blocks;
let payload = LegacyObjectVersionFixture {
version_type: LegacyObjectVersionTypeFixture::Object,
object: Some(LegacyObjectFixture {
version_id: Some(sample_version_id().as_bytes().to_vec()),
data_dir: Some(Uuid::from_u128(42).as_bytes().to_vec()),
erasure_algorithm: "ReedSolomon".to_string(),
erasure_m: data_blocks,
erasure_n: parity_blocks,
erasure_block_size: 1_048_576,
erasure_index: 1,
erasure_dist: (1..=drive_count)
.map(|idx| u8::try_from(idx).expect("test drive index should fit u8"))
.collect(),
bitrot_checksum_algo: "HighwayHash".to_string(),
part_numbers: vec![1],
part_etags: vec!["etag-1".to_string()],
part_sizes: vec![11],
part_actual_sizes: vec![11],
part_indices: vec![Vec::new()],
size: 11,
mod_time: Some(sample_mod_time()),
meta_sys: HashMap::new(),
meta_user: HashMap::from([("content-type".to_string(), "text/plain".to_string())]),
}),
delete_marker: None,
write_version: 3,
};
rmp_serde::to_vec_named(&payload).expect("legacy object payload should marshal")
}
#[test]
fn version_header_unmarshal_v1_uses_legacy_layout_defaults() {
let expected = sample_header();
@@ -3087,6 +3210,38 @@ mod tests {
assert_eq!(decoded.ec_m, 0);
}
#[test]
fn shallow_version_write_quorum_uses_legacy_object_payload_when_header_lacks_ec() {
let expected = sample_header();
let encoded = encode_v2_header(&expected);
let mut header = FileMetaVersionHeader::default();
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
let version = FileMetaShallowVersion {
header,
meta: encode_legacy_v2_object_body(7, 1),
};
assert_eq!(version.header.write_quorum(5), 5);
assert_eq!(version.write_quorum(5), 7);
}
#[test]
fn shallow_version_write_quorum_uses_zero_parity_object_payload() {
let expected = sample_header();
let encoded = encode_v2_header(&expected);
let mut header = FileMetaVersionHeader::default();
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
let version = FileMetaShallowVersion {
header,
meta: encode_legacy_v2_object_body(4, 0),
};
assert_eq!(version.header.write_quorum(2), 2);
assert_eq!(version.write_quorum(2), 4);
}
#[test]
fn version_header_unmarshal_v3_round_trips_current_layout() {
let expected = sample_header();
+335 -11
View File
@@ -14,7 +14,7 @@
use crate::{
Error, FileInfo, FileInfoOpts, FileInfoVersions, FileMeta, FileMetaShallowVersion, Result, VersionType, get_file_info,
merge_file_meta_versions,
merge_file_meta_versions, merge_file_meta_versions_with_write_quorum,
};
use arc_swap::ArcSwapOption;
use rmp::Marker;
@@ -334,7 +334,15 @@ impl MetaCacheEntries {
&self.0
}
pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
pub fn resolve(&self, params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
self.resolve_inner(params, false)
}
pub fn resolve_with_write_quorum(&self, params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
self.resolve_inner(params, true)
}
fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option<MetaCacheEntry> {
if self.0.is_empty() {
debug!(
bucket = %params.bucket,
@@ -412,7 +420,7 @@ impl MetaCacheEntries {
return Some(selected);
}
// If we would never be able to reach read quorum.
// If we would never be able to reach the required object quorum.
if objs_valid < params.obj_quorum {
debug!(
objs_valid,
@@ -423,13 +431,26 @@ impl MetaCacheEntries {
}
if objs_agree == objs_valid {
debug!(
selected = %selected.name,
objs_agree,
objs_valid,
"metacache resolve reused selected candidate because all valid object entries agreed"
);
return Some(selected);
let required_quorum = if enforce_write_quorum {
selected
.cached
.as_ref()
.and_then(|cached| cached.versions.first())
.map(|version| version.write_quorum(params.obj_quorum).max(params.obj_quorum))
.unwrap_or(params.obj_quorum)
} else {
params.obj_quorum
};
if objs_agree >= required_quorum {
debug!(
selected = %selected.name,
objs_agree,
objs_valid,
"metacache resolve reused selected candidate because all valid object entries agreed"
);
return Some(selected);
}
}
let Some(cached) = selected.cached else {
@@ -437,7 +458,16 @@ impl MetaCacheEntries {
return None;
};
let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
let versions = if enforce_write_quorum {
merge_file_meta_versions_with_write_quorum(
params.obj_quorum,
params.strict,
params.requested_versions,
&params.candidates,
)
} else {
merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates)
};
if versions.is_empty() {
debug!(
selected = %selected.name,
@@ -1272,6 +1302,300 @@ mod tests {
assert_ne!(extended_versions, cached.versions.len());
}
fn metacache_entry_with_mod_time(mod_time: OffsetDateTime, etag: &str) -> MetaCacheEntry {
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), etag.to_string());
let mut meta = FileMeta::new();
meta.add_version(FileInfo {
volume: "bucket".to_string(),
name: "object".to_string(),
size: 1,
mod_time: Some(mod_time),
metadata,
..Default::default()
})
.expect("test file metadata should accept object version");
let encoded = meta.marshal_msg().expect("test file metadata should marshal");
MetaCacheEntry {
name: "object".to_string(),
metadata: encoded,
cached: Some(meta),
reusable: false,
}
}
fn metacache_entry_with_erasure(
mod_time: OffsetDateTime,
etag: &str,
data_blocks: usize,
parity_blocks: usize,
) -> MetaCacheEntry {
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), etag.to_string());
let mut fi = FileInfo::new("object", data_blocks, parity_blocks);
fi.volume = "bucket".to_string();
fi.name = "object".to_string();
fi.size = 1;
fi.mod_time = Some(mod_time);
fi.metadata = metadata;
let mut meta = FileMeta::new();
meta.add_version(fi).expect("test file metadata should accept object version");
let encoded = meta.marshal_msg().expect("test file metadata should marshal");
MetaCacheEntry {
name: "object".to_string(),
metadata: encoded,
cached: Some(meta),
reusable: false,
}
}
fn metacache_entry_with_erasure_versions(versions: &[(OffsetDateTime, &str, usize, usize)]) -> MetaCacheEntry {
let mut meta = FileMeta::new();
for (idx, (mod_time, etag, data_blocks, parity_blocks)) in versions.iter().enumerate() {
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), (*etag).to_string());
let mut fi = FileInfo::new("object", *data_blocks, *parity_blocks);
fi.volume = "bucket".to_string();
fi.name = "object".to_string();
let version_idx = u128::try_from(idx + 1).expect("test version index should fit u128");
fi.version_id = Some(Uuid::from_u128(version_idx));
fi.versioned = true;
fi.size = 1;
fi.mod_time = Some(*mod_time);
fi.metadata = metadata;
meta.add_version(fi).expect("test file metadata should accept object version");
}
let encoded = meta.marshal_msg().expect("test file metadata should marshal");
MetaCacheEntry {
name: "object".to_string(),
metadata: encoded,
cached: Some(meta),
reusable: false,
}
}
fn metacache_entry_without_header_ec(mut entry: MetaCacheEntry) -> MetaCacheEntry {
let mut cached = entry.cached.take().expect("test entry should have cached metadata");
for version in cached.versions.iter_mut() {
version.header.ec_m = 0;
version.header.ec_n = 0;
}
entry.metadata = cached.marshal_msg().expect("test file metadata should marshal");
entry.cached = Some(cached);
entry
}
fn metacache_dir_entry(name: &str) -> MetaCacheEntry {
MetaCacheEntry {
name: name.to_string(),
..Default::default()
}
}
#[test]
fn resolve_rejects_partial_latest_and_returns_committed_previous_metadata() {
let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let old_entry = metacache_entry_with_mod_time(old_mod_time, "old-etag");
let new_entry = metacache_entry_with_mod_time(new_mod_time, "new-etag");
let resolved = MetaCacheEntries(vec![
Some(new_entry.clone()),
Some(new_entry),
Some(old_entry.clone()),
Some(old_entry.clone()),
Some(old_entry.clone()),
Some(old_entry.clone()),
Some(old_entry),
])
.resolve(MetadataResolutionParams {
obj_quorum: 5,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
})
.expect("previous committed metadata should still satisfy write quorum");
let info = resolved
.to_fileinfo("bucket")
.expect("resolved committed metadata should decode as file info");
assert_eq!(info.mod_time, Some(old_mod_time));
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
}
#[test]
fn resolve_rejects_low_parity_partial_latest_below_required_object_quorum() {
let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 7, 1);
let new_entry = metacache_entry_with_erasure(new_mod_time, "new-etag", 7, 1);
let resolved = MetaCacheEntries(vec![
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry),
Some(old_entry.clone()),
Some(old_entry),
])
.resolve_with_write_quorum(MetadataResolutionParams {
obj_quorum: 5,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
});
assert!(resolved.is_none());
}
#[test]
fn resolve_rejects_zero_parity_partial_latest_below_required_object_quorum() {
let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let new_entry = metacache_entry_with_erasure(new_mod_time, "new-etag", 7, 0);
let resolved = MetaCacheEntries(vec![
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry),
])
.resolve_with_write_quorum(MetadataResolutionParams {
obj_quorum: 5,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
});
assert!(resolved.is_none());
}
#[test]
fn resolve_accepts_low_parity_latest_at_required_object_quorum() {
let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let new_entry = metacache_entry_with_erasure(new_mod_time, "new-etag", 7, 1);
let resolved = MetaCacheEntries(vec![
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry.clone()),
Some(new_entry),
])
.resolve_with_write_quorum(MetadataResolutionParams {
obj_quorum: 5,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
})
.expect("latest metadata should resolve after satisfying its own write quorum");
let info = resolved
.to_fileinfo("bucket")
.expect("resolved committed metadata should decode as file info");
assert_eq!(info.mod_time, Some(new_mod_time));
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new-etag"));
}
#[test]
fn resolve_skips_low_parity_partial_latest_and_returns_committed_previous_version() {
let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 4, 4);
let new_and_old_entry =
metacache_entry_with_erasure_versions(&[(old_mod_time, "old-etag", 4, 4), (new_mod_time, "new-etag", 7, 1)]);
let resolved = MetaCacheEntries(vec![
Some(new_and_old_entry.clone()),
Some(new_and_old_entry.clone()),
Some(new_and_old_entry.clone()),
Some(new_and_old_entry.clone()),
Some(new_and_old_entry),
Some(old_entry.clone()),
Some(old_entry),
])
.resolve_with_write_quorum(MetadataResolutionParams {
obj_quorum: 5,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
})
.expect("previous committed metadata should resolve after rejecting partial latest");
let info = resolved
.to_fileinfo("bucket")
.expect("resolved committed metadata should decode as file info");
assert_eq!(info.mod_time, Some(old_mod_time));
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
}
#[test]
fn resolve_skips_legacy_header_low_parity_partial_latest_using_payload_quorum() {
let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
let new_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp");
let old_entry = metacache_entry_with_erasure(old_mod_time, "old-etag", 4, 4);
let new_and_old_entry = metacache_entry_without_header_ec(metacache_entry_with_erasure_versions(&[
(old_mod_time, "old-etag", 4, 4),
(new_mod_time, "new-etag", 7, 1),
]));
let resolved = MetaCacheEntries(vec![
Some(new_and_old_entry.clone()),
Some(new_and_old_entry.clone()),
Some(new_and_old_entry.clone()),
Some(new_and_old_entry.clone()),
Some(new_and_old_entry),
Some(old_entry.clone()),
Some(old_entry),
])
.resolve_with_write_quorum(MetadataResolutionParams {
obj_quorum: 5,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
})
.expect("previous committed metadata should resolve after rejecting legacy-header partial latest");
let info = resolved
.to_fileinfo("bucket")
.expect("resolved committed metadata should decode as file info");
assert_eq!(info.mod_time, Some(old_mod_time));
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("old-etag"));
}
#[test]
fn resolve_rejects_partial_directory_below_dir_quorum() {
let partial_dir = metacache_dir_entry("prefix/");
let resolved = MetaCacheEntries(vec![Some(partial_dir.clone()), Some(partial_dir)]).resolve(MetadataResolutionParams {
dir_quorum: 5,
obj_quorum: 5,
bucket: "bucket".to_string(),
strict: true,
..Default::default()
});
assert!(resolved.is_none());
}
fn build_hashmap_cache(update_size: usize) -> Arc<Cache<HashMap<usize, usize>>> {
let generation = Arc::new(AtomicUsize::new(0));
Arc::new(Cache::new(