gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 18:28:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): recover late parity after exact quorum (#6927)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-31 11:26:20 +08:00
committed by GitHub
parent c876df53f5
commit 45a2ccb734
5 changed files with 740 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/read.rs
+153 -13
View File
@@ -609,7 +609,20 @@ impl SetDisks {
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks))
if !metadata_fanout_complete
&& allow_early_stop
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
&& late_materialization_candidate_is_safe(&fi)
{
Ok(GetObjectFileInfo::owned_with_late_metadata_fanout(
fi,
parts_metadata,
op_online_disks,
disks,
))
} else {
Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks))
}
}
#[hotpath::measure(impl_type = "SetDisks")]
@@ -819,6 +832,7 @@ impl SetDisks {
pool_index: usize,
skip_verify_bitrot: bool,
prefer_data_blocks_first_reader_setup: bool,
require_reconstruction_surplus: bool,
metrics_path: &'static str,
metrics_object_class: &'static str,
metrics_size_bucket: &'static str,
@@ -1083,6 +1097,9 @@ impl SetDisks {
}
let nil_count = reader_setup.available_shards();
if require_reconstruction_surplus && nil_count <= erasure.data_shards {
return Err(Error::other("insufficient reconstruction surplus for two-phase read"));
}
if nil_count < erasure.data_shards {
if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards)
{
@@ -1190,18 +1207,34 @@ impl SetDisks {
let readers = reader_setup.readers;
let deferred_stripe_handles = reader_setup.deferred_stripe_handles;
let deferred_reopeners = reader_setup.deferred_reopeners;
let (written, err) = erasure
.decode_with_stripe_handles_and_reopeners(
writer,
readers,
part_offset,
part_length,
part_size,
read_costs,
deferred_stripe_handles,
deferred_reopeners,
)
.await;
let (written, err, exact_quorum) = if require_reconstruction_surplus {
erasure
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
writer,
readers,
part_offset,
part_length,
part_size,
read_costs,
deferred_stripe_handles,
deferred_reopeners,
)
.await
} else {
let (written, err) = erasure
.decode_with_stripe_handles_and_reopeners(
writer,
readers,
part_offset,
part_length,
part_size,
read_costs,
deferred_stripe_handles,
deferred_reopeners,
)
.await;
(written, err, false)
};
let decode_elapsed = decode_stage_start.elapsed();
rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64());
rustfs_io_metrics::record_get_object_stage_duration_by_size(
@@ -1211,6 +1244,9 @@ impl SetDisks {
metrics_size_bucket,
decode_elapsed.as_secs_f64(),
);
if exact_quorum && err.is_none() {
return Err(Error::other("two-phase read completed with exact reconstruction quorum"));
}
if decode_elapsed >= SLOW_OBJECT_READ_LOG_THRESHOLD && err.is_none() {
warn!(
event = EVENT_SET_DISK_READ,
@@ -1758,6 +1794,102 @@ fn multipart_reader_setup_prefetch_enabled(policy: GetObjectReadPolicy) -> bool
policy.allows_multipart_setup_prefetch() && is_multipart_reader_setup_prefetch_enabled()
}
pub(super) struct LateMetadataIdentity {
volume: String,
name: String,
algorithm: String,
block_size: usize,
uses_legacy_checksum: bool,
quorum_hash: [u8; 32],
distribution: Vec<usize>,
parity_blocks: usize,
}
impl LateMetadataIdentity {
pub(super) fn from_file_info(file_info: &FileInfo) -> Self {
Self {
volume: file_info.volume.clone(),
name: file_info.name.clone(),
algorithm: file_info.erasure.algorithm.clone(),
block_size: file_info.erasure.block_size,
uses_legacy_checksum: file_info.uses_legacy_checksum,
quorum_hash: SetDisks::file_info_quorum_hash(file_info),
distribution: file_info.erasure.distribution.clone(),
parity_blocks: file_info.erasure.parity_blocks,
}
}
}
fn late_metadata_read_identity_matches(expected: &LateMetadataIdentity, actual: &FileInfo) -> bool {
expected.volume == actual.volume
&& expected.name == actual.name
&& expected.algorithm == actual.erasure.algorithm
&& expected.block_size == actual.erasure.block_size
&& expected.uses_legacy_checksum == actual.uses_legacy_checksum
&& expected.quorum_hash == SetDisks::file_info_quorum_hash(actual)
}
fn late_metadata_shard_matches(expected: &LateMetadataIdentity, actual: &FileInfo, disk_index: usize) -> bool {
expected
.distribution
.get(disk_index)
.is_some_and(|mapped_index| *mapped_index == actual.erasure.index)
&& late_metadata_read_identity_matches(expected, actual)
}
impl SetDisks {
pub(super) async fn refresh_late_metadata_fanout(
fallback_disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
expected: &LateMetadataIdentity,
metrics_path: &'static str,
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
let (mut parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed(
fallback_disks,
"",
bucket,
object,
"",
true,
false,
false,
false,
expected.parity_blocks,
)
.await?;
diagnostics.record(metrics_path);
let (read_quorum, write_quorum) = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, expected.parity_blocks)
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
let read_quorum =
usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
let write_quorum = usize::try_from(write_quorum)
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
return Err(to_object_err(err.into(), vec![bucket, object]));
}
let (mut online_disks, full_fi, _) =
SetDisks::select_valid_fileinfo(fallback_disks, &parts_metadata, &errs, "", read_quorum, write_quorum)?;
if !late_metadata_read_identity_matches(expected, &full_fi) {
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
}
for (disk_index, (metadata, disk)) in parts_metadata.iter_mut().zip(online_disks.iter_mut()).enumerate() {
if !late_metadata_shard_matches(expected, metadata, disk_index) {
*metadata = FileInfo::default();
*disk = None;
}
}
if online_disks.iter().filter(|disk| disk.is_some()).count() < read_quorum {
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
}
Ok((full_fi, parts_metadata, online_disks))
}
}
/// Run one part's bitrot reader setup and measure its wall-clock duration.
///
/// Shared by the synchronous path and the prefetch task in
@@ -2349,6 +2481,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2380,6 +2513,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2404,6 +2538,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2426,6 +2561,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2450,6 +2586,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2488,6 +2625,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"empty",
@@ -2521,6 +2659,7 @@ mod metadata_cache_tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -4882,6 +5021,7 @@ mod tests {
0,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"test-object-class",
"test-size-bucket",