gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 19:16:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(core-storage): fix critical correctness defects from core-storage reliability audit (#4222)

Browse Source 
* fix(core-storage): fix critical correctness defects from core-storage audit

Fixes verified defects found in a deep audit of the core storage path
(erasure coding, disk persistence, quorum, heal, replication resync):

- ecstore/disk: rewrite live xl.meta atomically (temp+rename) in
  delete_versions_internal and write_metadata instead of in-place
  truncate, which exposed torn metadata to concurrent readers and
  crashes on the DeleteObjects hot path
- ecstore/erasure: allow heal to reconstruct from exactly data_shards
  bitrot-verified sources; requiring data_shards+1 made objects
  permanently unhealable after losing parity_shards disks
- ecstore/set_disk: direct-memory inline GET applied the erasure
  distribution permutation twice (shuffled inputs re-indexed through
  distribution), concatenating wrong shards into the response body in
  degraded reads; collect from canonical disk-ordered inputs
- ecstore/set_disk: heal now preserves the committed inline layout
  instead of recomputing it with a hardcoded unversioned threshold,
  which split quorum identity of healed replicas and caused endless
  re-heal churn
- ecstore/replication: resync results channel switched from
  broadcast(1) to mpsc; a lagged broadcast receiver ended the stats
  collector and every subsequent failure went uncounted, letting
  failed resyncs be marked completed
- ecstore/replication: ignore an empty persisted resync checkpoint;
  resuming with one skipped every object and marked the resync
  completed without replicating anything
- ecstore/replication: fix inverted not-found error classification in
  replicate_object/replicate_delete logging paths
- ecstore/erasure: guard decode paths against zero block_size or
  data_shards from corrupt on-disk metadata (divide-by-zero panic)
- ecstore/disk: os::read_dir no longer consumes the entry limit on
  entries it does not return (is_empty_dir misjudgment); create_file
  opens with O_TRUNC to avoid stale trailing bytes
- filemeta: treat Some(nil) version id as a null version in
  matches_not_strict; disk-loaded headers never store None, so the
  mod_time quorum guard for unversioned overwrites never fired and an
  interrupted overwrite could displace the committed version in merge
- filemeta: fix msgpack skip lengths for fixext (missed the ext type
  byte) and ext16/32 (over-skipped) unknown fields
- filemeta: return FileCorrupt instead of usize underflow when
  xl.meta is truncated inside the CRC trailer
- filemeta: surface delete-marker insertion failure in delete_version
  instead of reporting success when the data dir is shared

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(replication): drop duplicate cfg(test) etag import from boundary module

The test module already imports content_matches_by_etag locally, so the
top-level cfg(test) import is unused under -D warnings and fails clippy.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Zhengchao An
2026-07-03 11:37:02 +08:00
committed by GitHub
parent aecac5c0ae
commit cf056b39e3
10 changed files with 89 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/heal.rs
+5 -4
View File
@@ -441,10 +441,11 @@ impl SetDisks {
}
}
let is_inline_buffer = runtime_sources::storage_class_should_inline(
erasure.shard_file_size(latest_meta.size),
false,
);
// Preserve the committed layout: recomputing inline-ness here
// (with a hardcoded unversioned threshold) makes healed replicas
// diverge from healthy ones in quorum identity, so heal would
// flag them forever.
let is_inline_buffer = latest_meta.inline_data();
// create writers for all disk positions, but only for outdated disks
for (index, disk_op) in out_dated_disks.iter().enumerate() {
if let Some(outdated_disk) = disk_op {
crates/ecstore/src/set_disk/read.rs
+15 -2
View File
@@ -2527,7 +2527,6 @@ impl SetDisks {
return Ok(None);
}
let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi);
let checksum_info = fi.erasure.get_checksum_info(part.number);
let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
HashAlgorithm::HighwayHash256SLegacy
@@ -2537,7 +2536,11 @@ impl SetDisks {
let read_length = erasure.shard_file_offset(0, object_size, object_size);
if fi.data.is_some() {
let Some(data_files) = collect_inline_data_shard_fileinfos_by_index(&files, fi, erasure.data_shards, |index| {
// Collect from the canonical disk-ordered inputs: the helper indexes
// fi.erasure.distribution by disk position, so passing shard-ordered
// (shuffled) arrays would apply the permutation twice and concatenate
// the wrong shards into the response body.
let Some(data_files) = collect_inline_data_shard_fileinfos_by_index(files, fi, erasure.data_shards, |index| {
disks.get(index).is_some_and(Option::is_some)
}) else {
return Ok(None);
@@ -2587,6 +2590,7 @@ impl SetDisks {
return Ok(body);
}
let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi);
let use_mmap_read = object_mmap_read_enabled();
let reader_setup_stage_start = Instant::now();
@@ -2746,6 +2750,15 @@ impl SetDisks {
fi.uses_legacy_checksum,
);
// Erasure params come from on-disk metadata; zero values must fail the read
// instead of panicking on the block/shard divisions below.
if erasure.block_size == 0 || erasure.data_shards == 0 {
return Err(Error::other(format!(
"invalid erasure metadata for {bucket}/{object}: block_size={}, data_blocks={}",
erasure.block_size, erasure.data_shards
)));
}
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");