gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-30 16:59:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): handle stalled recovery reads and listings (#3790)

Browse Source 
* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
GatewayJ
2026-06-27 10:21:09 +08:00
committed by GitHub
parent 3fb4dcd52e
commit 675597ec16
28 changed files with 2991 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/metadata/set_disk.rs
+54 -10
View File
@@ -347,6 +347,52 @@ impl SetDisks {
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
}
fn update_file_info_quorum_hash(hasher: &mut Sha256, meta: &FileInfo) {
hasher.update(meta.size.to_le_bytes());
hasher.update([meta.deleted as u8, meta.mark_deleted as u8]);
if let Some(version_id) = meta.version_id {
hasher.update(version_id.as_bytes());
}
if let Some(data_dir) = meta.data_dir {
hasher.update(data_dir.as_bytes());
}
if let Some(checksum) = &meta.checksum {
hasher.update(checksum);
}
for part in meta.parts.iter() {
hasher.update(format!("part.{}", part.number).as_bytes());
hasher.update(format!("part.{}", part.size).as_bytes());
hasher.update(part.actual_size.to_le_bytes());
hasher.update(part.etag.as_bytes());
if let Some(mod_time) = part.mod_time {
hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
}
if let Some(index) = &part.index {
hasher.update(index);
}
if let Some(checksums) = &part.checksums {
let mut checksum_entries = checksums.iter().collect::<Vec<_>>();
checksum_entries.sort_by(|left, right| left.0.cmp(right.0));
for (name, value) in checksum_entries {
hasher.update(name.as_bytes());
hasher.update(value.as_bytes());
}
}
}
if !meta.deleted && meta.size != 0 {
hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes());
hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes());
}
}
pub(super) fn find_file_info_in_quorum(
metas: &[FileInfo],
mod_time: &Option<OffsetDateTime>,
@@ -387,15 +433,7 @@ impl SetDisks {
let mod_valid = mod_time == &meta.mod_time;
if etag_only || mod_valid {
for part in meta.parts.iter() {
hasher.update(format!("part.{}", part.number).as_bytes());
hasher.update(format!("part.{}", part.size).as_bytes());
}
if !meta.deleted && meta.size != 0 {
hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes());
hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes());
}
Self::update_file_info_quorum_hash(&mut hasher, meta);
if meta.is_remote() {
// TODO:
@@ -435,7 +473,13 @@ impl SetDisks {
}
if max_count < quorum {
warn!("find_file_info_in_quorum: max_count < quorum, max_val={:?}", max_val);
warn!(
quorum,
max_count,
max_val = ?max_val,
count_map = ?count_map,
"find_file_info_in_quorum: fileinfo content identity did not reach quorum"
);
return Err(DiskError::ErasureReadQuorum);
}