gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-22 12:26:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): handle stalled recovery reads and listings (#3790)

Browse Source 
* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
GatewayJ
2026-06-27 10:21:09 +08:00
committed by GitHub
parent 3fb4dcd52e
commit 675597ec16
28 changed files with 2991 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/cache_value/metacache_set.rs
+225 -86
View File
@@ -63,6 +63,7 @@ fn is_missing_path_error(err: &DiskError) -> bool {
#[derive(Clone)]
pub(crate) enum TestReaderBehavior {
Eof,
Entries(Vec<MetaCacheEntry>),
Stall,
IgnoreCancel,
ProducerError(DiskError),
@@ -146,6 +147,13 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
match behavior {
TestReaderBehavior::Eof => return Ok(()),
TestReaderBehavior::Entries(entries) => {
let mut wr = wr;
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
out.write(&entries).await.expect("test entries should be written");
out.close().await.expect("test entries should close");
return Ok(());
}
TestReaderBehavior::Stall => {
let _held_writer = wr;
cancel_rx_clone.cancelled().await;
@@ -360,6 +368,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
for _ in 0..readers.len() {
errs.push(None);
}
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
loop {
let mut current = MetaCacheEntry::default();
@@ -387,90 +396,94 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
continue;
}
let entry = match peek_with_timeout(r, peek_timeout).await {
PeekOutcome::Ready(res) => {
if let Some(entry) = res {
// info!("read entry disk: {}, name: {}", i, entry.name);
entry
} else {
let entry = if let Some(entry) = pending_entries[i].take() {
entry
} else {
match peek_with_timeout(r, peek_timeout).await {
PeekOutcome::Ready(res) => {
if let Some(entry) = res {
// info!("read entry disk: {}, name: {}", i, entry.name);
entry
} else {
if let Some(err) = producer_error(&producer_errs, i) {
has_err += 1;
errs[i] = Some(err);
continue;
}
// eof
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
}
}
PeekOutcome::Error(err) => {
if let Some(err) = producer_error(&producer_errs, i) {
has_err += 1;
errs[i] = Some(err);
continue;
}
// eof
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
if err == rustfs_filemeta::Error::Unexpected {
at_eof += 1;
// warn!("list_path_raw: peek err eof, disk: {}", i);
continue;
}
// warn!("list_path_raw: peek err00, err: {:?}", err);
if is_io_eof(&err) {
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
}
if err == rustfs_filemeta::Error::FileNotFound {
at_eof += 1;
fnf += 1;
// warn!("list_path_raw: peek fnf, disk: {}", i);
continue;
} else if err == rustfs_filemeta::Error::VolumeNotFound {
at_eof += 1;
fnf += 1;
vnf += 1;
// warn!("list_path_raw: peek vnf, disk: {}", i);
continue;
} else {
has_err += 1;
errs[i] = Some(err.into());
// warn!("list_path_raw: peek err, disk: {}", i);
continue;
}
}
}
PeekOutcome::Error(err) => {
if let Some(err) = producer_error(&producer_errs, i) {
PeekOutcome::TimedOut => {
has_err += 1;
errs[i] = Some(err);
errs[i] = Some(DiskError::Timeout);
let endpoint = opts
.disks
.get(i)
.and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
.unwrap_or_else(|| "missing".to_string());
counter!(
"rustfs_list_path_raw_stall_total",
"drive" => endpoint.clone()
)
.increment(1);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
drive = %endpoint,
bucket = %opts.bucket,
path = %opts.path,
timeout_ms = peek_timeout.as_millis(),
state = "peek_timed_out",
"Metacache reader peek timed out"
);
let (detached_rd, write_half) = tokio::io::duplex(1);
drop(write_half);
*r = MetacacheReader::new(detached_rd);
continue;
}
if err == rustfs_filemeta::Error::Unexpected {
at_eof += 1;
// warn!("list_path_raw: peek err eof, disk: {}", i);
continue;
}
// warn!("list_path_raw: peek err00, err: {:?}", err);
if is_io_eof(&err) {
at_eof += 1;
// warn!("list_path_raw: peek eof, disk: {}", i);
continue;
}
if err == rustfs_filemeta::Error::FileNotFound {
at_eof += 1;
fnf += 1;
// warn!("list_path_raw: peek fnf, disk: {}", i);
continue;
} else if err == rustfs_filemeta::Error::VolumeNotFound {
at_eof += 1;
fnf += 1;
vnf += 1;
// warn!("list_path_raw: peek vnf, disk: {}", i);
continue;
} else {
has_err += 1;
errs[i] = Some(err.into());
// warn!("list_path_raw: peek err, disk: {}", i);
continue;
}
}
PeekOutcome::TimedOut => {
has_err += 1;
errs[i] = Some(DiskError::Timeout);
let endpoint = opts
.disks
.get(i)
.and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
.unwrap_or_else(|| "missing".to_string());
counter!(
"rustfs_list_path_raw_stall_total",
"drive" => endpoint.clone()
)
.increment(1);
warn!(
event = EVENT_METACACHE_LISTING,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_METACACHE,
drive = %endpoint,
bucket = %opts.bucket,
path = %opts.path,
timeout_ms = peek_timeout.as_millis(),
state = "peek_timed_out",
"Metacache reader peek timed out"
);
let (detached_rd, write_half) = tokio::io::duplex(1);
drop(write_half);
*r = MetacacheReader::new(detached_rd);
continue;
}
};
@@ -498,11 +511,14 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
}
// We got different entries
if entry.name > current.name {
pending_entries[i] = Some(entry);
continue;
}
for item in top_entries.iter_mut().take(i) {
*item = None;
for (idx, item) in top_entries.iter_mut().enumerate().take(i) {
if let Some(entry) = item.take() {
pending_entries[idx] = Some(entry);
}
}
agree = 1;
@@ -564,13 +580,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
{
finished_fn(&errs).await;
}
// All remaining readers are either at EOF or failed. Earlier logic
// returned Timeout here for even a single stalled drive, despite
// `has_err` being within the tolerated drive-failure budget. That
// makes small distributed listings fail once healthy quorum readers
// reach EOF but one remote walk stream is slow/stalled. Only the
// `has_err > opts.disks.len() - opts.min_disks` branch above should
// turn tolerated reader failures into request failures.
// Tolerated reader failures, including timeouts, must not turn a
// quorum EOF into a failed listing. The quorum failure branch above
// is the single place that escalates too many reader errors.
// error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs);
break;
}
@@ -717,7 +729,10 @@ fn producer_error(producer_errs: &[OnceLock<DiskError>], idx: usize) -> Option<D
mod tests {
use super::*;
use rustfs_filemeta::MetacacheWriter;
use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams};
use std::sync::Mutex;
use time::OffsetDateTime;
use uuid::Uuid;
#[tokio::test]
async fn list_path_raw_empty_disks_returns_read_quorum() {
@@ -769,7 +784,49 @@ mod tests {
},
)
.await
.expect("listing should complete when healthy quorum reached EOF and only a tolerated drive stalled");
.expect("stalled reader within the tolerated failure budget should not fail quorum EOF");
}
#[tokio::test]
async fn list_path_raw_completes_after_partial_quorum_when_reader_stalls() {
let entry = MetaCacheEntry {
name: "bucket/object".to_string(),
metadata: vec![1, 2, 3],
cached: None,
reusable: false,
};
let seen = Arc::new(Mutex::new(Vec::new()));
let seen_clone = seen.clone();
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None, None],
min_disks: 2,
test_reader_behaviors: vec![
TestReaderBehavior::Entries(vec![entry.clone()]),
TestReaderBehavior::Entries(vec![entry]),
TestReaderBehavior::Stall,
],
peek_timeout: Some(Duration::from_millis(20)),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
let seen = seen_clone.clone();
Box::pin(async move {
let mut names = entries.0.iter().flatten().map(|entry| entry.name.clone());
if let Some(name) = names.next()
&& names.any(|next| next == name)
{
seen.lock().expect("seen mutex poisoned").push(name);
}
})
})),
..Default::default()
},
)
.await
.expect("stalled reader within failure budget should not fail a quorum listing");
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
}
#[tokio::test]
@@ -794,7 +851,7 @@ mod tests {
.await;
let listing = result.expect("list_path_raw should abort unresponsive producer instead of hanging");
assert!(listing.is_ok());
listing.expect("unresponsive producer within failure budget should not fail quorum EOF");
}
#[tokio::test]
@@ -829,6 +886,88 @@ mod tests {
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
}
#[tokio::test]
async fn list_path_raw_continues_after_single_disk_delete_marker_gap() {
fn object_entry(name: &str, version_id: &str) -> MetaCacheEntry {
let mut meta = FileMeta::default();
let mut fi = FileInfo::new(name, 1, 1);
fi.version_id = Some(Uuid::parse_str(version_id).expect("test version id should parse"));
fi.mod_time = Some(OffsetDateTime::now_utc());
meta.add_version(fi).expect("object metadata should be valid");
MetaCacheEntry {
name: name.to_string(),
metadata: meta.marshal_msg().expect("object metadata should encode"),
cached: Some(meta),
reusable: false,
}
}
fn delete_marker_entry(name: &str, version_id: &str) -> MetaCacheEntry {
let mut meta = FileMeta::default();
meta.add_version(FileInfo {
deleted: true,
version_id: Some(Uuid::parse_str(version_id).expect("test version id should parse")),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("delete marker metadata should be valid");
MetaCacheEntry {
name: name.to_string(),
metadata: meta.marshal_msg().expect("delete marker metadata should encode"),
cached: Some(meta),
reusable: false,
}
}
let visible_before = object_entry("object-000003", "11111111-1111-1111-1111-111111111111");
let hidden_delete = delete_marker_entry("object-000004", "22222222-2222-2222-2222-222222222222");
let visible_after = object_entry("object-000005", "33333333-3333-3333-3333-333333333333");
let seen = Arc::new(Mutex::new(Vec::new()));
let agreed_seen = seen.clone();
let partial_seen = seen.clone();
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None, None, None],
min_disks: 2,
test_reader_behaviors: vec![
TestReaderBehavior::Entries(vec![visible_before.clone(), hidden_delete, visible_after.clone()]),
TestReaderBehavior::Entries(vec![visible_before.clone(), visible_after.clone()]),
TestReaderBehavior::Entries(vec![visible_before.clone(), visible_after.clone()]),
TestReaderBehavior::Entries(vec![visible_before, visible_after]),
],
agreed: Some(Box::new(move |entry| {
let seen = agreed_seen.clone();
Box::pin(async move {
seen.lock().expect("seen mutex poisoned").push(entry.name);
})
})),
partial: Some(Box::new(move |entries, _| {
let seen = partial_seen.clone();
Box::pin(async move {
if let Some(entry) = entries.resolve(MetadataResolutionParams {
obj_quorum: 2,
requested_versions: 1,
bucket: "bucket".to_string(),
..Default::default()
}) {
seen.lock().expect("seen mutex poisoned").push(entry.name);
}
})
})),
..Default::default()
},
)
.await
.expect("single-disk delete marker gap should not end listing");
assert_eq!(
seen.lock().expect("seen mutex poisoned").as_slice(),
&["object-000003".to_string(), "object-000005".to_string()]
);
}
#[tokio::test]
async fn peek_with_timeout_times_out_on_silent_reader() {
let (_writer, reader) = tokio::io::duplex(64);