gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): handle stalled recovery reads and listings (#3790)

Browse Source 
* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
GatewayJ
2026-06-27 10:21:09 +08:00
committed by GitHub
parent 3fb4dcd52e
commit 675597ec16
28 changed files with 2991 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/disk_store.rs
+49
View File
@@ -197,6 +197,14 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
)
}
pub fn get_object_disk_read_timeout() -> Duration {
get_drive_timeout_duration(
rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT,
rustfs_config::DEFAULT_OBJECT_DISK_READ_TIMEOUT,
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
)
}
pub fn get_drive_active_check_interval() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64(
rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
@@ -1518,6 +1526,47 @@ mod tests {
});
}
#[test]
fn object_disk_read_timeout_uses_default_when_unset() {
temp_env::with_var_unset(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, || {
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || {
assert_eq!(
get_object_disk_read_timeout(),
Duration::from_secs(rustfs_config::DEFAULT_OBJECT_DISK_READ_TIMEOUT)
);
});
});
});
}
#[test]
fn object_disk_read_timeout_uses_high_latency_profile_when_unset() {
temp_env::with_var_unset(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, || {
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
temp_env::with_var(
rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY),
|| {
assert_eq!(
get_object_disk_read_timeout(),
Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS)
);
},
);
});
});
}
#[test]
fn object_disk_read_timeout_prefers_canonical_over_legacy() {
temp_env::with_var(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some("7"), || {
temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
assert_eq!(get_object_disk_read_timeout(), Duration::from_secs(7));
});
});
}
#[test]
fn drive_active_check_interval_uses_default_when_unset() {
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, || {
crates/ecstore/src/disk/local.rs
+150 -7
View File
@@ -14,6 +14,7 @@
use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
use crate::data_usage::local_snapshot::ensure_data_usage_layout;
use crate::disk::disk_store::get_object_disk_read_timeout;
use crate::disk::{
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
@@ -56,9 +57,9 @@ use std::{
};
use time::OffsetDateTime;
use tokio::fs::{self, File};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind, ReadBuf};
use tokio::sync::{Notify, RwLock};
use tokio::time::{Instant, interval_at, timeout};
use tokio::time::{Instant, Sleep, interval_at, timeout};
use tracing::{debug, error, info, warn};
use uuid::Uuid;
@@ -145,6 +146,63 @@ struct FileCacheReclaimReader {
reclaimed: bool,
}
struct StallTimeoutReader<R> {
inner: R,
timeout: Duration,
timer: Option<std::pin::Pin<Box<Sleep>>>,
}
impl<R> StallTimeoutReader<R> {
fn new(inner: R, timeout: Duration) -> Self {
Self {
inner,
timeout,
timer: None,
}
}
}
impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
let filled_before = buf.filled().len();
match std::pin::Pin::new(&mut self.inner).poll_read(cx, buf) {
std::task::Poll::Ready(result) => {
self.timer = None;
std::task::Poll::Ready(result)
}
std::task::Poll::Pending => {
if self.timeout.is_zero() {
return std::task::Poll::Pending;
}
if self.timer.is_none() {
self.timer = Some(Box::pin(tokio::time::sleep(self.timeout)));
}
if let Some(timer) = self.timer.as_mut()
&& std::future::Future::poll(timer.as_mut(), cx).is_ready()
{
self.timer = None;
return std::task::Poll::Ready(Err(std::io::Error::new(
ErrorKind::TimedOut,
"local disk read stall timeout",
)));
}
if buf.filled().len() > filled_before {
self.timer = None;
}
std::task::Poll::Pending
}
}
}
}
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
counter!("rustfs_page_cache_reclaim_requests_total", "kind" => kind.to_string(), "result" => "ok".to_string()).increment(1);
counter!("rustfs_page_cache_reclaim_bytes_total", "kind" => kind.to_string()).increment(reclaim_len as u64);
@@ -247,11 +305,11 @@ impl Drop for FileCacheReclaimReader {
}
}
impl tokio::io::AsyncRead for FileCacheReclaimReader {
impl AsyncRead for FileCacheReclaimReader {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
buf: &mut ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
}
@@ -1767,6 +1825,7 @@ impl LocalDisk {
error = ?er,
"Disk local scan failed"
);
return Err(er);
}
}
@@ -1836,9 +1895,20 @@ impl LocalDisk {
meta.name.push_str(SLASH_SEPARATOR);
schedule_dir(&mut dir_stack, meta.name, false, None);
}
continue;
}
continue;
error!(
event = EVENT_DISK_LOCAL_SCAN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
path = %fname,
operation = "read_metadata",
error = ?err,
"Disk local scan failed"
);
return Err(err);
}
};
}
@@ -1859,7 +1929,7 @@ impl LocalDisk {
&& let Err(er) =
Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned, skip_object, dir_to_skip)).await
{
warn!(
error!(
event = EVENT_DISK_LOCAL_SCAN_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
@@ -1868,6 +1938,7 @@ impl LocalDisk {
error = ?er,
"Disk local recursive scan failed"
);
return Err(er);
}
}
@@ -2652,7 +2723,8 @@ impl DiskAPI for LocalDisk {
}
let reclaim_on_drop = should_reclaim_file_cache_after_read(length);
Ok(Box::new(FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop)))
let reader = FileCacheReclaimReader::new(f, offset as u64, length, reclaim_on_drop);
Ok(Box::new(StallTimeoutReader::new(reader, get_object_disk_read_timeout())))
}
/// Zero-copy file read using memory mapping (Unix) or efficient read (non-Unix).
@@ -3737,6 +3809,10 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
#[cfg(test)]
mod test {
use super::*;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncReadExt, ReadBuf};
#[tokio::test]
async fn test_skip_access_checks() {
@@ -3756,6 +3832,28 @@ mod test {
}
}
#[derive(Debug, Default)]
struct PendingTestReader;
impl AsyncRead for PendingTestReader {
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
Poll::Pending
}
}
#[tokio::test(start_paused = true)]
async fn local_read_timeout_reader_times_out_when_inner_stalls() {
let mut reader = StallTimeoutReader::new(PendingTestReader, Duration::from_secs(10));
let mut buf = [0; 1];
let err = reader
.read(&mut buf)
.await
.expect_err("stalled local reader should return a timeout error");
assert_eq!(err.kind(), ErrorKind::TimedOut);
}
#[tokio::test]
async fn test_get_disk_id_invalidates_cache_after_format_removal() {
use crate::disk::FORMAT_CONFIG_FILE;
@@ -4240,6 +4338,51 @@ mod test {
assert_eq!(objs_returned, 1);
}
#[cfg(unix)]
#[tokio::test]
async fn test_scan_dir_propagates_metadata_read_errors() {
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
use tempfile::tempdir;
let dir = tempdir().unwrap();
let bucket = "test-bucket";
let bucket_dir = dir.path().join(bucket);
let object_dir = bucket_dir.join("broken");
let meta_path = object_dir.join(STORAGE_FORMAT_FILE);
fs::create_dir_all(&object_dir).await.unwrap();
fs::write(&meta_path, b"meta").await.unwrap();
let original_permissions = fs::metadata(&meta_path).await.unwrap().permissions();
fs::set_permissions(&meta_path, Permissions::from_mode(0o000)).await.unwrap();
if fs::File::open(&meta_path).await.is_ok() {
fs::set_permissions(&meta_path, original_permissions).await.unwrap();
return;
}
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let (_reader, mut writer) = tokio::io::duplex(4096);
let mut out = MetacacheWriter::new(&mut writer);
let opts = WalkDirOptions {
bucket: bucket.to_string(),
base_dir: "".to_string(),
recursive: true,
..Default::default()
};
let mut objs_returned = 0;
let result = disk
.scan_dir("".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false, None)
.await;
fs::set_permissions(&meta_path, original_permissions).await.unwrap();
assert!(matches!(result, Err(DiskError::FileAccessDenied)));
}
#[tokio::test]
async fn test_walk_dir_ignore_multipart_dirs() {
use rustfs_filemeta::MetacacheReader;