gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 17:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): handle stalled recovery reads and listings (#3790)

Browse Source 
* fix(ecstore): handle stalled recovery reads and listings

* fix(rio): start HTTP stall timeout on read

* fix(ecstore): handle stalled reads and partial lists

* fix(ecstore): retire stalled shards and list errors

* fix(ecstore): preserve list merge lookahead entries

* fix(ecstore): bound zero-copy shard reads

* fix(ecstore): hedge stalled shard reads

* fix(ecstore): retire abandoned shard reads

* fix(ecstore): include part identity in metadata quorum

* fix(ecstore): validate heal shard sources

* fix(ecstore): verify reconstructed read shards

* chore(ecstore): log slow object read stages

* fix(heal): throttle auto heal during recovery

* fix(scanner): yield to foreground reads

* fix(scanner): track streaming object reads

* fix(ecstore): avoid false read heal fanout

* fix(ecstore): verify codec streaming reconstruction sources

* fix(ecstore): preserve quorum progress on slow shards

* fix(storage): restore read timeout facade

* fix(ecstore): retain fallback readers after quorum

* chore: allow decode helper argument lists

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
GatewayJ
2026-06-27 10:21:09 +08:00
committed by GitHub
parent 3fb4dcd52e
commit 675597ec16
28 changed files with 2991 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/cluster/rpc/internode_data_transport.rs
+5 -1
View File
@@ -76,6 +76,7 @@ pub struct ReadStreamRequest {
pub path: String,
pub offset: usize,
pub length: usize,
pub stall_timeout: Option<Duration>,
}
#[derive(Debug, Clone)]
@@ -122,7 +123,9 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
let url = build_read_file_stream_url(&request);
let mut headers = json_headers();
build_auth_headers(&url, &Method::GET, &mut headers)?;
Ok(Box::new(HttpReader::new(url, Method::GET, headers, None).await?))
Ok(Box::new(
HttpReader::new_with_stall_timeout(url, Method::GET, headers, None, request.stall_timeout).await?,
))
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
@@ -260,6 +263,7 @@ mod tests {
path: "pool.bin/../part.1".to_string(),
offset: 7,
length: 11,
stall_timeout: None,
});
assert_eq!(
crates/ecstore/src/cluster/rpc/remote_disk.rs
+41 -16
View File
@@ -26,6 +26,7 @@ use crate::disk::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
get_drive_metadata_timeout, get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
get_object_disk_read_timeout,
},
endpoint::Endpoint,
health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
@@ -1693,6 +1694,7 @@ impl DiskAPI for RemoteDisk {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
let stall_timeout = get_object_disk_read_timeout();
self.data_transport
.open_read(ReadStreamRequest {
endpoint: self.endpoint.grid_host(),
@@ -1701,6 +1703,7 @@ impl DiskAPI for RemoteDisk {
path: path.to_string(),
offset,
length,
stall_timeout: (!stall_timeout.is_zero()).then_some(stall_timeout),
})
.await
}
@@ -2927,25 +2930,47 @@ mod tests {
#[tokio::test]
async fn test_remote_disk_read_file_stream_uses_configured_data_transport() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let expected_disk = remote_disk.disk_ref().await;
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, None::<&str>)], async {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let expected_disk = remote_disk.disk_ref().await;
let _reader = remote_disk.read_file_stream("bucket", "object/part.1", 7, 11).await.unwrap();
let _reader = remote_disk.read_file_stream("bucket", "object/part.1", 7, 11).await.unwrap();
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::Read(request) => {
assert_eq!(request.endpoint, "http://remote-node:9000");
assert_eq!(request.disk, expected_disk);
assert_eq!(request.volume, "bucket");
assert_eq!(request.path, "object/part.1");
assert_eq!(request.offset, 7);
assert_eq!(request.length, 11);
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::Read(request) => {
assert_eq!(request.endpoint, "http://remote-node:9000");
assert_eq!(request.disk, expected_disk);
assert_eq!(request.volume, "bucket");
assert_eq!(request.path, "object/part.1");
assert_eq!(request.offset, 7);
assert_eq!(request.length, 11);
assert_eq!(request.stall_timeout, Some(get_object_disk_read_timeout()));
}
other => panic!("expected read transport call, got {other:?}"),
}
other => panic!("expected read transport call, got {other:?}"),
}
})
.await;
}
#[tokio::test]
async fn test_remote_disk_read_file_stream_disables_stall_timeout_when_configured_zero() {
temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some("0"))], async {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let _reader = remote_disk.read_file_stream("bucket", "object/part.1", 7, 11).await.unwrap();
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::Read(request) => assert_eq!(request.stall_timeout, None),
other => panic!("expected read transport call, got {other:?}"),
}
})
.await;
}
#[tokio::test]