gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 08:06:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/issue #1001 dead node detection (#1054)

Browse Source 
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: Jitterx69 <mohit@example.com>
This commit is contained in:
Jitter
2025-12-08 09:59:46 +05:30
committed by GitHub
parent 834025d9e3
commit 76d25d9a20
9 changed files with 369 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/notification_sys.rs
+35 -13
View File
@@ -190,16 +190,32 @@ impl NotificationSys {
pub async fn storage_info<S: StorageAPI>(&self, api: &S) -> rustfs_madmin::StorageInfo {
let mut futures = Vec::with_capacity(self.peer_clients.len());
let endpoints = get_global_endpoints();
let peer_timeout = Duration::from_secs(2); // Same timeout as server_info
for client in self.peer_clients.iter() {
let endpoints = endpoints.clone();
futures.push(async move {
if let Some(client) = client {
match client.local_storage_info().await {
Ok(info) => Some(info),
Err(_) => Some(rustfs_madmin::StorageInfo {
disks: get_offline_disks(&client.host.to_string(), &get_global_endpoints()),
..Default::default()
}),
let host = client.host.to_string();
// Wrap in timeout to ensure we don't hang on dead peers
match timeout(peer_timeout, client.local_storage_info()).await {
Ok(Ok(info)) => Some(info),
Ok(Err(err)) => {
warn!("peer {} storage_info failed: {}", host, err);
Some(rustfs_madmin::StorageInfo {
disks: get_offline_disks(&host, &endpoints),
..Default::default()
})
}
Err(_) => {
warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
client.evict_connection().await;
Some(rustfs_madmin::StorageInfo {
disks: get_offline_disks(&host, &endpoints),
..Default::default()
})
}
}
} else {
None
@@ -230,13 +246,19 @@ impl NotificationSys {
futures.push(async move {
if let Some(client) = client {
let host = client.host.to_string();
call_peer_with_timeout(
peer_timeout,
&host,
|| client.server_info(),
|| offline_server_properties(&host, &endpoints),
)
.await
match timeout(peer_timeout, client.server_info()).await {
Ok(Ok(info)) => info,
Ok(Err(err)) => {
warn!("peer {} server_info failed: {}", host, err);
// client.server_info handles eviction internally on error, but fallback needed
offline_server_properties(&host, &endpoints)
}
Err(_) => {
warn!("peer {} server_info timed out after {:?}", host, peer_timeout);
client.evict_connection().await;
offline_server_properties(&host, &endpoints)
}
}
} else {
ServerProperties::default()
}