fix(ecstore): avoid offline disks on admin timeout (#3263)

* fix(ecstore): avoid offline disks on admin timeout

* fix(ecstore): handle poisoned admin cache locks

* fix(ecstore): recover poisoned admin cache on success

---------

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>

crates/e2e_test/src/admin_timeout_regression_test.rs

@@ -0,0 +1,208 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestClusterEnvironment, init_logging, local_http_client};
+use aws_sdk_s3::primitives::ByteStream;
+use http::header::HOST;
+use reqwest::StatusCode;
+use rustfs_madmin::{ITEM_OFFLINE, InfoMessage, StorageInfo};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+use std::process::Command;
+use tokio::time::{Duration, sleep, timeout};
+use uuid::Uuid;
+
+const BUCKET: &str = "issue-2525-admin-timeout";
+
+async fn signed_admin_get(
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let request = http::Request::builder()
+        .method(http::Method::GET)
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+    let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
+
+    let mut builder = local_http_client().get(url);
+    for (name, value) in signed.headers() {
+        builder = builder.header(name, value);
+    }
+
+    Ok(builder.send().await?)
+}
+
+async fn fetch_info(cluster: &RustFSTestClusterEnvironment) -> Result<InfoMessage, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/info", cluster.nodes[0].url);
+    let response = signed_admin_get(&url, &cluster.access_key, &cluster.secret_key).await?;
+    parse_json_response(response, "cluster info").await
+}
+
+async fn fetch_storage_info(cluster: &RustFSTestClusterEnvironment) -> Result<StorageInfo, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/storageinfo", cluster.nodes[0].url);
+    let response = signed_admin_get(&url, &cluster.access_key, &cluster.secret_key).await?;
+    parse_json_response(response, "storage info").await
+}
+
+async fn parse_json_response<T: serde::de::DeserializeOwned>(
+    response: reqwest::Response,
+    label: &str,
+) -> Result<T, Box<dyn Error + Send + Sync>> {
+    let status = response.status();
+    let body = response.bytes().await?;
+    if status != StatusCode::OK {
+        return Err(format!("{label} request failed: {status} {}", String::from_utf8_lossy(body.as_ref())).into());
+    }
+
+    Ok(serde_json::from_slice(&body)?)
+}
+
+fn signal_process(pid: u32, signal: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let output = Command::new("kill").arg(format!("-{signal}")).arg(pid.to_string()).output()?;
+    if output.status.success() {
+        return Ok(());
+    }
+
+    Err(format!("kill -{signal} {pid} failed: {}", String::from_utf8_lossy(&output.stderr)).into())
+}
+
+fn offline_server_count(info: &InfoMessage) -> usize {
+    info.servers
+        .as_ref()
+        .map(|servers| servers.iter().filter(|server| server.state == ITEM_OFFLINE).count())
+        .unwrap_or_default()
+}
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let warm_info = fetch_info(&cluster).await?;
+    assert_eq!(
+        warm_info
+            .backend
+            .as_ref()
+            .map(|backend| backend.offline_disks)
+            .unwrap_or_default(),
+        0,
+        "cluster should start with zero offline disks"
+    );
+
+    let warm_storage = fetch_storage_info(&cluster).await?;
+    assert_eq!(
+        warm_storage.backend.offline_disks.sum(),
+        0,
+        "storage info should start with zero offline disks"
+    );
+
+    let suspended_pid = cluster.nodes[1]
+        .process
+        .as_ref()
+        .ok_or("cluster node 1 process missing")?
+        .id();
+    signal_process(suspended_pid, "STOP")?;
+
+    let resume = tokio::spawn(async move {
+        sleep(Duration::from_secs(6)).await;
+        signal_process(suspended_pid, "CONT")
+    });
+
+    let client = cluster.create_s3_client(0)?;
+    let key = format!("issue-2525/{}", Uuid::new_v4().simple());
+    let payload = b"admin timeout regression payload".to_vec();
+
+    let during = timeout(Duration::from_secs(20), async {
+        tokio::join!(fetch_info(&cluster), fetch_storage_info(&cluster), async {
+            client
+                .put_object()
+                .bucket(BUCKET)
+                .key(&key)
+                .body(ByteStream::from(payload.clone()))
+                .send()
+                .await?;
+            Ok::<(), Box<dyn Error + Send + Sync>>(())
+        })
+    })
+    .await;
+
+    resume.await??;
+
+    let (info_during, storage_during, put_result) =
+        during.map_err(|_| "timed out while waiting for admin queries during suspended peer")?;
+    let info_during = info_during?;
+    let storage_during = storage_during?;
+    put_result?;
+
+    assert_eq!(
+        info_during
+            .backend
+            .as_ref()
+            .map(|backend| backend.offline_disks)
+            .unwrap_or_default(),
+        0,
+        "single admin timeout must not synthesize offline disks in /info"
+    );
+    assert_eq!(
+        offline_server_count(&info_during),
+        0,
+        "single admin timeout must not mark any server offline in /info"
+    );
+    assert_eq!(
+        storage_during.backend.offline_disks.sum(),
+        0,
+        "single admin timeout must not synthesize offline disks in /storageinfo"
+    );
+    assert!(
+        storage_during.disks.iter().all(|disk| disk.state != ITEM_OFFLINE),
+        "single admin timeout must not synthesize offline disk entries in /storageinfo"
+    );
+
+    let recovered_info = fetch_info(&cluster).await?;
+    assert_eq!(
+        recovered_info
+            .backend
+            .as_ref()
+            .map(|backend| backend.offline_disks)
+            .unwrap_or_default(),
+        0,
+        "offline disk count should stay at zero after the peer resumes"
+    );
+
+    let body = client
+        .get_object()
+        .bucket(BUCKET)
+        .key(&key)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    assert_eq!(body.as_ref(), b"admin timeout regression payload");
+
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -149,4 +149,7 @@ mod snowball_auto_extract_test;
 #[cfg(test)]
 mod namespace_lock_quorum_test;
 
+#[cfg(test)]
+mod admin_timeout_regression_test;
+
 pub mod tls_gen;

crates/ecstore/src/notification_sys.rs

@@ -25,15 +25,37 @@ use lazy_static::lazy_static;
 use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices};
 use rustfs_madmin::metrics::RealtimeMetrics;
 use rustfs_madmin::net::NetInfo;
-use rustfs_madmin::{ItemState, ServerProperties};
+use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
 use std::collections::hash_map::DefaultHasher;
 use std::future::Future;
 use std::hash::{Hash, Hasher};
-use std::sync::OnceLock;
+use std::sync::{Mutex, OnceLock};
 use std::time::{Duration, SystemTime};
 use tokio::time::timeout;
 use tracing::{error, warn};
 
+/// After this many consecutive admin-call failures, mark the peer as offline.
+const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
+
+/// Cached result from the last successful admin call to a peer.
+struct PeerAdminCache {
+    last_storage_info: Option<StorageInfo>,
+    last_server_info: Option<ServerProperties>,
+    storage_failures: u32,
+    server_failures: u32,
+}
+
+impl PeerAdminCache {
+    fn new() -> Self {
+        Self {
+            last_storage_info: None,
+            last_server_info: None,
+            storage_failures: 0,
+            server_failures: 0,
+        }
+    }
+}
+
 lazy_static! {
     pub static ref GLOBAL_NotificationSys: OnceLock<NotificationSys> = OnceLock::new();
 }
@@ -53,14 +75,17 @@ pub struct NotificationSys {
     pub peer_clients: Vec<Option<PeerRestClient>>,
     #[allow(dead_code)]
     pub all_peer_clients: Vec<Option<PeerRestClient>>,
+    peer_admin_caches: Vec<Mutex<PeerAdminCache>>,
 }
 
 impl NotificationSys {
     pub async fn new(eps: EndpointServerPools) -> Self {
         let (peer_clients, all_peer_clients) = PeerRestClient::new_clients(eps).await;
+        let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
         Self {
             peer_clients,
             all_peer_clients,
+            peer_admin_caches,
         }
     }
 }
@@ -251,30 +276,27 @@ impl NotificationSys {
     pub async fn storage_info<S: StorageAPI>(&self, api: &S) -> rustfs_madmin::StorageInfo {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         let endpoints = get_global_endpoints();
-        let peer_timeout = Duration::from_secs(2); // Same timeout as server_info
+        let peer_timeout = Duration::from_secs(5);
 
-        for client in self.peer_clients.iter() {
+        for (idx, client) in self.peer_clients.iter().enumerate() {
             let endpoints = endpoints.clone();
+            let cache = self.peer_admin_caches.get(idx);
             futures.push(async move {
                 if let Some(client) = client {
                     let host = client.host.to_string();
-                    // Wrap in timeout to ensure we don't hang on dead peers
                     match timeout(peer_timeout, client.local_storage_info()).await {
-                        Ok(Ok(info)) => Some(info),
+                        Ok(Ok(info)) => {
+                            update_storage_info_cache(cache, &host, &info);
+                            Some(info)
+                        }
                         Ok(Err(err)) => {
                             warn!("peer {} storage_info failed: {}", host, err);
-                            Some(rustfs_madmin::StorageInfo {
+                            handle_peer_failure(cache, &host, &endpoints)
-                                disks: get_offline_disks(&host, &endpoints),
-                                ..Default::default()
-                            })
                         }
                         Err(_) => {
                             warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
                             client.evict_connection().await;
-                            Some(rustfs_madmin::StorageInfo {
+                            handle_peer_failure(cache, &host, &endpoints)
-                                disks: get_offline_disks(&host, &endpoints),
-                                ..Default::default()
-                            })
                         }
                     }
                 } else {
@@ -299,24 +321,27 @@ impl NotificationSys {
     pub async fn server_info(&self) -> Vec<ServerProperties> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         let endpoints = get_global_endpoints();
-        let peer_timeout = Duration::from_secs(2);
+        let peer_timeout = Duration::from_secs(5);
 
-        for client in self.peer_clients.iter() {
+        for (idx, client) in self.peer_clients.iter().enumerate() {
             let endpoints = endpoints.clone();
+            let cache = self.peer_admin_caches.get(idx);
             futures.push(async move {
                 if let Some(client) = client {
                     let host = client.host.to_string();
                     match timeout(peer_timeout, client.server_info()).await {
-                        Ok(Ok(info)) => info,
+                        Ok(Ok(info)) => {
+                            update_server_info_cache(cache, &host, &info);
+                            info
+                        }
                         Ok(Err(err)) => {
                             warn!("peer {} server_info failed: {}", host, err);
-                            // client.server_info handles eviction internally on error, but fallback needed
+                            handle_server_info_failure(cache, &host, &endpoints)
-                            offline_server_properties(&host, &endpoints)
                         }
                         Err(_) => {
                             warn!("peer {} server_info timed out after {:?}", host, peer_timeout);
                             client.evict_connection().await;
-                            offline_server_properties(&host, &endpoints)
+                            handle_server_info_failure(cache, &host, &endpoints)
                         }
                     }
                 } else {
@@ -841,6 +866,113 @@ where
     }
 }
 
+/// Handle a peer failure for storage_info: return cached data if available,
+/// or mark offline only after consecutive failures exceed the threshold.
+fn handle_peer_failure(
+    cache: Option<&Mutex<PeerAdminCache>>,
+    host: &str,
+    endpoints: &EndpointServerPools,
+) -> Option<StorageInfo> {
+    let cache = cache?;
+
+    let mut c = match cache.lock() {
+        Ok(cache) => cache,
+        Err(poisoned) => {
+            warn!("peer {host} storage_info cache mutex poisoned");
+            poisoned.into_inner()
+        }
+    };
+    c.storage_failures += 1;
+
+    if let Some(ref cached) = c.last_storage_info
+        && c.storage_failures < CONSECUTIVE_FAILURE_THRESHOLD
+    {
+        return Some(cached.clone());
+    }
+
+    if c.storage_failures >= CONSECUTIVE_FAILURE_THRESHOLD {
+        return Some(StorageInfo {
+            disks: get_offline_disks(host, endpoints),
+            ..Default::default()
+        });
+    }
+
+    None
+}
+
+fn update_storage_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &StorageInfo) {
+    let Some(cache) = cache else {
+        return;
+    };
+
+    let mut c = match cache.lock() {
+        Ok(cache) => cache,
+        Err(poisoned) => {
+            warn!("peer {host} storage_info cache mutex poisoned");
+            poisoned.into_inner()
+        }
+    };
+    c.last_storage_info = Some(info.clone());
+    c.storage_failures = 0;
+}
+
+/// Handle a peer failure for server_info: return cached data if available,
+/// or mark offline only after consecutive failures exceed the threshold.
+fn handle_server_info_failure(
+    cache: Option<&Mutex<PeerAdminCache>>,
+    host: &str,
+    endpoints: &EndpointServerPools,
+) -> ServerProperties {
+    let Some(cache) = cache else {
+        return initializing_server_properties(host);
+    };
+
+    let mut c = match cache.lock() {
+        Ok(cache) => cache,
+        Err(poisoned) => {
+            warn!("peer {host} server_info cache mutex poisoned");
+            poisoned.into_inner()
+        }
+    };
+    c.server_failures += 1;
+
+    if let Some(ref cached) = c.last_server_info
+        && c.server_failures < CONSECUTIVE_FAILURE_THRESHOLD
+    {
+        return cached.clone();
+    }
+
+    if c.server_failures >= CONSECUTIVE_FAILURE_THRESHOLD {
+        return offline_server_properties(host, endpoints);
+    }
+
+    initializing_server_properties(host)
+}
+
+fn update_server_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &ServerProperties) {
+    let Some(cache) = cache else {
+        return;
+    };
+
+    let mut c = match cache.lock() {
+        Ok(cache) => cache,
+        Err(poisoned) => {
+            warn!("peer {host} server_info cache mutex poisoned");
+            poisoned.into_inner()
+        }
+    };
+    c.last_server_info = Some(info.clone());
+    c.server_failures = 0;
+}
+
+fn initializing_server_properties(host: &str) -> ServerProperties {
+    ServerProperties {
+        endpoint: host.to_string(),
+        state: ItemState::Initializing.to_string().to_owned(),
+        ..Default::default()
+    }
+}
+
 fn offline_server_properties(host: &str, endpoints: &EndpointServerPools) -> ServerProperties {
     ServerProperties {
         uptime: GLOBAL_BOOT_TIME
@@ -964,6 +1096,7 @@ mod tests {
         let sys = NotificationSys {
             peer_clients: vec![None],
             all_peer_clients: Vec::new(),
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
         };
 
         let err = sys
@@ -982,6 +1115,7 @@ mod tests {
         let sys = NotificationSys {
             peer_clients: vec![None],
             all_peer_clients: Vec::new(),
+            peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
         };
 
         let results = sys.load_transition_tier_config().await;
@@ -990,4 +1124,243 @@ mod tests {
         assert!(results[0].err.is_some());
         assert!(results[0].err.as_ref().unwrap().to_string().contains("peer is not reachable"));
     }
+
+    // --- Tests for handle_peer_failure / handle_server_info_failure caching ---
+
+    #[test]
+    fn handle_peer_failure_first_failure_returns_none_when_no_cache() {
+        let cache = Mutex::new(PeerAdminCache::new());
+        let endpoints = EndpointServerPools::default();
+
+        let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
+        assert!(result.is_none());
+        assert_eq!(cache.lock().unwrap().storage_failures, 1);
+    }
+
+    #[test]
+    fn handle_peer_failure_returns_cached_data_on_single_failure() {
+        let cached_info = StorageInfo {
+            disks: vec![rustfs_madmin::Disk {
+                endpoint: "disk-0".to_string(),
+                state: "ok".to_string(),
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        let cache = Mutex::new(PeerAdminCache {
+            last_storage_info: Some(cached_info),
+            last_server_info: None,
+            storage_failures: 0,
+            server_failures: 0,
+        });
+        let endpoints = EndpointServerPools::default();
+
+        // First failure: should return cached data
+        let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
+        let info = result.unwrap();
+        assert_eq!(info.disks.len(), 1);
+        assert_eq!(info.disks[0].state, "ok");
+        assert_eq!(cache.lock().unwrap().storage_failures, 1);
+    }
+
+    #[test]
+    fn handle_peer_failure_returns_offline_after_threshold_exceeded() {
+        let cached_info = StorageInfo {
+            disks: vec![rustfs_madmin::Disk {
+                endpoint: "disk-0".to_string(),
+                state: "ok".to_string(),
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        let cache = Mutex::new(PeerAdminCache {
+            last_storage_info: Some(cached_info),
+            last_server_info: None,
+            storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
+            server_failures: 0,
+        });
+        let endpoints = EndpointServerPools::default();
+
+        // This failure pushes us to the threshold => offline
+        let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
+        assert!(result.is_some());
+        assert_eq!(cache.lock().unwrap().storage_failures, CONSECUTIVE_FAILURE_THRESHOLD);
+    }
+
+    #[test]
+    fn handle_server_info_failure_returns_cached_on_single_failure() {
+        let cached_props = ServerProperties {
+            endpoint: "peer-1".to_string(),
+            state: "online".to_string(),
+            ..Default::default()
+        };
+
+        let cache = Mutex::new(PeerAdminCache {
+            last_storage_info: None,
+            last_server_info: Some(cached_props),
+            storage_failures: 0,
+            server_failures: 0,
+        });
+        let endpoints = EndpointServerPools::default();
+
+        let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
+        assert_eq!(result.endpoint, "peer-1");
+        assert_eq!(result.state, "online");
+        assert_eq!(cache.lock().unwrap().server_failures, 1);
+    }
+
+    #[test]
+    fn handle_server_info_failure_returns_initializing_before_threshold_without_cache() {
+        let cache = Mutex::new(PeerAdminCache::new());
+        let endpoints = EndpointServerPools::default();
+
+        let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
+        assert_eq!(result.endpoint, "peer-1");
+        assert_eq!(result.state, ItemState::Initializing.to_string());
+        assert!(result.disks.is_empty());
+        assert_eq!(cache.lock().unwrap().server_failures, 1);
+    }
+
+    #[test]
+    fn handle_server_info_failure_returns_offline_after_threshold() {
+        let cached_props = ServerProperties {
+            endpoint: "peer-1".to_string(),
+            state: "online".to_string(),
+            ..Default::default()
+        };
+
+        let cache = Mutex::new(PeerAdminCache {
+            last_storage_info: None,
+            last_server_info: Some(cached_props),
+            storage_failures: 0,
+            server_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
+        });
+        let endpoints = EndpointServerPools::default();
+
+        let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
+        assert_eq!(result.state, ItemState::Offline.to_string());
+        assert_eq!(cache.lock().unwrap().server_failures, CONSECUTIVE_FAILURE_THRESHOLD);
+    }
+
+    #[test]
+    fn success_resets_failure_counters_independently() {
+        let cache = Mutex::new(PeerAdminCache {
+            last_storage_info: None,
+            last_server_info: None,
+            storage_failures: 2,
+            server_failures: 2,
+        });
+
+        {
+            let mut c = cache.lock().unwrap();
+            c.last_storage_info = Some(StorageInfo::default());
+            c.storage_failures = 0;
+        }
+
+        let cache = cache.lock().unwrap();
+        assert_eq!(cache.storage_failures, 0);
+        assert_eq!(cache.server_failures, 2);
+    }
+
+    #[test]
+    fn storage_failures_do_not_affect_server_failures() {
+        let cache = Mutex::new(PeerAdminCache {
+            last_storage_info: Some(StorageInfo::default()),
+            last_server_info: Some(ServerProperties {
+                endpoint: "peer-1".to_string(),
+                state: "online".to_string(),
+                ..Default::default()
+            }),
+            storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
+            server_failures: 0,
+        });
+        let endpoints = EndpointServerPools::default();
+
+        let storage_result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
+        assert!(storage_result.is_some());
+
+        let server_result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
+        assert_eq!(server_result.state, "online");
+        assert_eq!(cache.lock().unwrap().server_failures, 1);
+    }
+
+    #[test]
+    fn poisoned_admin_cache_mutex_still_returns_fallbacks() {
+        let storage_cache = Mutex::new(PeerAdminCache::new());
+        let server_cache = Mutex::new(PeerAdminCache::new());
+        let endpoints = EndpointServerPools::default();
+
+        let _ = std::panic::catch_unwind(|| {
+            let _guard = storage_cache.lock().expect("test: poison storage cache mutex");
+            panic!("poison storage cache mutex");
+        });
+        let _ = std::panic::catch_unwind(|| {
+            let _guard = server_cache.lock().expect("test: poison server cache mutex");
+            panic!("poison server cache mutex");
+        });
+
+        let storage_result = handle_peer_failure(Some(&storage_cache), "peer-1", &endpoints);
+        assert!(storage_result.is_none());
+
+        let server_result = handle_server_info_failure(Some(&server_cache), "peer-1", &endpoints);
+        assert_eq!(server_result.endpoint, "peer-1");
+        assert_eq!(server_result.state, ItemState::Initializing.to_string());
+    }
+
+    #[test]
+    fn poisoned_admin_cache_recovers_on_success_and_resets_failures() {
+        let storage_cache = Mutex::new(PeerAdminCache {
+            last_storage_info: None,
+            last_server_info: None,
+            storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
+            server_failures: 0,
+        });
+        let server_cache = Mutex::new(PeerAdminCache {
+            last_storage_info: None,
+            last_server_info: None,
+            storage_failures: 0,
+            server_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
+        });
+        let endpoints = EndpointServerPools::default();
+
+        let _ = std::panic::catch_unwind(|| {
+            let _guard = storage_cache.lock().expect("test: poison storage cache mutex");
+            panic!("poison storage cache mutex");
+        });
+        let _ = std::panic::catch_unwind(|| {
+            let _guard = server_cache.lock().expect("test: poison server cache mutex");
+            panic!("poison server cache mutex");
+        });
+
+        update_storage_info_cache(
+            Some(&storage_cache),
+            "peer-1",
+            &StorageInfo {
+                disks: vec![rustfs_madmin::Disk {
+                    endpoint: "disk-0".to_string(),
+                    state: "ok".to_string(),
+                    ..Default::default()
+                }],
+                ..Default::default()
+            },
+        );
+        update_server_info_cache(
+            Some(&server_cache),
+            "peer-1",
+            &ServerProperties {
+                endpoint: "peer-1".to_string(),
+                state: "online".to_string(),
+                ..Default::default()
+            },
+        );
+
+        let storage_result = handle_peer_failure(Some(&storage_cache), "peer-1", &endpoints);
+        assert!(storage_result.is_some());
+        assert_eq!(storage_result.unwrap().disks[0].state, "ok");
+
+        let server_result = handle_server_info_failure(Some(&server_cache), "peer-1", &endpoints);
+        assert_eq!(server_result.state, "online");
+    }
 }

crates/madmin/src/info_commands.rs

@@ -135,7 +135,7 @@ pub struct HealingDisk {
     pub finished: bool,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub enum BackendByte {
     #[default]
     Unknown,
@@ -143,13 +143,13 @@ pub enum BackendByte {
     Erasure,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct StorageInfo {
     pub disks: Vec<Disk>,
     pub backend: BackendInfo,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct BackendDisks(pub HashMap<String, usize>);
 
 impl BackendDisks {
@@ -161,7 +161,7 @@ impl BackendDisks {
     }
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 #[serde(rename_all = "PascalCase", default)]
 pub struct BackendInfo {
     pub backend_type: BackendByte,
@@ -187,7 +187,7 @@ pub const ITEM_OFFLINE: &str = "offline";
 pub const ITEM_INITIALIZING: &str = "initializing";
 pub const ITEM_ONLINE: &str = "online";
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct MemStats {
     pub alloc: u64,
     pub total_alloc: u64,
@@ -196,7 +196,7 @@ pub struct MemStats {
     pub heap_alloc: u64,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct ServerProperties {
     pub state: String,
     pub endpoint: String,