From ddd35badad9444ab2f472b5903ab775a67006371 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 8 Jun 2026 16:35:36 +0800 Subject: [PATCH] fix(ecstore): avoid offline disks on admin timeout (#3263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ecstore): avoid offline disks on admin timeout * fix(ecstore): handle poisoned admin cache locks * fix(ecstore): recover poisoned admin cache on success --------- Co-authored-by: houseme Co-authored-by: 安正超 --- .../src/admin_timeout_regression_test.rs | 208 +++++++++ crates/e2e_test/src/lib.rs | 3 + crates/ecstore/src/notification_sys.rs | 413 +++++++++++++++++- crates/madmin/src/info_commands.rs | 12 +- 4 files changed, 610 insertions(+), 26 deletions(-) create mode 100644 crates/e2e_test/src/admin_timeout_regression_test.rs diff --git a/crates/e2e_test/src/admin_timeout_regression_test.rs b/crates/e2e_test/src/admin_timeout_regression_test.rs new file mode 100644 index 000000000..9fb10acae --- /dev/null +++ b/crates/e2e_test/src/admin_timeout_regression_test.rs @@ -0,0 +1,208 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::common::{RustFSTestClusterEnvironment, init_logging, local_http_client}; +use aws_sdk_s3::primitives::ByteStream; +use http::header::HOST; +use reqwest::StatusCode; +use rustfs_madmin::{ITEM_OFFLINE, InfoMessage, StorageInfo}; +use rustfs_signer::constants::UNSIGNED_PAYLOAD; +use rustfs_signer::sign_v4; +use s3s::Body; +use serial_test::serial; +use std::error::Error; +use std::process::Command; +use tokio::time::{Duration, sleep, timeout}; +use uuid::Uuid; + +const BUCKET: &str = "issue-2525-admin-timeout"; + +async fn signed_admin_get( + url: &str, + access_key: &str, + secret_key: &str, +) -> Result> { + let uri = url.parse::()?; + let authority = uri.authority().ok_or("request URL missing authority")?.to_string(); + let request = http::Request::builder() + .method(http::Method::GET) + .uri(uri) + .header(HOST, authority) + .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) + .body(Body::empty())?; + let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1"); + + let mut builder = local_http_client().get(url); + for (name, value) in signed.headers() { + builder = builder.header(name, value); + } + + Ok(builder.send().await?) +} + +async fn fetch_info(cluster: &RustFSTestClusterEnvironment) -> Result> { + let url = format!("{}/rustfs/admin/v3/info", cluster.nodes[0].url); + let response = signed_admin_get(&url, &cluster.access_key, &cluster.secret_key).await?; + parse_json_response(response, "cluster info").await +} + +async fn fetch_storage_info(cluster: &RustFSTestClusterEnvironment) -> Result> { + let url = format!("{}/rustfs/admin/v3/storageinfo", cluster.nodes[0].url); + let response = signed_admin_get(&url, &cluster.access_key, &cluster.secret_key).await?; + parse_json_response(response, "storage info").await +} + +async fn parse_json_response( + response: reqwest::Response, + label: &str, +) -> Result> { + let status = response.status(); + let body = response.bytes().await?; + if status != StatusCode::OK { + return Err(format!("{label} request failed: {status} {}", String::from_utf8_lossy(body.as_ref())).into()); + } + + Ok(serde_json::from_slice(&body)?) +} + +fn signal_process(pid: u32, signal: &str) -> Result<(), Box> { + let output = Command::new("kill").arg(format!("-{signal}")).arg(pid.to_string()).output()?; + if output.status.success() { + return Ok(()); + } + + Err(format!("kill -{signal} {pid} failed: {}", String::from_utf8_lossy(&output.stderr)).into()) +} + +fn offline_server_count(info: &InfoMessage) -> usize { + info.servers + .as_ref() + .map(|servers| servers.iter().filter(|server| server.state == ITEM_OFFLINE).count()) + .unwrap_or_default() +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box> { + init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + cluster.start().await?; + cluster.create_test_bucket(BUCKET).await?; + + let warm_info = fetch_info(&cluster).await?; + assert_eq!( + warm_info + .backend + .as_ref() + .map(|backend| backend.offline_disks) + .unwrap_or_default(), + 0, + "cluster should start with zero offline disks" + ); + + let warm_storage = fetch_storage_info(&cluster).await?; + assert_eq!( + warm_storage.backend.offline_disks.sum(), + 0, + "storage info should start with zero offline disks" + ); + + let suspended_pid = cluster.nodes[1] + .process + .as_ref() + .ok_or("cluster node 1 process missing")? + .id(); + signal_process(suspended_pid, "STOP")?; + + let resume = tokio::spawn(async move { + sleep(Duration::from_secs(6)).await; + signal_process(suspended_pid, "CONT") + }); + + let client = cluster.create_s3_client(0)?; + let key = format!("issue-2525/{}", Uuid::new_v4().simple()); + let payload = b"admin timeout regression payload".to_vec(); + + let during = timeout(Duration::from_secs(20), async { + tokio::join!(fetch_info(&cluster), fetch_storage_info(&cluster), async { + client + .put_object() + .bucket(BUCKET) + .key(&key) + .body(ByteStream::from(payload.clone())) + .send() + .await?; + Ok::<(), Box>(()) + }) + }) + .await; + + resume.await??; + + let (info_during, storage_during, put_result) = + during.map_err(|_| "timed out while waiting for admin queries during suspended peer")?; + let info_during = info_during?; + let storage_during = storage_during?; + put_result?; + + assert_eq!( + info_during + .backend + .as_ref() + .map(|backend| backend.offline_disks) + .unwrap_or_default(), + 0, + "single admin timeout must not synthesize offline disks in /info" + ); + assert_eq!( + offline_server_count(&info_during), + 0, + "single admin timeout must not mark any server offline in /info" + ); + assert_eq!( + storage_during.backend.offline_disks.sum(), + 0, + "single admin timeout must not synthesize offline disks in /storageinfo" + ); + assert!( + storage_during.disks.iter().all(|disk| disk.state != ITEM_OFFLINE), + "single admin timeout must not synthesize offline disk entries in /storageinfo" + ); + + let recovered_info = fetch_info(&cluster).await?; + assert_eq!( + recovered_info + .backend + .as_ref() + .map(|backend| backend.offline_disks) + .unwrap_or_default(), + 0, + "offline disk count should stay at zero after the peer resumes" + ); + + let body = client + .get_object() + .bucket(BUCKET) + .key(&key) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(body.as_ref(), b"admin timeout regression payload"); + + Ok(()) +} diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 4fe216f03..133de9a0a 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -149,4 +149,7 @@ mod snowball_auto_extract_test; #[cfg(test)] mod namespace_lock_quorum_test; +#[cfg(test)] +mod admin_timeout_regression_test; + pub mod tls_gen; diff --git a/crates/ecstore/src/notification_sys.rs b/crates/ecstore/src/notification_sys.rs index 452508e3e..968808ceb 100644 --- a/crates/ecstore/src/notification_sys.rs +++ b/crates/ecstore/src/notification_sys.rs @@ -25,15 +25,37 @@ use lazy_static::lazy_static; use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices}; use rustfs_madmin::metrics::RealtimeMetrics; use rustfs_madmin::net::NetInfo; -use rustfs_madmin::{ItemState, ServerProperties}; +use rustfs_madmin::{ItemState, ServerProperties, StorageInfo}; use std::collections::hash_map::DefaultHasher; use std::future::Future; use std::hash::{Hash, Hasher}; -use std::sync::OnceLock; +use std::sync::{Mutex, OnceLock}; use std::time::{Duration, SystemTime}; use tokio::time::timeout; use tracing::{error, warn}; +/// After this many consecutive admin-call failures, mark the peer as offline. +const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3; + +/// Cached result from the last successful admin call to a peer. +struct PeerAdminCache { + last_storage_info: Option, + last_server_info: Option, + storage_failures: u32, + server_failures: u32, +} + +impl PeerAdminCache { + fn new() -> Self { + Self { + last_storage_info: None, + last_server_info: None, + storage_failures: 0, + server_failures: 0, + } + } +} + lazy_static! { pub static ref GLOBAL_NotificationSys: OnceLock = OnceLock::new(); } @@ -53,14 +75,17 @@ pub struct NotificationSys { pub peer_clients: Vec>, #[allow(dead_code)] pub all_peer_clients: Vec>, + peer_admin_caches: Vec>, } impl NotificationSys { pub async fn new(eps: EndpointServerPools) -> Self { let (peer_clients, all_peer_clients) = PeerRestClient::new_clients(eps).await; + let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect(); Self { peer_clients, all_peer_clients, + peer_admin_caches, } } } @@ -251,30 +276,27 @@ impl NotificationSys { pub async fn storage_info(&self, api: &S) -> rustfs_madmin::StorageInfo { let mut futures = Vec::with_capacity(self.peer_clients.len()); let endpoints = get_global_endpoints(); - let peer_timeout = Duration::from_secs(2); // Same timeout as server_info + let peer_timeout = Duration::from_secs(5); - for client in self.peer_clients.iter() { + for (idx, client) in self.peer_clients.iter().enumerate() { let endpoints = endpoints.clone(); + let cache = self.peer_admin_caches.get(idx); futures.push(async move { if let Some(client) = client { let host = client.host.to_string(); - // Wrap in timeout to ensure we don't hang on dead peers match timeout(peer_timeout, client.local_storage_info()).await { - Ok(Ok(info)) => Some(info), + Ok(Ok(info)) => { + update_storage_info_cache(cache, &host, &info); + Some(info) + } Ok(Err(err)) => { warn!("peer {} storage_info failed: {}", host, err); - Some(rustfs_madmin::StorageInfo { - disks: get_offline_disks(&host, &endpoints), - ..Default::default() - }) + handle_peer_failure(cache, &host, &endpoints) } Err(_) => { warn!("peer {} storage_info timed out after {:?}", host, peer_timeout); client.evict_connection().await; - Some(rustfs_madmin::StorageInfo { - disks: get_offline_disks(&host, &endpoints), - ..Default::default() - }) + handle_peer_failure(cache, &host, &endpoints) } } } else { @@ -299,24 +321,27 @@ impl NotificationSys { pub async fn server_info(&self) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); let endpoints = get_global_endpoints(); - let peer_timeout = Duration::from_secs(2); + let peer_timeout = Duration::from_secs(5); - for client in self.peer_clients.iter() { + for (idx, client) in self.peer_clients.iter().enumerate() { let endpoints = endpoints.clone(); + let cache = self.peer_admin_caches.get(idx); futures.push(async move { if let Some(client) = client { let host = client.host.to_string(); match timeout(peer_timeout, client.server_info()).await { - Ok(Ok(info)) => info, + Ok(Ok(info)) => { + update_server_info_cache(cache, &host, &info); + info + } Ok(Err(err)) => { warn!("peer {} server_info failed: {}", host, err); - // client.server_info handles eviction internally on error, but fallback needed - offline_server_properties(&host, &endpoints) + handle_server_info_failure(cache, &host, &endpoints) } Err(_) => { warn!("peer {} server_info timed out after {:?}", host, peer_timeout); client.evict_connection().await; - offline_server_properties(&host, &endpoints) + handle_server_info_failure(cache, &host, &endpoints) } } } else { @@ -841,6 +866,113 @@ where } } +/// Handle a peer failure for storage_info: return cached data if available, +/// or mark offline only after consecutive failures exceed the threshold. +fn handle_peer_failure( + cache: Option<&Mutex>, + host: &str, + endpoints: &EndpointServerPools, +) -> Option { + let cache = cache?; + + let mut c = match cache.lock() { + Ok(cache) => cache, + Err(poisoned) => { + warn!("peer {host} storage_info cache mutex poisoned"); + poisoned.into_inner() + } + }; + c.storage_failures += 1; + + if let Some(ref cached) = c.last_storage_info + && c.storage_failures < CONSECUTIVE_FAILURE_THRESHOLD + { + return Some(cached.clone()); + } + + if c.storage_failures >= CONSECUTIVE_FAILURE_THRESHOLD { + return Some(StorageInfo { + disks: get_offline_disks(host, endpoints), + ..Default::default() + }); + } + + None +} + +fn update_storage_info_cache(cache: Option<&Mutex>, host: &str, info: &StorageInfo) { + let Some(cache) = cache else { + return; + }; + + let mut c = match cache.lock() { + Ok(cache) => cache, + Err(poisoned) => { + warn!("peer {host} storage_info cache mutex poisoned"); + poisoned.into_inner() + } + }; + c.last_storage_info = Some(info.clone()); + c.storage_failures = 0; +} + +/// Handle a peer failure for server_info: return cached data if available, +/// or mark offline only after consecutive failures exceed the threshold. +fn handle_server_info_failure( + cache: Option<&Mutex>, + host: &str, + endpoints: &EndpointServerPools, +) -> ServerProperties { + let Some(cache) = cache else { + return initializing_server_properties(host); + }; + + let mut c = match cache.lock() { + Ok(cache) => cache, + Err(poisoned) => { + warn!("peer {host} server_info cache mutex poisoned"); + poisoned.into_inner() + } + }; + c.server_failures += 1; + + if let Some(ref cached) = c.last_server_info + && c.server_failures < CONSECUTIVE_FAILURE_THRESHOLD + { + return cached.clone(); + } + + if c.server_failures >= CONSECUTIVE_FAILURE_THRESHOLD { + return offline_server_properties(host, endpoints); + } + + initializing_server_properties(host) +} + +fn update_server_info_cache(cache: Option<&Mutex>, host: &str, info: &ServerProperties) { + let Some(cache) = cache else { + return; + }; + + let mut c = match cache.lock() { + Ok(cache) => cache, + Err(poisoned) => { + warn!("peer {host} server_info cache mutex poisoned"); + poisoned.into_inner() + } + }; + c.last_server_info = Some(info.clone()); + c.server_failures = 0; +} + +fn initializing_server_properties(host: &str) -> ServerProperties { + ServerProperties { + endpoint: host.to_string(), + state: ItemState::Initializing.to_string().to_owned(), + ..Default::default() + } +} + fn offline_server_properties(host: &str, endpoints: &EndpointServerPools) -> ServerProperties { ServerProperties { uptime: GLOBAL_BOOT_TIME @@ -964,6 +1096,7 @@ mod tests { let sys = NotificationSys { peer_clients: vec![None], all_peer_clients: Vec::new(), + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], }; let err = sys @@ -982,6 +1115,7 @@ mod tests { let sys = NotificationSys { peer_clients: vec![None], all_peer_clients: Vec::new(), + peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())], }; let results = sys.load_transition_tier_config().await; @@ -990,4 +1124,243 @@ mod tests { assert!(results[0].err.is_some()); assert!(results[0].err.as_ref().unwrap().to_string().contains("peer is not reachable")); } + + // --- Tests for handle_peer_failure / handle_server_info_failure caching --- + + #[test] + fn handle_peer_failure_first_failure_returns_none_when_no_cache() { + let cache = Mutex::new(PeerAdminCache::new()); + let endpoints = EndpointServerPools::default(); + + let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints); + assert!(result.is_none()); + assert_eq!(cache.lock().unwrap().storage_failures, 1); + } + + #[test] + fn handle_peer_failure_returns_cached_data_on_single_failure() { + let cached_info = StorageInfo { + disks: vec![rustfs_madmin::Disk { + endpoint: "disk-0".to_string(), + state: "ok".to_string(), + ..Default::default() + }], + ..Default::default() + }; + + let cache = Mutex::new(PeerAdminCache { + last_storage_info: Some(cached_info), + last_server_info: None, + storage_failures: 0, + server_failures: 0, + }); + let endpoints = EndpointServerPools::default(); + + // First failure: should return cached data + let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints); + let info = result.unwrap(); + assert_eq!(info.disks.len(), 1); + assert_eq!(info.disks[0].state, "ok"); + assert_eq!(cache.lock().unwrap().storage_failures, 1); + } + + #[test] + fn handle_peer_failure_returns_offline_after_threshold_exceeded() { + let cached_info = StorageInfo { + disks: vec![rustfs_madmin::Disk { + endpoint: "disk-0".to_string(), + state: "ok".to_string(), + ..Default::default() + }], + ..Default::default() + }; + + let cache = Mutex::new(PeerAdminCache { + last_storage_info: Some(cached_info), + last_server_info: None, + storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1, + server_failures: 0, + }); + let endpoints = EndpointServerPools::default(); + + // This failure pushes us to the threshold => offline + let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints); + assert!(result.is_some()); + assert_eq!(cache.lock().unwrap().storage_failures, CONSECUTIVE_FAILURE_THRESHOLD); + } + + #[test] + fn handle_server_info_failure_returns_cached_on_single_failure() { + let cached_props = ServerProperties { + endpoint: "peer-1".to_string(), + state: "online".to_string(), + ..Default::default() + }; + + let cache = Mutex::new(PeerAdminCache { + last_storage_info: None, + last_server_info: Some(cached_props), + storage_failures: 0, + server_failures: 0, + }); + let endpoints = EndpointServerPools::default(); + + let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints); + assert_eq!(result.endpoint, "peer-1"); + assert_eq!(result.state, "online"); + assert_eq!(cache.lock().unwrap().server_failures, 1); + } + + #[test] + fn handle_server_info_failure_returns_initializing_before_threshold_without_cache() { + let cache = Mutex::new(PeerAdminCache::new()); + let endpoints = EndpointServerPools::default(); + + let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints); + assert_eq!(result.endpoint, "peer-1"); + assert_eq!(result.state, ItemState::Initializing.to_string()); + assert!(result.disks.is_empty()); + assert_eq!(cache.lock().unwrap().server_failures, 1); + } + + #[test] + fn handle_server_info_failure_returns_offline_after_threshold() { + let cached_props = ServerProperties { + endpoint: "peer-1".to_string(), + state: "online".to_string(), + ..Default::default() + }; + + let cache = Mutex::new(PeerAdminCache { + last_storage_info: None, + last_server_info: Some(cached_props), + storage_failures: 0, + server_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1, + }); + let endpoints = EndpointServerPools::default(); + + let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints); + assert_eq!(result.state, ItemState::Offline.to_string()); + assert_eq!(cache.lock().unwrap().server_failures, CONSECUTIVE_FAILURE_THRESHOLD); + } + + #[test] + fn success_resets_failure_counters_independently() { + let cache = Mutex::new(PeerAdminCache { + last_storage_info: None, + last_server_info: None, + storage_failures: 2, + server_failures: 2, + }); + + { + let mut c = cache.lock().unwrap(); + c.last_storage_info = Some(StorageInfo::default()); + c.storage_failures = 0; + } + + let cache = cache.lock().unwrap(); + assert_eq!(cache.storage_failures, 0); + assert_eq!(cache.server_failures, 2); + } + + #[test] + fn storage_failures_do_not_affect_server_failures() { + let cache = Mutex::new(PeerAdminCache { + last_storage_info: Some(StorageInfo::default()), + last_server_info: Some(ServerProperties { + endpoint: "peer-1".to_string(), + state: "online".to_string(), + ..Default::default() + }), + storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1, + server_failures: 0, + }); + let endpoints = EndpointServerPools::default(); + + let storage_result = handle_peer_failure(Some(&cache), "peer-1", &endpoints); + assert!(storage_result.is_some()); + + let server_result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints); + assert_eq!(server_result.state, "online"); + assert_eq!(cache.lock().unwrap().server_failures, 1); + } + + #[test] + fn poisoned_admin_cache_mutex_still_returns_fallbacks() { + let storage_cache = Mutex::new(PeerAdminCache::new()); + let server_cache = Mutex::new(PeerAdminCache::new()); + let endpoints = EndpointServerPools::default(); + + let _ = std::panic::catch_unwind(|| { + let _guard = storage_cache.lock().expect("test: poison storage cache mutex"); + panic!("poison storage cache mutex"); + }); + let _ = std::panic::catch_unwind(|| { + let _guard = server_cache.lock().expect("test: poison server cache mutex"); + panic!("poison server cache mutex"); + }); + + let storage_result = handle_peer_failure(Some(&storage_cache), "peer-1", &endpoints); + assert!(storage_result.is_none()); + + let server_result = handle_server_info_failure(Some(&server_cache), "peer-1", &endpoints); + assert_eq!(server_result.endpoint, "peer-1"); + assert_eq!(server_result.state, ItemState::Initializing.to_string()); + } + + #[test] + fn poisoned_admin_cache_recovers_on_success_and_resets_failures() { + let storage_cache = Mutex::new(PeerAdminCache { + last_storage_info: None, + last_server_info: None, + storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1, + server_failures: 0, + }); + let server_cache = Mutex::new(PeerAdminCache { + last_storage_info: None, + last_server_info: None, + storage_failures: 0, + server_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1, + }); + let endpoints = EndpointServerPools::default(); + + let _ = std::panic::catch_unwind(|| { + let _guard = storage_cache.lock().expect("test: poison storage cache mutex"); + panic!("poison storage cache mutex"); + }); + let _ = std::panic::catch_unwind(|| { + let _guard = server_cache.lock().expect("test: poison server cache mutex"); + panic!("poison server cache mutex"); + }); + + update_storage_info_cache( + Some(&storage_cache), + "peer-1", + &StorageInfo { + disks: vec![rustfs_madmin::Disk { + endpoint: "disk-0".to_string(), + state: "ok".to_string(), + ..Default::default() + }], + ..Default::default() + }, + ); + update_server_info_cache( + Some(&server_cache), + "peer-1", + &ServerProperties { + endpoint: "peer-1".to_string(), + state: "online".to_string(), + ..Default::default() + }, + ); + + let storage_result = handle_peer_failure(Some(&storage_cache), "peer-1", &endpoints); + assert!(storage_result.is_some()); + assert_eq!(storage_result.unwrap().disks[0].state, "ok"); + + let server_result = handle_server_info_failure(Some(&server_cache), "peer-1", &endpoints); + assert_eq!(server_result.state, "online"); + } } diff --git a/crates/madmin/src/info_commands.rs b/crates/madmin/src/info_commands.rs index aad577d6d..1e35c7cea 100644 --- a/crates/madmin/src/info_commands.rs +++ b/crates/madmin/src/info_commands.rs @@ -135,7 +135,7 @@ pub struct HealingDisk { pub finished: bool, } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub enum BackendByte { #[default] Unknown, @@ -143,13 +143,13 @@ pub enum BackendByte { Erasure, } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct StorageInfo { pub disks: Vec, pub backend: BackendInfo, } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct BackendDisks(pub HashMap); impl BackendDisks { @@ -161,7 +161,7 @@ impl BackendDisks { } } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase", default)] pub struct BackendInfo { pub backend_type: BackendByte, @@ -187,7 +187,7 @@ pub const ITEM_OFFLINE: &str = "offline"; pub const ITEM_INITIALIZING: &str = "initializing"; pub const ITEM_ONLINE: &str = "online"; -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct MemStats { pub alloc: u64, pub total_alloc: u64, @@ -196,7 +196,7 @@ pub struct MemStats { pub heap_alloc: u64, } -#[derive(Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ServerProperties { pub state: String, pub endpoint: String,