gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 16:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: unify runtime readiness publication and graceful shutdown flow (#3087)

Browse Source 
* fix(server): unify runtime readiness and shutdown flow

* refactor(server): decouple readiness shared types

* refactor(server): keep readiness types crate-private

* refactor(server): await protocol shutdown handles

* fix(server): bypass cached startup readiness
This commit is contained in:
houseme
2026-05-27 02:49:42 +08:00
committed by GitHub
parent 0478505839
commit 658b8dea66
9 changed files with 582 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/app/admin_usecase.rs
+4 -288
View File
@@ -17,6 +17,7 @@
use crate::app::context::{AppContext, get_global_app_context};
use crate::capacity::resolve_admin_used_capacity;
use crate::error::ApiError;
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
use rustfs_data_usage::DataUsageInfo;
use rustfs_ecstore::admin_server_info::get_server_info;
use rustfs_ecstore::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
@@ -24,12 +25,9 @@ use rustfs_ecstore::endpoints::EndpointServerPools;
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::pools::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::store_api::StorageAPI;
use rustfs_madmin::{Disk, InfoMessage, StorageInfo};
use rustfs_madmin::{InfoMessage, StorageInfo};
use s3s::S3ErrorCode;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
pub type AdminUsecaseResult<T> = Result<T, ApiError>;
@@ -49,12 +47,6 @@ impl std::fmt::Debug for QueryServerInfoResponse {
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct DependencyReadiness {
pub storage_ready: bool,
pub iam_ready: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct QueryPoolStatusRequest {
pub pool: String,
@@ -88,16 +80,7 @@ pub struct DefaultAdminUsecase {
context: Option<Arc<AppContext>>,
}
#[derive(Debug, Clone, Copy)]
struct StorageReadinessCacheEntry {
captured_at: Instant,
storage_ready: bool,
}
impl DefaultAdminUsecase {
const DISK_STATE_OK: &'static str = "ok";
const DISK_STATE_UNFORMATTED: &'static str = "unformatted";
const RUNTIME_STATE_RETURNING: &'static str = "returning";
const POOL_STATUS_ACTIVE: &'static str = "active";
const POOL_STATUS_CANCELED: &'static str = "canceled";
const POOL_STATUS_COMPLETE: &'static str = "complete";
@@ -321,160 +304,8 @@ impl DefaultAdminUsecase {
used_size as f64 / total_size as f64
}
fn disk_is_online_for_readiness(disk: &Disk) -> bool {
let state_is_acceptable = disk.state.eq_ignore_ascii_case(Self::DISK_STATE_OK)
|| disk.state.eq_ignore_ascii_case(rustfs_madmin::ITEM_ONLINE)
|| disk.state.eq_ignore_ascii_case(Self::DISK_STATE_UNFORMATTED);
if let Some(runtime_state) = disk.runtime_state.as_deref() {
let runtime_state_is_acceptable = runtime_state.eq_ignore_ascii_case(rustfs_madmin::ITEM_ONLINE)
|| runtime_state.eq_ignore_ascii_case(Self::RUNTIME_STATE_RETURNING);
return runtime_state_is_acceptable && state_is_acceptable;
}
state_is_acceptable
}
fn health_readiness_cache_ttl() -> Duration {
Duration::from_millis(rustfs_utils::get_env_u64(
rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS,
rustfs_config::DEFAULT_HEALTH_READINESS_CACHE_TTL_MS,
))
}
fn storage_readiness_cache() -> &'static Mutex<Option<StorageReadinessCacheEntry>> {
static CACHE: OnceLock<Mutex<Option<StorageReadinessCacheEntry>>> = OnceLock::new();
CACHE.get_or_init(|| Mutex::new(None))
}
async fn load_cached_storage_readiness() -> Option<bool> {
let ttl = Self::health_readiness_cache_ttl();
if ttl.is_zero() {
return None;
}
let cache = Self::storage_readiness_cache().lock().await;
let entry = cache.as_ref()?;
if entry.captured_at.elapsed() <= ttl {
return Some(entry.storage_ready);
}
None
}
async fn update_storage_readiness_cache(storage_ready: bool) {
if Self::health_readiness_cache_ttl().is_zero() {
return;
}
let mut cache = Self::storage_readiness_cache().lock().await;
*cache = Some(StorageReadinessCacheEntry {
captured_at: Instant::now(),
storage_ready,
});
}
fn pool_write_quorum(info: &StorageInfo, pool_idx: usize, set_drive_count: usize) -> usize {
if set_drive_count == 0 {
return 1;
}
let data_drives = info
.backend
.standard_sc_data
.get(pool_idx)
.copied()
.filter(|count| *count > 0)
.unwrap_or_else(|| (set_drive_count / 2).max(1));
let parity_drives = if let Some(drives_per_set) = info.backend.drives_per_set.get(pool_idx).copied() {
drives_per_set.saturating_sub(data_drives)
} else if let Some(parity) = info.backend.standard_sc_parities.get(pool_idx).copied() {
parity
} else if let Some(parity) = info.backend.standard_sc_parity {
parity
} else {
set_drive_count.saturating_sub(data_drives)
};
let mut write_quorum = data_drives;
if data_drives == parity_drives {
write_quorum += 1;
}
write_quorum.max(1)
}
fn storage_ready_from_runtime_state(info: &StorageInfo) -> bool {
if info.disks.is_empty() {
return false;
}
let mut total_online = 0usize;
let mut set_online_counts: HashMap<(usize, usize), usize> = HashMap::new();
let mut set_drive_counts: HashMap<(usize, usize), usize> = HashMap::new();
let mut seen_disks: HashSet<(String, String, i32, i32, i32)> = HashSet::new();
for disk in &info.disks {
if disk.pool_index < 0 || disk.set_index < 0 {
continue;
}
let dedup_key = (
disk.endpoint.clone(),
disk.drive_path.clone(),
disk.pool_index,
disk.set_index,
disk.disk_index,
);
if !seen_disks.insert(dedup_key) {
continue;
}
let pool_idx = disk.pool_index as usize;
let set_idx = disk.set_index as usize;
let key = (pool_idx, set_idx);
*set_drive_counts.entry(key).or_default() += 1;
if Self::disk_is_online_for_readiness(disk) {
total_online += 1;
*set_online_counts.entry(key).or_default() += 1;
}
}
if total_online == 0 {
return false;
}
if set_drive_counts.is_empty() {
return false;
}
set_drive_counts.into_iter().all(|((pool_idx, set_idx), set_drive_count)| {
let online = set_online_counts.get(&(pool_idx, set_idx)).copied().unwrap_or_default();
let write_quorum = Self::pool_write_quorum(info, pool_idx, set_drive_count);
online >= write_quorum
})
}
pub async fn execute_collect_dependency_readiness(&self) -> DependencyReadiness {
let iam_ready = self.context.as_ref().map(|context| context.iam().is_ready()).unwrap_or(false);
let storage_ready = if let Some(cached) = Self::load_cached_storage_readiness().await {
cached
} else {
let computed = if let Some(store) = new_object_layer_fn() {
let storage_info = store.storage_info().await;
Self::storage_ready_from_runtime_state(&storage_info)
} else {
false
};
Self::update_storage_readiness_cache(computed).await;
computed
};
DependencyReadiness {
storage_ready,
iam_ready: iam_ready && storage_ready,
}
collect_runtime_dependency_readiness().await
}
}
@@ -509,121 +340,6 @@ mod tests {
let _ = readiness.iam_ready;
}
#[test]
fn storage_ready_from_runtime_state_returns_false_when_all_disks_faulty() {
let info = StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![1],
drives_per_set: vec![1],
..Default::default()
},
disks: vec![Disk {
pool_index: 0,
set_index: 0,
state: "offline".to_string(),
runtime_state: Some("offline".to_string()),
..Default::default()
}],
};
assert!(!DefaultAdminUsecase::storage_ready_from_runtime_state(&info));
}
#[test]
fn storage_ready_from_runtime_state_returns_true_when_set_meets_write_quorum() {
let info = StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![1],
drives_per_set: vec![1],
..Default::default()
},
disks: vec![Disk {
pool_index: 0,
set_index: 0,
state: "ok".to_string(),
runtime_state: Some("online".to_string()),
..Default::default()
}],
};
assert!(DefaultAdminUsecase::storage_ready_from_runtime_state(&info));
}
#[test]
fn storage_ready_from_runtime_state_deduplicates_duplicate_disk_rows() {
let duplicate_disk = Disk {
endpoint: "127.0.0.1:9000".to_string(),
drive_path: "/data0".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 0,
state: "ok".to_string(),
runtime_state: Some("online".to_string()),
..Default::default()
};
let info = StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![2],
drives_per_set: vec![4],
..Default::default()
},
disks: vec![duplicate_disk.clone(), duplicate_disk],
};
assert!(
!DefaultAdminUsecase::storage_ready_from_runtime_state(&info),
"duplicate rows must not satisfy write quorum"
);
}
#[test]
fn disk_online_for_readiness_requires_runtime_and_state_both_acceptable() {
let disk = Disk {
state: "disk io error".to_string(),
runtime_state: Some("online".to_string()),
..Default::default()
};
assert!(!DefaultAdminUsecase::disk_is_online_for_readiness(&disk));
}
#[test]
fn storage_ready_from_runtime_state_requires_all_sets_meet_quorum() {
let info = StorageInfo {
backend: rustfs_madmin::BackendInfo {
standard_sc_data: vec![1],
drives_per_set: vec![2],
..Default::default()
},
disks: vec![
Disk {
endpoint: "127.0.0.1:9000".to_string(),
drive_path: "/set0d0".to_string(),
pool_index: 0,
set_index: 0,
disk_index: 0,
state: "ok".to_string(),
runtime_state: Some("online".to_string()),
..Default::default()
},
Disk {
endpoint: "127.0.0.1:9000".to_string(),
drive_path: "/set1d0".to_string(),
pool_index: 0,
set_index: 1,
disk_index: 0,
state: "offline".to_string(),
runtime_state: Some("offline".to_string()),
..Default::default()
},
],
};
assert!(
!DefaultAdminUsecase::storage_ready_from_runtime_state(&info),
"if any set fails write quorum, readiness must be false"
);
}
#[test]
fn admin_pool_list_item_maps_capacity_and_active_status() {
let now = OffsetDateTime::UNIX_EPOCH;