From 9badf3939c90dbd8bae9a80b418e337d7182920d Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 7 Sep 2026 19:55:20 +0800 Subject: [PATCH] fix(admin): report pool metadata write blocks (#7396) Co-authored-by: zhi22915 --- rustfs/src/admin/handlers/cluster_snapshot.rs | 36 ++++++ rustfs/src/server/readiness.rs | 121 +++++++++++++++--- rustfs/src/shared_types.rs | 2 + 3 files changed, 139 insertions(+), 20 deletions(-) diff --git a/rustfs/src/admin/handlers/cluster_snapshot.rs b/rustfs/src/admin/handlers/cluster_snapshot.rs index f9d2d35c6..d5cf28a40 100644 --- a/rustfs/src/admin/handlers/cluster_snapshot.rs +++ b/rustfs/src/admin/handlers/cluster_snapshot.rs @@ -656,6 +656,7 @@ fn summarize_storage_readiness(snapshot: &ClusterReadOnlySnapshot) -> Capability .iter() .filter_map(|reason| match reason { ReadinessDegradedReason::StorageQuorumUnavailable + | ReadinessDegradedReason::PoolMetaWriteBlocked | ReadinessDegradedReason::StorageAndIamUnavailable | ReadinessDegradedReason::StorageAndLockUnavailable | ReadinessDegradedReason::StorageIamAndLockUnavailable => Some(reason.as_str()), @@ -1282,6 +1283,41 @@ mod tests { assert_eq!(view.components.usage.last_usage_save_result, "skipped_stale"); } + #[test] + fn cluster_snapshot_storage_summary_reports_pool_meta_write_blocked() { + let snapshot = ClusterReadOnlySnapshot { + topology: TopologySnapshot::default(), + membership: ClusterMembershipSnapshot::default(), + pool_state: ClusterPoolStateSnapshot::default(), + local_storage: ClusterLocalNodeStorageSnapshot::default(), + peer_health: ClusterPeerHealthSnapshot::default(), + rpc_boundary: sample_rpc_boundary_snapshot(), + observability: ObservabilitySnapshot::default(), + workload_admission: WorkloadAdmissionRegistrySnapshot::new(Vec::new()), + runtime_status: ClusterRuntimeStatusSnapshot { + readiness: DependencyReadiness { + storage_ready: false, + iam_ready: true, + lock_quorum_ready: true, + peer_health_ready: true, + }, + state: ClusterRuntimeReadinessState::Degraded, + degraded_reasons: vec![ReadinessDegradedReason::PoolMetaWriteBlocked], + }, + usage_freshness: ClusterUsageFreshnessSnapshot::default(), + listing_diagnostics: ClusterListingDiagnosticsSnapshot::default(), + }; + + let view = ClusterSnapshotView::from(snapshot); + + assert_eq!(view.components.storage.status.state, CapabilityState::Unknown); + assert_eq!( + view.components.storage.status.reason.as_deref(), + Some("storage readiness degraded: pool_meta_write_blocked") + ); + assert_eq!(view.components.storage.condition, "degraded"); + } + #[test] fn cluster_snapshot_listing_component_keeps_historical_stalls_as_evidence() { let snapshot = ClusterReadOnlySnapshot { diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index db35d7966..f6bf5991e 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -256,7 +256,13 @@ pub async fn publish_ready_when_runtime_ready( #[derive(Debug, Clone, Copy)] struct StorageReadinessCacheEntry { captured_at: Instant, - storage_ready: bool, + status: StorageWriteReadinessStatus, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct StorageWriteReadinessStatus { + ready: bool, + pool_meta_write_blocked: bool, } #[derive(Debug, Clone, Copy)] @@ -356,7 +362,7 @@ async fn reset_cluster_health_report_caches() { *cluster_read_health_report_cache().lock().await = None; } -async fn load_cached_storage_readiness() -> Option { +async fn load_cached_storage_readiness() -> Option { let ttl = health_readiness_cache_ttl(); if ttl.is_zero() { return None; @@ -365,7 +371,7 @@ async fn load_cached_storage_readiness() -> Option { let cache = storage_readiness_cache().lock().await; let entry = cache.as_ref()?; if entry.captured_at.elapsed() <= ttl { - return Some(entry.storage_ready); + return Some(entry.status); } None @@ -398,7 +404,7 @@ async fn update_cluster_health_report_cache(kind: ClusterHealthProbeKind, report }); } -async fn update_storage_readiness_cache(storage_ready: bool) { +async fn update_storage_readiness_cache(status: StorageWriteReadinessStatus) { if health_readiness_cache_ttl().is_zero() { return; } @@ -406,7 +412,7 @@ async fn update_storage_readiness_cache(storage_ready: bool) { let mut cache = storage_readiness_cache().lock().await; *cache = Some(StorageReadinessCacheEntry { captured_at: Instant::now(), - storage_ready, + status, }); } @@ -669,6 +675,20 @@ fn degraded_reasons(readiness: DependencyReadiness) -> Vec Vec { + let mut reasons = degraded_reasons(readiness); + if pool_meta_write_blocked { + reasons.retain(|reason| *reason != ReadinessDegradedReason::StorageQuorumUnavailable); + if !reasons.contains(&ReadinessDegradedReason::PoolMetaWriteBlocked) { + reasons.insert(0, ReadinessDegradedReason::PoolMetaWriteBlocked); + } + } + reasons +} + fn record_readiness_report(report: &DependencyReadinessReport) { let ready = report.readiness.storage_ready && report.readiness.iam_ready @@ -692,24 +712,34 @@ fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) -> } } +fn dependency_readiness_report_from_write_status( + readiness: DependencyReadiness, + storage: StorageWriteReadinessStatus, +) -> DependencyReadinessReport { + DependencyReadinessReport { + degraded_reasons: degraded_reasons_with_pool_meta_status(readiness, storage.pool_meta_write_blocked), + readiness, + } +} + pub async fn collect_dependency_readiness_report() -> DependencyReadinessReport { let iam_ready_raw = runtime_sources::current_iam_ready(); - let storage_ready = if let Some(cached) = load_cached_storage_readiness().await { + let storage = if let Some(cached) = load_cached_storage_readiness().await { cached } else { - let computed = collect_storage_readiness_uncached().await; + let computed = collect_storage_write_readiness_uncached().await; update_storage_readiness_cache(computed).await; computed }; let lock_quorum_status = collect_lock_quorum_status().await; let readiness = DependencyReadiness { - storage_ready, + storage_ready: storage.ready, iam_ready: iam_ready_raw, lock_quorum_ready: lock_quorum_status.ready, peer_health_ready: collect_peer_health_readiness(), }; - let report = dependency_readiness_report_from_readiness(readiness); + let report = dependency_readiness_report_from_write_status(readiness, storage); record_readiness_report(&report); report } @@ -723,16 +753,14 @@ pub async fn collect_cluster_read_health_report() -> DependencyReadinessReport { } pub async fn collect_node_readiness_report() -> DependencyReadinessReport { + let storage = node_pool_meta_write_readiness().await; let readiness = DependencyReadiness { - storage_ready: match runtime_sources::current_object_store_handle() { - Some(store) => store.pool_meta_writes_ready().await, - None => false, - }, + storage_ready: storage.ready, iam_ready: runtime_sources::current_iam_ready(), lock_quorum_ready: collect_lock_quorum_status().await.ready, peer_health_ready: collect_peer_health_readiness(), }; - let report = dependency_readiness_report_from_readiness(readiness); + let report = dependency_readiness_report_from_write_status(readiness, storage); record_readiness_report(&report); report } @@ -795,14 +823,15 @@ pub async fn collect_cluster_read_dependency_readiness_report() -> DependencyRea } pub(crate) async fn snapshot_dependency_readiness_report() -> DependencyReadinessReport { + let storage = collect_storage_write_readiness_uncached().await; let readiness = DependencyReadiness { - storage_ready: collect_storage_readiness_uncached().await, + storage_ready: storage.ready, iam_ready: runtime_sources::current_iam_ready(), lock_quorum_ready: collect_lock_quorum_status_uncached().await.ready, peer_health_ready: collect_peer_health_readiness(), }; - dependency_readiness_report_from_readiness(readiness) + dependency_readiness_report_from_write_status(readiness, storage) } async fn collect_lock_quorum_status() -> LockQuorumStatus { @@ -815,15 +844,33 @@ async fn collect_lock_quorum_status() -> LockQuorumStatus { } } -async fn collect_storage_readiness_uncached() -> bool { +async fn node_pool_meta_write_readiness() -> StorageWriteReadinessStatus { + if let Some(store) = runtime_sources::current_object_store_handle() { + let ready = store.pool_meta_writes_ready().await; + return StorageWriteReadinessStatus { + ready, + pool_meta_write_blocked: !ready, + }; + } + + StorageWriteReadinessStatus::default() +} + +async fn collect_storage_write_readiness_uncached() -> StorageWriteReadinessStatus { if let Some(store) = runtime_sources::current_object_store_handle() { if !store.pool_meta_writes_ready().await { - return false; + return StorageWriteReadinessStatus { + ready: false, + pool_meta_write_blocked: true, + }; } let storage_info = StorageAdminApi::storage_info(store.as_ref()).await; - storage_ready_from_runtime_state(&storage_info) + StorageWriteReadinessStatus { + ready: storage_ready_from_runtime_state(&storage_info), + pool_meta_write_blocked: false, + } } else { - false + StorageWriteReadinessStatus::default() } } @@ -1802,6 +1849,40 @@ mod tests { ); } + #[test] + fn degraded_reasons_report_pool_meta_write_blocked() { + let readiness = DependencyReadiness { + storage_ready: false, + iam_ready: true, + lock_quorum_ready: true, + peer_health_ready: true, + }; + + assert_eq!( + degraded_reasons_with_pool_meta_status(readiness, true), + vec![ReadinessDegradedReason::PoolMetaWriteBlocked] + ); + } + + #[test] + fn degraded_reasons_keep_pool_meta_source_with_other_failures() { + let readiness = DependencyReadiness { + storage_ready: false, + iam_ready: true, + lock_quorum_ready: false, + peer_health_ready: false, + }; + + assert_eq!( + degraded_reasons_with_pool_meta_status(readiness, true), + vec![ + ReadinessDegradedReason::PoolMetaWriteBlocked, + ReadinessDegradedReason::StorageAndLockUnavailable, + ReadinessDegradedReason::PeerHealthUnavailable, + ] + ); + } + #[test] fn degraded_reasons_append_peer_health_gate_failures() { let readiness = DependencyReadiness { diff --git a/rustfs/src/shared_types.rs b/rustfs/src/shared_types.rs index d307d5520..ebbafc834 100644 --- a/rustfs/src/shared_types.rs +++ b/rustfs/src/shared_types.rs @@ -43,6 +43,7 @@ pub enum ReadinessDegradedReason { KmsNotReady, ObjectReadStalled, ObjectWriteStalled, + PoolMetaWriteBlocked, ClusterHealthTimeout, PeerHealthUnavailable, StartupFinalizationPending, @@ -61,6 +62,7 @@ impl ReadinessDegradedReason { ReadinessDegradedReason::KmsNotReady => "kms_not_ready", ReadinessDegradedReason::ObjectReadStalled => "object_read_stalled", ReadinessDegradedReason::ObjectWriteStalled => "object_write_stalled", + ReadinessDegradedReason::PoolMetaWriteBlocked => "pool_meta_write_blocked", ReadinessDegradedReason::ClusterHealthTimeout => "cluster_health_timeout", ReadinessDegradedReason::PeerHealthUnavailable => "peer_health_unavailable", ReadinessDegradedReason::StartupFinalizationPending => "startup_finalization_pending",