From 27a491a730f15e55f0aa865931efe97080cb9352 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 29 Jun 2026 19:55:38 +0800 Subject: [PATCH] refactor(runtime): guard replication stats globals (#4058) --- .../src/bucket/replication/replication_pool.rs | 4 ++++ crates/obs/src/metrics/mod.rs | 4 ++-- crates/obs/src/metrics/runtime_sources.rs | 6 +++--- crates/obs/src/metrics/storage_api.rs | 6 +++--- docs/architecture/global-state-inventory.md | 2 +- rustfs/src/storage/storage_api.rs | 6 +++--- scripts/check_architecture_migration_rules.sh | 13 +++++++++++++ 7 files changed, 29 insertions(+), 12 deletions(-) diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 5449d522b..e7663bf77 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -1407,6 +1407,10 @@ pub fn get_global_replication_pool() -> Option> { runtime_sources::replication_pool() } +pub fn get_global_replication_stats() -> Option> { + runtime_sources::replication_stats() +} + pub async fn schedule_replication( oi: ObjectInfo, o: Arc, diff --git a/crates/obs/src/metrics/mod.rs b/crates/obs/src/metrics/mod.rs index b58f75355..966b3b610 100644 --- a/crates/obs/src/metrics/mod.rs +++ b/crates/obs/src/metrics/mod.rs @@ -31,8 +31,8 @@ pub use scheduler::{ init_metrics_runtime, metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot, }; pub(crate) use storage_api::metrics::{ - BucketOperations, BucketOptions, OBS_GLOBAL_REPLICATION_STATS, ObsBucketBandwidthMonitor, ObsEcstoreResult, - ObsReplicationStats, ObsStore, StorageAdminApi, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config, + BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsReplicationStats, ObsStore, StorageAdminApi, + obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_global_replication_stats, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_load_data_usage_from_backend, obs_resolve_object_store_handle, obs_transition_state_handle, }; diff --git a/crates/obs/src/metrics/runtime_sources.rs b/crates/obs/src/metrics/runtime_sources.rs index 545894d06..bffc88416 100644 --- a/crates/obs/src/metrics/runtime_sources.rs +++ b/crates/obs/src/metrics/runtime_sources.rs @@ -13,8 +13,8 @@ // limitations under the License. use crate::metrics::{ - OBS_GLOBAL_REPLICATION_STATS, ObsBucketBandwidthMonitor, ObsReplicationStats, obs_expiry_state_handle, - obs_get_global_bucket_monitor, obs_transition_state_handle, + ObsBucketBandwidthMonitor, ObsReplicationStats, obs_expiry_state_handle, obs_get_global_bucket_monitor, + obs_get_global_replication_stats, obs_transition_state_handle, }; use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot}; use std::sync::Arc; @@ -64,7 +64,7 @@ pub(crate) fn bucket_monitor_available() -> bool { } pub(crate) fn replication_stats_handle() -> Option { - OBS_GLOBAL_REPLICATION_STATS.get().cloned() + obs_get_global_replication_stats() } pub(crate) async fn ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot { diff --git a/crates/obs/src/metrics/storage_api.rs b/crates/obs/src/metrics/storage_api.rs index 2e3076b13..30bd45188 100644 --- a/crates/obs/src/metrics/storage_api.rs +++ b/crates/obs/src/metrics/storage_api.rs @@ -15,7 +15,7 @@ pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config; pub(crate) use rustfs_ecstore::api::bucket::replication::{ - GLOBAL_REPLICATION_STATS as OBS_GLOBAL_REPLICATION_STATS, ReplicationStats as ObsReplicationStats, + ReplicationStats as ObsReplicationStats, get_global_replication_stats as obs_get_global_replication_stats, }; pub(crate) use rustfs_ecstore::api::capacity::{ get_total_usable_capacity as obs_get_total_usable_capacity, @@ -34,8 +34,8 @@ pub(crate) mod metrics { pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi}; pub(crate) use super::{ - OBS_GLOBAL_REPLICATION_STATS, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsReplicationStats, ObsStore, - obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, + ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsReplicationStats, ObsStore, obs_expiry_state_handle, + obs_get_global_bucket_monitor, obs_get_global_replication_stats, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_load_data_usage_from_backend, obs_resolve_object_store_handle, obs_transition_state_handle, }; diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 0733b32b1..cbf9a9a00 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -43,7 +43,7 @@ migration PR removes or replaces each item. | `GLOBAL_Endpoints`, `GLOBAL_IsErasure`, `GLOBAL_IsDistErasure`, `GLOBAL_IsErasureSD`, `GLOBAL_RootDiskThreshold` | `crates/ecstore/src/runtime/global.rs` and `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Endpoint, setup-type, and root-disk-threshold access now stays behind ECStore runtime helpers; move endpoint ownership only after readiness and quorum behavior have explicit coverage. | | `GLOBAL_LOCAL_DISK_MAP`, `GLOBAL_LOCAL_DISK_ID_MAP`, `GLOBAL_LOCAL_DISK_SET_DRIVES` | `crates/ecstore/src/runtime/global.rs` and `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Local disk map, disk-id cache, and set-drive access now stay behind ECStore runtime-source helpers instead of direct global access; preserve disk lookup, remote/local classification, and test reset hooks in later ownership changes. | | `GLOBAL_ExpiryState`, `GLOBAL_TransitionState`, `GLOBAL_LifecycleSys` | `crates/ecstore/src/bucket/lifecycle/*`, `crates/ecstore/src/runtime/global.rs`, and `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | `GLOBAL_LifecycleSys` access now stays behind ECStore runtime-source helpers; `GLOBAL_ExpiryState` remains the first code-bearing ownership candidate because AppContext already has an `ExpiryStateInterface`, resolver, and tests. | -| `GLOBAL_REPLICATION_POOL`, `GLOBAL_REPLICATION_STATS`, `GLOBAL_BUCKET_MONITOR` | `crates/ecstore/src/bucket/replication/*`, `crates/ecstore/src/runtime/global.rs` | Runtime migration target | Keep behind AppContext replication and bucket-monitor resolvers until queue admission, stats, and admin reporting coverage are explicit. | +| `GLOBAL_REPLICATION_POOL`, `GLOBAL_REPLICATION_STATS`, `GLOBAL_BUCKET_MONITOR` | `crates/ecstore/src/bucket/replication/*`, `crates/ecstore/src/runtime/global.rs` | Runtime migration target | Replication pool/stat access now stays behind replication owner and ECStore runtime-source helpers; keep bucket-monitor access behind AppContext and runtime-source resolvers until queue admission, stats, and admin reporting coverage are explicit. | | `GLOBAL_TierConfigMgr`, `GLOBAL_STORAGE_CLASS`, `GLOBAL_CONFIG_SYS`, `GLOBAL_SERVER_CONFIG` | `crates/ecstore/src/config`, `crates/config`, `rustfs/src/app/context/runtime_sources.rs` | Runtime migration target | Tier config manager reads and reloads now use the ECStore runtime-source helper; move remaining config state through config/runtime-source owners only, without combining storage-class behavior or persistence changes. | | `GLOBAL_EventNotifier`, `GLOBAL_NotificationSys` | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs`, and `crates/ecstore/src/services/*` | Runtime migration target | `GLOBAL_EventNotifier` access now stays behind ECStore runtime-source helpers; move remaining notification ownership only through notify/runtime-source boundaries. | | `GLOBAL_BOOT_TIME`, `globalDeploymentIDPtr`, `GLOBAL_REGION`, `GLOBAL_RUSTFS_PORT`, `GLOBAL_LocalNodeName`, `GLOBAL_LocalNodeNameHex` | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Boot time and ECStore local-node-name fallback reads now stay behind the ECStore runtime-source API; the remaining scalar handles may migrate in small PRs after call sites use AppContext or owner-local runtime-source readers. | diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 39e375891..d63d0cea8 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -567,7 +567,7 @@ pub(crate) fn get_global_replication_pool() -> Option> { } pub(crate) fn get_global_replication_stats() -> Option> { - ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().cloned() + ecstore_bucket::replication::get_global_replication_stats() } pub(crate) fn get_global_boot_time() -> Option { @@ -619,7 +619,7 @@ pub(crate) async fn prewarm_local_disk_id_map() { } pub(crate) fn replication_queue_current_count() -> Option { - ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| { + get_global_replication_stats().and_then(|stats| { stats .q_cache .try_lock() @@ -1046,7 +1046,7 @@ pub(crate) fn check_retention_for_modification( } pub(crate) async fn record_replication_proxy(bucket: &str, api: &str, is_err: bool) { - if let Some(stats) = ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() { + if let Some(stats) = get_global_replication_stats() { stats.inc_proxy(bucket, api, is_err).await; } } diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index ac12d3941..d37c42f9c 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -182,6 +182,7 @@ EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_ FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt" EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE="${TMP_DIR}/external_ecstore_api_boundary_hits.txt" REPLICATION_FACADE_BYPASS_HITS_FILE="${TMP_DIR}/replication_facade_bypass_hits.txt" +GLOBAL_REPLICATION_STATE_BYPASS_HITS_FILE="${TMP_DIR}/global_replication_state_bypass_hits.txt" GLOBAL_ENDPOINTS_BYPASS_HITS_FILE="${TMP_DIR}/global_endpoints_bypass_hits.txt" GLOBAL_IS_ERASURE_BYPASS_HITS_FILE="${TMP_DIR}/global_is_erasure_bypass_hits.txt" GLOBAL_IS_DIST_ERASURE_BYPASS_HITS_FILE="${TMP_DIR}/global_is_dist_erasure_bypass_hits.txt" @@ -2308,6 +2309,18 @@ if [[ -s "$REPLICATION_FACADE_BYPASS_HITS_FILE" ]]; then report_failure "replication facade imports must stay in local storage_api boundaries: $(paste -sd '; ' "$REPLICATION_FACADE_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename '\bGLOBAL_REPLICATION_(POOL|STATS)\b' \ + crates rustfs fuzz \ + --glob '*.rs' | + rg -v '^crates/ecstore/src/(bucket/replication/replication_pool|runtime/sources)\.rs:' || true +) >"$GLOBAL_REPLICATION_STATE_BYPASS_HITS_FILE" + +if [[ -s "$GLOBAL_REPLICATION_STATE_BYPASS_HITS_FILE" ]]; then + report_failure "GLOBAL_REPLICATION_POOL/STATS access must stay behind replication owner or ECStore runtime-source helpers: $(paste -sd '; ' "$GLOBAL_REPLICATION_STATE_BYPASS_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename '\bGLOBAL_Endpoints\b' \