From 1a8ff1ea3e445981bbe87a9d5c8d1fcb56db5633 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 24 Jun 2026 10:41:33 +0800 Subject: [PATCH] refactor: centralize ecstore setup runtime sources (#3812) --- crates/ecstore/src/layout/pool_space.rs | 5 ++-- crates/ecstore/src/rpc/peer_rest_client.rs | 4 +-- crates/ecstore/src/runtime_sources.rs | 21 ++++++++++++-- crates/ecstore/src/set_disk.rs | 5 ++-- crates/ecstore/src/set_disk/lock.rs | 9 ++++-- crates/ecstore/src/sets.rs | 12 +++----- crates/ecstore/src/store.rs | 7 ++--- crates/ecstore/src/store/init.rs | 9 +++--- docs/architecture/migration-progress.md | 33 ++++++++++++++++++---- 9 files changed, 71 insertions(+), 34 deletions(-) diff --git a/crates/ecstore/src/layout/pool_space.rs b/crates/ecstore/src/layout/pool_space.rs index 37d297a6b..e7806f8cb 100644 --- a/crates/ecstore/src/layout/pool_space.rs +++ b/crates/ecstore/src/layout/pool_space.rs @@ -17,7 +17,8 @@ use std::slice::Iter; use crate::bucket::utils::is_meta_bucketname; use crate::disk::DiskInfo; use crate::error::{Error, Result}; -use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, is_erasure_sd}; +use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES}; +use crate::runtime_sources; #[derive(Debug, Default, Clone)] pub struct PoolAvailableSpace { @@ -97,7 +98,7 @@ pub async fn has_space_for(dis: &[Option], size: i64) -> Result let per_disk = size / disks_num as u64; for disk in dis.iter().flatten() { - if !is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 { + if !runtime_sources::setup_is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 { return Ok(false); } diff --git a/crates/ecstore/src/rpc/peer_rest_client.rs b/crates/ecstore/src/rpc/peer_rest_client.rs index 9173e7f50..8d42f4eb8 100644 --- a/crates/ecstore/src/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/rpc/peer_rest_client.rs @@ -17,8 +17,8 @@ use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node use crate::{ disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout}, endpoints::EndpointServerPools, - global::is_dist_erasure, metrics_realtime::{CollectMetricsOpts, MetricType}, + runtime_sources, }; use rmp_serde::{Deserializer, Serializer}; use rustfs_madmin::{ @@ -98,7 +98,7 @@ impl PeerRestClient { } } pub async fn new_clients(eps: EndpointServerPools) -> (Vec>, Vec>) { - if !is_dist_erasure().await { + if !runtime_sources::setup_is_dist_erasure().await { return (Vec::new(), Vec::new()); } diff --git a/crates/ecstore/src/runtime_sources.rs b/crates/ecstore/src/runtime_sources.rs index dd4392e27..4ffa8cf68 100644 --- a/crates/ecstore/src/runtime_sources.rs +++ b/crates/ecstore/src/runtime_sources.rs @@ -28,8 +28,8 @@ use crate::{ GLOBAL_BOOT_TIME, GLOBAL_EventNotifier, GLOBAL_IsErasureSD, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LifecycleSys, GLOBAL_LocalNodeName, GLOBAL_RootDiskThreshold, GLOBAL_TierConfigMgr, TypeLocalDiskSetDrives, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints, - get_global_endpoints_opt, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_first_cluster_node_local, - resolve_object_store_handle, set_global_deployment_id, + get_global_endpoints_opt, get_global_lock_clients, init_global_bucket_monitor, is_dist_erasure, is_erasure, + is_first_cluster_node_local, resolve_object_store_handle, set_global_deployment_id, }, notification_sys::{NotificationSys, get_global_notification_sys}, store::ECStore, @@ -38,6 +38,7 @@ use crate::{ use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service}; +use rustfs_lock::client::LockClient; use s3s::dto::BucketLifecycleConfiguration; use tokio::sync::RwLock; use tonic::transport::Channel; @@ -59,6 +60,10 @@ pub(crate) fn endpoint_pools() -> Option { get_global_endpoints_opt() } +pub(crate) fn endpoint_pools_or_default() -> EndpointServerPools { + get_global_endpoints() +} + pub(crate) fn endpoint_erasure_set_count() -> Option { endpoint_pools().map(|endpoints| endpoints.es_count()) } @@ -82,6 +87,10 @@ pub(crate) async fn setup_is_dist_erasure() -> bool { is_dist_erasure().await } +pub(crate) async fn setup_is_erasure_sd() -> bool { + *GLOBAL_IsErasureSD.read().await +} + pub(crate) async fn local_node_name() -> String { GLOBAL_LOCAL_NODE_NAME.read().await.clone() } @@ -102,6 +111,10 @@ pub(crate) fn boot_uptime_secs() -> u64 { .as_secs() } +pub(crate) async fn ensure_boot_time() { + GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await; +} + pub(crate) async fn scanner_init_time() -> Option> { rustfs_common::get_global_init_time().await } @@ -169,6 +182,10 @@ pub(crate) fn global_lock_manager() -> Arc { rustfs_lock::get_global_lock_manager() } +pub(crate) fn global_lock_clients() -> Option<&'static HashMap>> { + get_global_lock_clients() +} + pub(crate) fn notification_sys() -> Option<&'static NotificationSys> { get_global_notification_sys() } diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 4ce751166..143e5914a 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -53,7 +53,6 @@ use crate::{ error::{StorageError, to_object_err}, // event::name::EventName, event_notification::{EventArgs, send_event}, - global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure}, object_api::{GetObjectReader, ObjectInfo, PutObjReader}, store_init::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file}, }; @@ -1767,7 +1766,7 @@ impl rustfs_storage_api::NamespaceLocking for SetDisks { #[tracing::instrument(skip(self))] async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result { - let set_lock = if is_dist_erasure().await { + let set_lock = if runtime_sources::setup_is_dist_erasure().await { // Calculate quorum based on lockers count (majority) let lockers_count = self.lockers.len(); let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 }; @@ -2250,7 +2249,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks { let mut _local_batch_guards: Vec = Vec::with_capacity(batch.requests.len()); let mut locked_objects = HashSet::new(); - let dist_erasure = is_dist_erasure().await; + let dist_erasure = runtime_sources::setup_is_dist_erasure().await; let mut dist_batch_lock_ids = vec![Vec::new(); self.lockers.len()]; if dist_erasure { diff --git a/crates/ecstore/src/set_disk/lock.rs b/crates/ecstore/src/set_disk/lock.rs index 0c0e0deff..e6200b9fa 100644 --- a/crates/ecstore/src/set_disk/lock.rs +++ b/crates/ecstore/src/set_disk/lock.rs @@ -14,6 +14,7 @@ use super::*; use crate::disk::health_state::DriveMembershipSnapshot; +use crate::runtime_sources; impl SetDisks { pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String { @@ -303,12 +304,14 @@ impl SetDisks { new_disk.enable_health_check(); if new_disk.is_local() { - let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await; + let local_disk_map = runtime_sources::local_disk_map_handle(); + let mut global_local_disk_map = local_disk_map.write().await; let path = new_disk.endpoint().to_string(); global_local_disk_map.insert(path, Some(new_disk.clone())); - if is_dist_erasure().await { - let mut local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await; + if runtime_sources::setup_is_dist_erasure().await { + let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle(); + let mut local_set_drives = local_disk_set_drives.write().await; local_set_drives[self.pool_index][set_idx][disk_idx] = Some(new_disk.clone()); } } diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 03b96a92a..78c3dd898 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -25,7 +25,6 @@ use crate::{ }, endpoints::{Endpoints, PoolEndpoints}, error::StorageError, - global::{get_global_lock_clients, is_dist_erasure}, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, runtime_sources, set_disk::SetDisks, @@ -37,10 +36,7 @@ use futures::{ }; use http::HeaderMap; use rustfs_common::heal_channel::HealOpts; -use rustfs_common::{ - GLOBAL_LOCAL_NODE_NAME, - heal_channel::{DriveState, HealItemType}, -}; +use rustfs_common::heal_channel::{DriveState, HealItemType}; use rustfs_filemeta::FileInfo; use rustfs_lock::NamespaceLockWrapper; use rustfs_lock::client::LockClient; @@ -109,7 +105,7 @@ impl Sets { let mut disk_set = Vec::with_capacity(set_count); // Get lock clients from global storage - let lock_clients = get_global_lock_clients(); + let lock_clients = runtime_sources::global_lock_clients(); for i in 0..set_count { let mut set_drive = Vec::with_capacity(set_drive_count); @@ -138,7 +134,7 @@ impl Sets { continue; } - if disk.as_ref().unwrap().is_local() && is_dist_erasure().await { + if disk.as_ref().unwrap().is_local() && runtime_sources::setup_is_dist_erasure().await { let local_disk = runtime_sources::local_disk_set_drive(pool_idx, i, j).await; if local_disk.is_none() { @@ -172,7 +168,7 @@ impl Sets { let lockers = set_lock_clients.values().cloned().collect::>>(); let set_disks = SetDisks::new( - GLOBAL_LOCAL_NODE_NAME.read().await.to_string(), + runtime_sources::local_node_name().await, Arc::new(RwLock::new(set_drive)), set_drive_count, parity_count, diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 9a616d9df..ad7cbbed2 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -44,13 +44,13 @@ use crate::error::{ }; use crate::event_notification::EventNotifier; use crate::global::{ - DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, TypeLocalDiskSetDrives, get_global_endpoints, get_global_region, - get_global_tier_config_mgr, set_object_layer, + DISK_RESERVE_FRACTION, TypeLocalDiskSetDrives, get_global_region, get_global_tier_config_mgr, set_object_layer, }; use crate::notification_sys::get_global_notification_sys; use crate::pools::PoolMeta; use crate::rebalance::RebalanceMeta; use crate::rpc::RemoteClient; +use crate::runtime_sources; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::tier::tier::TierConfigMgr; use crate::{ @@ -84,7 +84,6 @@ use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; use std::net::SocketAddr; use std::process::exit; -use std::time::SystemTime; use std::{ collections::HashMap, sync::{Arc, OnceLock}, @@ -265,7 +264,7 @@ impl ECStore { /// Get the global endpoints pub fn endpoints(&self) -> EndpointServerPools { - get_global_endpoints() + runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into()) } /// Get the global region diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index b9319557e..a97302336 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -14,7 +14,6 @@ use super::*; use crate::error::is_err_decommission_running; -use crate::global::{is_dist_erasure, is_first_cluster_node_local}; use crate::pools::local_decommission_queue_prefix; use crate::runtime_sources; use tracing::{debug, error, info, warn}; @@ -299,7 +298,7 @@ impl ECStore { } // Replace the local disk - if !is_dist_erasure().await { + if !runtime_sources::setup_is_dist_erasure().await { runtime_sources::record_local_disks(local_disks).await; } @@ -363,7 +362,7 @@ impl ECStore { #[instrument(level = "debug", skip(self, rx))] pub async fn init(self: &Arc, rx: CancellationToken) -> Result<()> { - GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await; + runtime_sources::ensure_boot_time().await; let mut meta = PoolMeta::default(); resolve_store_init_stage_result( @@ -378,8 +377,8 @@ impl ECStore { "load_pool_meta", )?; let update = meta.validate(self.pools.clone())?; - let endpoints = get_global_endpoints(); - let should_persist_pool_meta = is_first_cluster_node_local().await; + let endpoints = runtime_sources::endpoint_pools_or_default(); + let should_persist_pool_meta = runtime_sources::first_cluster_node_is_local().await; let installed_pool_meta = if !update { meta.clone() diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index e01027fd1..21ba5cc78 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-ecstore-owner-runtime-sources-batch` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194`. -- Based on: stacked on API-193 PR branch while PR #3810 is pending. +- Branch: `overtrue/arch-ecstore-setup-runtime-sources-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195`. +- Based on: stacked on API-194 PR branch while PR #3811 is pending. - PR type for this branch: `consumer-migration` - Runtime behavior changes: none. - Rust code changes: route replication pool, outbound TLS generation, runtime @@ -26,7 +26,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block config/tier first-node checks, rebalance endpoint-locality checks, bucket metadata object-store reads, metadata endpoint/setup reads, lifecycle object-store reads, replication object-store readiness checks, and ECStore - pools/tier/notification owner-root runtime source reads, + pools/tier/notification owner-root runtime source reads, plus ECStore setup + state, boot-time, endpoint snapshot, local node, local disk map, and lock + client reads in store initialization, sets, set-disk, pool-space, and peer + client paths, through AppContext-first or owner-crate resolver boundaries. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup @@ -36,7 +39,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block and storage owner thin bridge regressions, plus app context and notify event-bridge thin module regressions; accept the reviewed AppContext resolver reverse dependencies in the layer baseline. -- Docs changes: record the API-136 through API-194 owner facade cleanup. +- Docs changes: record the API-136 through API-195 owner facade cleanup. ## Phase 0 Tasks @@ -4827,6 +4830,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block migration/layer guards, PR-before-push quality gate, and three-expert review. +- [x] `API-195` Centralize ECStore setup runtime source reads. + - Do: route ECStore setup-state checks, boot-time initialization, endpoint + snapshots, local node names, lock-client lookup, local disk map updates, + and local disk set-drive updates through the ECStore-owned runtime-source + module. + - Acceptance: store initialization, sets, set-disk, pool-space, and peer REST + client code no longer import those runtime globals directly outside the + runtime-source boundary. + - Must preserve: store boot-time initialization, local disk registration, + distributed lock selection, delete-object batch locking, recovered-disk + local map updates, pool-space inode checks, peer-client creation, and + pool-meta persistence behavior. + - Verification: ECStore compile coverage, focused setup/set-disk lock tests, + formatting, diff hygiene, residual runtime-source scan, Rust risk scan, + migration/layer guards, PR-before-push quality gate, and three-expert + review. + ## Next PRs 1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries. @@ -4835,6 +4855,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-195 keeps setup-state, boot-time, endpoint, local-node, local-disk, and lock-client reads behind the ECStore runtime-source boundary. | +| Migration preservation | pass | Store init, local disk registration, distributed lock selection, recovered-disk map updates, pool-space inode checks, peer-client creation, and pool-meta persistence preserve existing behavior. | +| Testing/verification | pass | ECStore compile, focused setup/set-disk tests, formatting, migration/layer guards, residual scan, Rust risk scan, and PR gate are planned for API-195. | | Quality/architecture | pass | API-194 keeps pools, tier, and notification owner-root runtime reads behind the ECStore runtime-source boundary. | | Migration preservation | pass | Decommission, tier-save, notification rebalance, peer fallback, and offline uptime behavior preserve existing error/fallback semantics. | | Testing/verification | pass | ECStore compile, focused owner-root tests, formatting, migration/layer guards, residual scan, Rust risk scan, and PR gate are planned for API-194. |