gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-27 00:38:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: centralize ecstore setup runtime sources (#3812)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-24 10:41:33 +08:00
committed by GitHub
parent 55b402ec2e
commit 1a8ff1ea3e
9 changed files with 71 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/layout/pool_space.rs
+3 -2
View File
@@ -17,7 +17,8 @@ use std::slice::Iter;
use crate::bucket::utils::is_meta_bucketname;
use crate::disk::DiskInfo;
use crate::error::{Error, Result};
use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, is_erasure_sd};
use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES};
use crate::runtime_sources;
#[derive(Debug, Default, Clone)]
pub struct PoolAvailableSpace {
@@ -97,7 +98,7 @@ pub async fn has_space_for(dis: &[Option<DiskInfo>], size: i64) -> Result<bool>
let per_disk = size / disks_num as u64;
for disk in dis.iter().flatten() {
if !is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 {
if !runtime_sources::setup_is_erasure_sd().await && disk.free_inodes < DISK_MIN_INODES && disk.used_inodes > 0 {
return Ok(false);
}
crates/ecstore/src/rpc/peer_rest_client.rs
+2 -2
View File
@@ -17,8 +17,8 @@ use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node
use crate::{
disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
endpoints::EndpointServerPools,
global::is_dist_erasure,
metrics_realtime::{CollectMetricsOpts, MetricType},
runtime_sources,
};
use rmp_serde::{Deserializer, Serializer};
use rustfs_madmin::{
@@ -98,7 +98,7 @@ impl PeerRestClient {
}
}
pub async fn new_clients(eps: EndpointServerPools) -> (Vec<Option<Self>>, Vec<Option<Self>>) {
if !is_dist_erasure().await {
if !runtime_sources::setup_is_dist_erasure().await {
return (Vec::new(), Vec::new());
}
crates/ecstore/src/runtime_sources.rs
+19 -2
View File
@@ -28,8 +28,8 @@ use crate::{
GLOBAL_BOOT_TIME, GLOBAL_EventNotifier, GLOBAL_IsErasureSD, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP,
GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LifecycleSys, GLOBAL_LocalNodeName, GLOBAL_RootDiskThreshold, GLOBAL_TierConfigMgr,
TypeLocalDiskSetDrives, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints,
get_global_endpoints_opt, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_first_cluster_node_local,
resolve_object_store_handle, set_global_deployment_id,
get_global_endpoints_opt, get_global_lock_clients, init_global_bucket_monitor, is_dist_erasure, is_erasure,
is_first_cluster_node_local, resolve_object_store_handle, set_global_deployment_id,
},
notification_sys::{NotificationSys, get_global_notification_sys},
store::ECStore,
@@ -38,6 +38,7 @@ use crate::{
use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
use rustfs_lock::client::LockClient;
use s3s::dto::BucketLifecycleConfiguration;
use tokio::sync::RwLock;
use tonic::transport::Channel;
@@ -59,6 +60,10 @@ pub(crate) fn endpoint_pools() -> Option<EndpointServerPools> {
get_global_endpoints_opt()
}
pub(crate) fn endpoint_pools_or_default() -> EndpointServerPools {
get_global_endpoints()
}
pub(crate) fn endpoint_erasure_set_count() -> Option<usize> {
endpoint_pools().map(|endpoints| endpoints.es_count())
}
@@ -82,6 +87,10 @@ pub(crate) async fn setup_is_dist_erasure() -> bool {
is_dist_erasure().await
}
pub(crate) async fn setup_is_erasure_sd() -> bool {
*GLOBAL_IsErasureSD.read().await
}
pub(crate) async fn local_node_name() -> String {
GLOBAL_LOCAL_NODE_NAME.read().await.clone()
}
@@ -102,6 +111,10 @@ pub(crate) fn boot_uptime_secs() -> u64 {
.as_secs()
}
pub(crate) async fn ensure_boot_time() {
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
}
pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
rustfs_common::get_global_init_time().await
}
@@ -169,6 +182,10 @@ pub(crate) fn global_lock_manager() -> Arc<rustfs_lock::GlobalLockManager> {
rustfs_lock::get_global_lock_manager()
}
pub(crate) fn global_lock_clients() -> Option<&'static HashMap<String, Arc<dyn LockClient>>> {
get_global_lock_clients()
}
pub(crate) fn notification_sys() -> Option<&'static NotificationSys> {
get_global_notification_sys()
}
crates/ecstore/src/set_disk.rs
+2 -3
View File
@@ -53,7 +53,6 @@ use crate::{
error::{StorageError, to_object_err},
// event::name::EventName,
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure},
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
store_init::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file},
};
@@ -1767,7 +1766,7 @@ impl rustfs_storage_api::NamespaceLocking for SetDisks {
#[tracing::instrument(skip(self))]
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
let set_lock = if is_dist_erasure().await {
let set_lock = if runtime_sources::setup_is_dist_erasure().await {
// Calculate quorum based on lockers count (majority)
let lockers_count = self.lockers.len();
let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 };
@@ -2250,7 +2249,7 @@ impl rustfs_storage_api::ObjectOperations for SetDisks {
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
let mut locked_objects = HashSet::new();
let dist_erasure = is_dist_erasure().await;
let dist_erasure = runtime_sources::setup_is_dist_erasure().await;
let mut dist_batch_lock_ids = vec![Vec::new(); self.lockers.len()];
if dist_erasure {
crates/ecstore/src/set_disk/lock.rs
+6 -3
View File
@@ -14,6 +14,7 @@
use super::*;
use crate::disk::health_state::DriveMembershipSnapshot;
use crate::runtime_sources;
impl SetDisks {
pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
@@ -303,12 +304,14 @@ impl SetDisks {
new_disk.enable_health_check();
if new_disk.is_local() {
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
let local_disk_map = runtime_sources::local_disk_map_handle();
let mut global_local_disk_map = local_disk_map.write().await;
let path = new_disk.endpoint().to_string();
global_local_disk_map.insert(path, Some(new_disk.clone()));
if is_dist_erasure().await {
let mut local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await;
if runtime_sources::setup_is_dist_erasure().await {
let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle();
let mut local_set_drives = local_disk_set_drives.write().await;
local_set_drives[self.pool_index][set_idx][disk_idx] = Some(new_disk.clone());
}
}
crates/ecstore/src/sets.rs
+4 -8
View File
@@ -25,7 +25,6 @@ use crate::{
},
endpoints::{Endpoints, PoolEndpoints},
error::StorageError,
global::{get_global_lock_clients, is_dist_erasure},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
runtime_sources,
set_disk::SetDisks,
@@ -37,10 +36,7 @@ use futures::{
};
use http::HeaderMap;
use rustfs_common::heal_channel::HealOpts;
use rustfs_common::{
GLOBAL_LOCAL_NODE_NAME,
heal_channel::{DriveState, HealItemType},
};
use rustfs_common::heal_channel::{DriveState, HealItemType};
use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_lock::client::LockClient;
@@ -109,7 +105,7 @@ impl Sets {
let mut disk_set = Vec::with_capacity(set_count);
// Get lock clients from global storage
let lock_clients = get_global_lock_clients();
let lock_clients = runtime_sources::global_lock_clients();
for i in 0..set_count {
let mut set_drive = Vec::with_capacity(set_drive_count);
@@ -138,7 +134,7 @@ impl Sets {
continue;
}
if disk.as_ref().unwrap().is_local() && is_dist_erasure().await {
if disk.as_ref().unwrap().is_local() && runtime_sources::setup_is_dist_erasure().await {
let local_disk = runtime_sources::local_disk_set_drive(pool_idx, i, j).await;
if local_disk.is_none() {
@@ -172,7 +168,7 @@ impl Sets {
let lockers = set_lock_clients.values().cloned().collect::<Vec<Arc<dyn LockClient>>>();
let set_disks = SetDisks::new(
GLOBAL_LOCAL_NODE_NAME.read().await.to_string(),
runtime_sources::local_node_name().await,
Arc::new(RwLock::new(set_drive)),
set_drive_count,
parity_count,
crates/ecstore/src/store.rs
+3 -4
View File
@@ -44,13 +44,13 @@ use crate::error::{
};
use crate::event_notification::EventNotifier;
use crate::global::{
DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME, TypeLocalDiskSetDrives, get_global_endpoints, get_global_region,
get_global_tier_config_mgr, set_object_layer,
DISK_RESERVE_FRACTION, TypeLocalDiskSetDrives, get_global_region, get_global_tier_config_mgr, set_object_layer,
};
use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::runtime_sources;
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
@@ -84,7 +84,6 @@ use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::net::SocketAddr;
use std::process::exit;
use std::time::SystemTime;
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
@@ -265,7 +264,7 @@ impl ECStore {
/// Get the global endpoints
pub fn endpoints(&self) -> EndpointServerPools {
get_global_endpoints()
runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
}
/// Get the global region
crates/ecstore/src/store/init.rs
+4 -5
View File
@@ -14,7 +14,6 @@
use super::*;
use crate::error::is_err_decommission_running;
use crate::global::{is_dist_erasure, is_first_cluster_node_local};
use crate::pools::local_decommission_queue_prefix;
use crate::runtime_sources;
use tracing::{debug, error, info, warn};
@@ -299,7 +298,7 @@ impl ECStore {
}
// Replace the local disk
if !is_dist_erasure().await {
if !runtime_sources::setup_is_dist_erasure().await {
runtime_sources::record_local_disks(local_disks).await;
}
@@ -363,7 +362,7 @@ impl ECStore {
#[instrument(level = "debug", skip(self, rx))]
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
runtime_sources::ensure_boot_time().await;
let mut meta = PoolMeta::default();
resolve_store_init_stage_result(
@@ -378,8 +377,8 @@ impl ECStore {
"load_pool_meta",
)?;
let update = meta.validate(self.pools.clone())?;
let endpoints = get_global_endpoints();
let should_persist_pool_meta = is_first_cluster_node_local().await;
let endpoints = runtime_sources::endpoint_pools_or_default();
let should_persist_pool_meta = runtime_sources::first_cluster_node_is_local().await;
let installed_pool_meta = if !update {
meta.clone()
docs/architecture/migration-progress.md
+28 -5
View File
@@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-ecstore-owner-runtime-sources-batch`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194`.
- Based on: stacked on API-193 PR branch while PR #3810 is pending.
- Branch: `overtrue/arch-ecstore-setup-runtime-sources-batch`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195`.
- Based on: stacked on API-194 PR branch while PR #3811 is pending.
- PR type for this branch: `consumer-migration`
- Runtime behavior changes: none.
- Rust code changes: route replication pool, outbound TLS generation, runtime
@@ -26,7 +26,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
config/tier first-node checks, rebalance endpoint-locality checks, bucket
metadata object-store reads, metadata endpoint/setup reads, lifecycle
object-store reads, replication object-store readiness checks, and ECStore
pools/tier/notification owner-root runtime source reads,
pools/tier/notification owner-root runtime source reads, plus ECStore setup
state, boot-time, endpoint snapshot, local node, local disk map, and lock
client reads in store initialization, sets, set-disk, pool-space, and peer
client paths,
through AppContext-first or owner-crate resolver boundaries.
- CI/script changes: lock completed owner and test/fuzz boundaries against
bare/glob imports, scattered raw ECStore facade subpaths, and startup
@@ -36,7 +39,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
and storage owner thin bridge regressions, plus app context and notify
event-bridge thin module regressions; accept the reviewed AppContext resolver
reverse dependencies in the layer baseline.
- Docs changes: record the API-136 through API-194 owner facade cleanup.
- Docs changes: record the API-136 through API-195 owner facade cleanup.
## Phase 0 Tasks
@@ -4827,6 +4830,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
migration/layer guards, PR-before-push quality gate, and three-expert
review.
- [x] `API-195` Centralize ECStore setup runtime source reads.
- Do: route ECStore setup-state checks, boot-time initialization, endpoint
snapshots, local node names, lock-client lookup, local disk map updates,
and local disk set-drive updates through the ECStore-owned runtime-source
module.
- Acceptance: store initialization, sets, set-disk, pool-space, and peer REST
client code no longer import those runtime globals directly outside the
runtime-source boundary.
- Must preserve: store boot-time initialization, local disk registration,
distributed lock selection, delete-object batch locking, recovered-disk
local map updates, pool-space inode checks, peer-client creation, and
pool-meta persistence behavior.
- Verification: ECStore compile coverage, focused setup/set-disk lock tests,
formatting, diff hygiene, residual runtime-source scan, Rust risk scan,
migration/layer guards, PR-before-push quality gate, and three-expert
review.
## Next PRs
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
@@ -4835,6 +4855,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | API-195 keeps setup-state, boot-time, endpoint, local-node, local-disk, and lock-client reads behind the ECStore runtime-source boundary. |
| Migration preservation | pass | Store init, local disk registration, distributed lock selection, recovered-disk map updates, pool-space inode checks, peer-client creation, and pool-meta persistence preserve existing behavior. |
| Testing/verification | pass | ECStore compile, focused setup/set-disk tests, formatting, migration/layer guards, residual scan, Rust risk scan, and PR gate are planned for API-195. |
| Quality/architecture | pass | API-194 keeps pools, tier, and notification owner-root runtime reads behind the ECStore runtime-source boundary. |
| Migration preservation | pass | Decommission, tier-save, notification rebalance, peer fallback, and offline uptime behavior preserve existing error/fallback semantics. |
| Testing/verification | pass | ECStore compile, focused owner-root tests, formatting, migration/layer guards, residual scan, Rust risk scan, and PR gate are planned for API-194. |