From 5b6c486b13ff5a9a1bb1f6fbeadbbc12b4d6fac2 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 24 Jun 2026 08:42:36 +0800 Subject: [PATCH] refactor: centralize ecstore locality runtime sources (#3808) --- crates/ecstore/src/config/com.rs | 4 ++-- crates/ecstore/src/rebalance/runtime.rs | 8 ++----- crates/ecstore/src/runtime_sources.rs | 14 ++++++++++++- crates/ecstore/src/tier/tier.rs | 4 ++-- docs/architecture/migration-progress.md | 28 ++++++++++++++++++++----- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 2851c196e..9dca16c22 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -15,8 +15,8 @@ use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass}; use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; -use crate::global::is_first_cluster_node_local; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; +use crate::runtime_sources; use crate::storage_api_contracts::EcstoreObjectIO; use http::HeaderMap; use rustfs_config::audit::{ @@ -1150,7 +1150,7 @@ where S: EcstoreObjectIO + StorageAdminApi, { warn!("Configuration not found ({}): Start initializing new configuration", context); - let cfg = if is_first_cluster_node_local().await { + let cfg = if runtime_sources::first_cluster_node_is_local().await { new_and_save_server_config(api.clone()).await? } else { let mut cfg = new_server_config(); diff --git a/crates/ecstore/src/rebalance/runtime.rs b/crates/ecstore/src/rebalance/runtime.rs index 78f91bf22..5dbd6b511 100644 --- a/crates/ecstore/src/rebalance/runtime.rs +++ b/crates/ecstore/src/rebalance/runtime.rs @@ -14,7 +14,7 @@ use super::{ RebalanceBucketOutcome, }; use crate::error::{Error, Result}; -use crate::global::get_global_endpoints; +use crate::runtime_sources; use crate::store::ECStore; use std::collections::HashSet; use std::sync::Arc; @@ -120,11 +120,7 @@ impl ECStore { continue; } - if !get_global_endpoints() - .as_ref() - .get(idx) - .is_some_and(|v| v.endpoints.as_ref().first().is_some_and(|e| e.is_local)) - { + if !runtime_sources::endpoint_pool_is_local(idx) { debug!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, diff --git a/crates/ecstore/src/runtime_sources.rs b/crates/ecstore/src/runtime_sources.rs index 4821d363e..31ebb6525 100644 --- a/crates/ecstore/src/runtime_sources.rs +++ b/crates/ecstore/src/runtime_sources.rs @@ -28,7 +28,8 @@ use crate::{ GLOBAL_BOOT_TIME, GLOBAL_EventNotifier, GLOBAL_IsErasureSD, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LifecycleSys, GLOBAL_LocalNodeName, GLOBAL_RootDiskThreshold, GLOBAL_TierConfigMgr, TypeLocalDiskSetDrives, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints, - get_global_endpoints_opt, init_global_bucket_monitor, resolve_object_store_handle, set_global_deployment_id, + get_global_endpoints_opt, init_global_bucket_monitor, is_first_cluster_node_local, resolve_object_store_handle, + set_global_deployment_id, }, notification_sys::{NotificationSys, get_global_notification_sys}, store::ECStore, @@ -58,6 +59,17 @@ pub(crate) fn endpoint_pools() -> Option { get_global_endpoints_opt() } +pub(crate) fn endpoint_pool_is_local(pool_index: usize) -> bool { + get_global_endpoints() + .as_ref() + .get(pool_index) + .is_some_and(|pool| pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)) +} + +pub(crate) async fn first_cluster_node_is_local() -> bool { + is_first_cluster_node_local().await +} + pub(crate) async fn local_node_name() -> String { GLOBAL_LOCAL_NODE_NAME.read().await.clone() } diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs index 51694f3ba..8ada33bb5 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/tier/tier.rs @@ -48,8 +48,8 @@ use crate::tier::{ use crate::{ config::com::{CONFIG_PREFIX, read_config}, disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, - global::is_first_cluster_node_local, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, + runtime_sources, storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations}, store::ECStore, }; @@ -1171,7 +1171,7 @@ async fn load_tier_config(api: Arc) -> std::result::Result { warn!("config not found, start to init"); - if is_first_cluster_node_local().await { + if runtime_sources::first_cluster_node_is_local().await { new_and_save_tiering_config(api).await.map_err(io::Error::other) } else { Ok(TierConfigMgr { diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index aa360f428..1fc38b1a9 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-ecstore-bucket-runtime-sources-batch` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191`. -- Based on: latest default branch after merge 3805. +- Branch: `overtrue/arch-owner-root-facade-source-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192`. +- Based on: latest default branch after merge 3806. - PR type for this branch: `consumer-migration` - Runtime behavior changes: none. - Rust code changes: route replication pool, outbound TLS generation, runtime @@ -22,7 +22,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block boot-time, init-time, root-disk threshold, and cached RPC channel reads, plus ECStore replication pool, replication stats, and event-host reads, plus ECStore lifecycle queue state, tier config, lifecycle config, deployment - id, event-host reads, bucket monitor reads, and replication worker pool reads, + id, event-host reads, bucket monitor reads, replication worker pool reads, + config/tier first-node checks, and rebalance endpoint-locality checks, through AppContext-first or owner-crate resolver boundaries. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup @@ -32,7 +33,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block and storage owner thin bridge regressions, plus app context and notify event-bridge thin module regressions; accept the reviewed AppContext resolver reverse dependencies in the layer baseline. -- Docs changes: record the API-136 through API-191 owner facade cleanup. +- Docs changes: record the API-136 through API-192 owner facade cleanup. ## Phase 0 Tasks @@ -4779,6 +4780,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block scan, migration/layer guards, PR-before-push pre-commit quality gate, and three-expert review. +- [x] `API-192` Centralize ECStore locality runtime source reads. + - Do: route rebalance endpoint-locality checks plus config and tier + first-cluster-node checks through the ECStore-owned runtime-source module. + - Acceptance: rebalance, config initialization, and tier config loading no + longer import global locality readers directly outside the runtime-source + boundary. + - Must preserve: rebalance local-pool worker selection, server-config + initialization ownership, non-first-node config lookup behavior, and tier + config initialization behavior. + - Verification: ECStore compile coverage, focused config/tier/rebalance + tests, formatting, diff hygiene, residual locality runtime-source scan, + Rust risk scan, migration/layer guards, PR-before-push pre-commit quality + gate, and three-expert review. + ## Next PRs 1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries. @@ -4787,6 +4802,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-192 keeps ECStore locality reads behind the runtime-source boundary without adding new public APIs. | +| Migration preservation | pass | Rebalance local-pool selection plus config and tier first-node behavior preserve existing fallback semantics. | +| Testing/verification | pass | ECStore compile, focused config/tier/rebalance tests, formatting, migration/layer guards, residual scan, Rust risk scan, and pre-commit planned before PR. | | Quality/architecture | pass | API-191 keeps bucket monitor and replication worker pool reads behind the ECStore runtime-source boundary without adding new public APIs. | | Migration preservation | pass | Bandwidth limit updates, monitored resync readers, active worker statistics, and bucket replication bandwidth reports keep existing fallback behavior. | | Testing/verification | pass | ECStore compile, focused replication-state test, formatting, migration/layer guards, residual scan, Rust risk scan, and pre-commit planned before PR. |