refactor: route ecstore runtime globals through facade (#3941)

This commit is contained in:
Zhengchao An
2026-06-27 12:27:03 +08:00
committed by GitHub
parent d3f1ff36af
commit 1b3dea012e
14 changed files with 83 additions and 62 deletions
+7
View File
@@ -120,6 +120,13 @@ pub mod global {
};
}
pub mod runtime {
pub use crate::runtime::sources::{
bucket_monitor, first_cluster_node_is_local, global_tier_config_mgr, local_disk_map_read, object_store_handle,
setup_is_erasure, setup_is_erasure_sd,
};
}
pub mod layout {
pub use crate::layout::disks_layout::DisksLayout;
pub use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType};
+11 -7
View File
@@ -50,7 +50,7 @@ use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
use rustfs_lock::client::LockClient;
use s3s::dto::BucketLifecycleConfiguration;
use s3s::region::Region;
use tokio::sync::RwLock;
use tokio::sync::{RwLock, RwLockReadGuard};
use tonic::transport::Channel;
use uuid::Uuid;
@@ -108,7 +108,7 @@ pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionSe
get_global_encryption_service().await
}
pub(crate) fn object_store_handle() -> Option<Arc<ECStore>> {
pub fn object_store_handle() -> Option<Arc<ECStore>> {
resolve_object_store_handle()
}
@@ -131,11 +131,11 @@ pub(crate) fn endpoint_pool_is_local(pool_index: usize) -> bool {
.is_some_and(|pool| pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local))
}
pub(crate) async fn first_cluster_node_is_local() -> bool {
pub async fn first_cluster_node_is_local() -> bool {
is_first_cluster_node_local().await
}
pub(crate) async fn setup_is_erasure() -> bool {
pub async fn setup_is_erasure() -> bool {
is_erasure().await
}
@@ -143,7 +143,7 @@ pub(crate) async fn setup_is_dist_erasure() -> bool {
is_dist_erasure().await
}
pub(crate) async fn setup_is_erasure_sd() -> bool {
pub async fn setup_is_erasure_sd() -> bool {
*GLOBAL_IsErasureSD.read().await
}
@@ -336,7 +336,7 @@ pub(crate) fn batch_processors() -> &'static GlobalBatchProcessors {
get_global_processors()
}
pub(crate) fn global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
pub fn global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
get_global_tier_config_mgr()
}
@@ -350,10 +350,14 @@ pub(crate) fn delete_bucket_monitor_entry(bucket: &str) {
}
}
pub(crate) fn bucket_monitor() -> Option<Arc<Monitor>> {
pub fn bucket_monitor() -> Option<Arc<Monitor>> {
get_global_bucket_monitor()
}
pub async fn local_disk_map_read() -> RwLockReadGuard<'static, HashMap<String, Option<DiskStore>>> {
GLOBAL_LOCAL_DISK_MAP.read().await
}
pub(crate) fn init_bucket_monitor_for_current_endpoints() {
let num_nodes = get_global_endpoints().get_nodes().len().try_into().unwrap_or(u64::MAX);
init_global_bucket_monitor(num_nodes);
+4 -4
View File
@@ -24,9 +24,9 @@ pub mod task;
pub mod utils;
use storage_api::owner::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_RUSTFS_META_BUCKET, EcstoreDeleteOptions, EcstoreDiskAPI,
EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, ecstore_local_disk_map_read,
};
#[cfg(test)]
use storage_api::owner::{EcstoreDiskOption, ecstore_new_disk};
@@ -55,7 +55,7 @@ pub(crate) static GLOBAL_LOCAL_DISK_MAP: GlobalLocalDiskMap = GlobalLocalDiskMap
impl GlobalLocalDiskMap {
pub(crate) async fn read(&self) -> tokio::sync::RwLockReadGuard<'static, LocalDiskMap> {
ECSTORE_GLOBAL_LOCAL_DISK_MAP.read().await
ecstore_local_disk_map_read().await
}
}
+4 -4
View File
@@ -22,7 +22,7 @@ pub(crate) use rustfs_ecstore::api::disk::{
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
use rustfs_storage_api as storage_contracts;
@@ -30,9 +30,9 @@ pub(crate) mod owner {
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
pub(crate) use super::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore,
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_RUSTFS_META_BUCKET, EcstoreDeleteOptions,
EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint,
EcstoreErrorType, EcstoreStorageError, EcstoreStore, ecstore_local_disk_map_read,
};
#[cfg(test)]
+2 -2
View File
@@ -24,10 +24,10 @@ use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
classify_system_path_failure_reason as ecstore_classify_system_path_failure_reason,
};
use rustfs_ecstore::api::global::is_first_cluster_node_local as ecstore_is_first_cluster_node_local;
use rustfs_ecstore::api::notification::{
NotificationPeerErr as EcstoreNotificationPeerErr, NotificationSys as EcstoreNotificationSys, get_global_notification_sys,
};
use rustfs_ecstore::api::runtime::first_cluster_node_is_local as ecstore_first_cluster_node_is_local;
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
use rustfs_storage_api as storage_contracts;
@@ -74,7 +74,7 @@ pub(crate) fn classify_iam_system_path_failure_reason(err: &IamEcstoreError) ->
}
pub(crate) async fn is_iam_first_cluster_node_local() -> bool {
ecstore_is_first_cluster_node_local().await
ecstore_first_cluster_node_is_local().await
}
pub(crate) fn notification_sys() -> Option<&'static IamNotificationSys> {
+1 -1
View File
@@ -18,7 +18,7 @@ use rustfs_ecstore::api::config::com::{
read_config_without_migrate as read_notify_config_without_migrate_from_backend,
save_server_config as save_notify_server_config_to_backend,
};
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_notify_object_store_handle_from_backend;
use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
+2 -2
View File
@@ -26,8 +26,8 @@ pub(crate) use rustfs_ecstore::api::capacity::{
};
pub(crate) use rustfs_ecstore::api::data_usage::load_data_usage_from_backend as obs_load_data_usage_from_backend;
pub(crate) use rustfs_ecstore::api::error::Result as ObsEcstoreResult;
pub(crate) use rustfs_ecstore::api::global::{
get_global_bucket_monitor as obs_get_global_bucket_monitor, resolve_object_store_handle as obs_resolve_object_store_handle,
pub(crate) use rustfs_ecstore::api::runtime::{
bucket_monitor as obs_get_global_bucket_monitor, object_store_handle as obs_resolve_object_store_handle,
};
pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
use rustfs_storage_api as storage_contracts;
+1 -1
View File
@@ -19,7 +19,7 @@ use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
};
pub(crate) use rustfs_ecstore::api::error::Result as SwiftStorageResult;
pub(crate) use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_swift_object_store_handle;
pub(crate) use rustfs_ecstore::api::runtime::object_store_handle as resolve_swift_object_store_handle;
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
use rustfs_storage_api as storage_contracts;
+1 -1
View File
@@ -20,7 +20,7 @@ use rustfs_ecstore::api::error::{
is_err_object_not_found as select_is_err_object_not_found_from_backend,
is_err_version_not_found as select_is_err_version_not_found_from_backend,
};
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_select_object_store_handle_from_backend;
use rustfs_ecstore::api::runtime::object_store_handle as resolve_select_object_store_handle_from_backend;
pub(crate) use rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE as SELECT_DEFAULT_READ_BUFFER_SIZE;
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
use rustfs_storage_api as storage_contracts;
+3 -3
View File
@@ -65,9 +65,9 @@ pub(crate) use rustfs_ecstore::api::disk::{
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
};
pub(crate) use rustfs_ecstore::api::global::{
get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
pub(crate) use rustfs_ecstore::api::runtime::{
global_tier_config_mgr as ecstore_get_global_tier_config_mgr, object_store_handle as ecstore_resolve_object_store_handle,
setup_is_erasure as ecstore_is_erasure, setup_is_erasure_sd as ecstore_is_erasure_sd,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
+1 -1
View File
@@ -24,8 +24,8 @@ pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::api::global::get_global_tier_config_mgr;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::runtime::global_tier_config_mgr as get_global_tier_config_mgr;
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::api::tier::warm_backend::{
@@ -12,7 +12,7 @@ control-plane boundaries are stable.
| `rustfs/src/app/context/runtime_sources.rs` | Default AppContext fallback adapters for KMS, IAM, object store, endpoints, config, metrics, and notification state. | This is an allowed fallback boundary, not a business logic owner. |
| `rustfs/src/*/runtime_sources.rs` | Root, admin, app, server, startup, and storage owner-local runtime-source boundaries. | Business modules use these boundaries instead of calling global state directly. |
| `rustfs/src/*/storage_api.rs` | Root, admin, app, and storage owner-local storage contract/facade boundaries. | Storage helper and ECStore facade access remains visible at local owner boundaries. |
| `crates/*/storage_api.rs` | External crate-local storage facade boundaries for IAM, scanner, heal, notify, observability, Swift, and S3 Select. | External runtime crates may consume ECStore facade globals only through their local storage API boundary. |
| `crates/*/storage_api.rs` | External crate-local storage facade boundaries for IAM, scanner, heal, notify, observability, Swift, and S3 Select. | External runtime crates consume ECStore runtime state through `rustfs_ecstore::api::runtime` instead of the direct global facade. |
| `crates/ecstore/src/runtime/global.rs` | ECStore bootstrap/runtime state owner. | Keep internal until ECStore has explicit owner handles for all remaining bootstrap state. |
| `crates/ecstore/src/runtime/sources.rs` | ECStore runtime-source adapter over global state. | Preferred ECStore-internal access path while shrinking direct `runtime::global` reads. |
@@ -33,14 +33,6 @@ consumers and process-global state. They must keep these properties:
The architecture guard snapshots the files currently allowed to reference
`rustfs_ecstore::api::global` directly:
- `crates/heal/src/heal/storage_api.rs`
- `crates/iam/src/storage_api.rs`
- `crates/notify/src/storage_api.rs`
- `crates/obs/src/metrics/storage_api.rs`
- `crates/protocols/src/swift/storage_api.rs`
- `crates/s3select-api/src/storage_api.rs`
- `crates/scanner/src/storage_api.rs`
- `crates/scanner/tests/storage_api/mod.rs`
- `rustfs/src/storage/storage_api.rs`
New direct uses must either move behind an existing owner-local boundary or
+45 -19
View File
@@ -5,24 +5,24 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-global-state-inventory-phase`
- Branch: `overtrue/arch-global-runtime-boundary-batch`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/API-243/API-244/API-245/API-246/API-247/API-248/API-249/API-250/API-251/API-252/API-253/API-254/CTX-002`.
- Current baseline also includes API-255 from PR #3923, API-256 from PR
#3925, CFG-009 from PR #3927, C-007/C-009 from PR #3935, C-008/C-010
from PR #3936, and DOC-001/DOC-002/DOC-003/DOC-004/DOC-005/
TEST-DOC-001 from PR #3938.
- Current phase PR: GLOB-001/GLOB-002/GLOB-003/GLOB-004/GLOB-005/GLOB-006/
CRATE-001/CRATE-002 global-state inventory and crate-split guard batch.
- Based on: `origin/main` after PR #3938 merged.
TEST-DOC-001 from PR #3938, plus GLOB-001/GLOB-002/GLOB-003/GLOB-004/
GLOB-005/GLOB-006/CRATE-001/CRATE-002 from PR #3939.
- Current phase PR: GLOB-006 ECStore runtime facade boundary batch.
- Based on: `origin/main` after PR #3939 merged.
- PR type for this branch: `ci-gate`.
- Runtime behavior changes: none.
- Rust code changes: none.
- CI/script changes: require the global-state/crate-split plan anchors and lock
direct `rustfs_ecstore::api::global` access to reviewed local `storage_api`
boundary files.
- Docs changes: add the global-state and crate-split plan, link it from the
architecture overview, and include it in the required architecture document
set.
- Runtime behavior changes: none intended.
- Rust code changes: add the selected `rustfs_ecstore::api::runtime` facade and
route external crate-local `storage_api` boundaries away from direct
`rustfs_ecstore::api::global` imports.
- CI/script changes: shrink the direct ECStore global facade allowlist to the
root storage owner boundary.
- Docs changes: update the global-state plan and this progress ledger for the
runtime facade batch.
## Phase 0 Tasks
@@ -2765,11 +2765,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
record the remaining owner boundaries and fallback removal sequence.
- Verification: architecture migration guard and diff hygiene.
- [~] `GLOB-006` Shrink `ecstore::global`.
- Current slice: guard the current reviewed files allowed to reference
`rustfs_ecstore::api::global` directly.
- Remaining work: move ECStore bootstrap/runtime globals behind explicit owner
handles where safe; do not remove bootstrap state in this guardrail PR.
- Verification: architecture migration guard and diff hygiene.
- Current slice: expose selected ECStore runtime-source reads through
`rustfs_ecstore::api::runtime`, migrate heal, IAM, notify, observability,
Swift, S3 Select, and scanner storage API boundaries away from direct
`rustfs_ecstore::api::global` imports, and keep the direct global facade
confined to the root storage owner boundary.
- Remaining work: move additional ECStore bootstrap/runtime globals behind
explicit owner handles where safe; do not remove bootstrap state in this
runtime facade PR.
- Verification: focused compile coverage, architecture migration guard, diff
hygiene, Rust risk scan, and full PR gate.
- [ ] `GLOB-007` Remove fallbacks.
- Remaining work: remove one fallback family per PR only after scans prove no
production caller depends on it.
@@ -6036,6 +6041,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | GLOB-006 exposes selected ECStore runtime-source reads through `rustfs_ecstore::api::runtime` and removes external crate direct global facade imports while leaving ECStore owner state intact. |
| Migration preservation | pass | Object-store, bucket-monitor, tier-config, erasure flags, IAM first-node, and local-disk-map reads delegate to the same ECStore runtime/global owners; startup, readiness, IAM/KMS, lock quorum, scanner, heal, Swift, S3 Select, and notification behavior stay unchanged. |
| Testing/verification | pass | Focused compile, formatting, shell syntax, architecture guard, global facade scan, diff hygiene, Rust risk scan, full `make pre-pr`, and post-rebase focused checks passed before PR. |
| Quality/architecture | pass | The Phase 7 batch documents remaining global owners and locks direct ECStore global facade access to reviewed storage_api boundaries without adding another abstraction layer. |
| Migration preservation | pass | This is docs and guardrail only; AppContext fallback, ECStore bootstrap globals, startup order, readiness, IAM/KMS, lock quorum, and storage behavior are unchanged. |
| Testing/verification | pass | Shell syntax, architecture migration guard, diff hygiene, and full `make pre-pr` passed after rebasing onto current `origin/main`; no Rust source changed. |
@@ -6408,7 +6416,10 @@ Passed before push:
- `git diff --check`: passed.
- Rust source risk scan: passed; no Rust source changed.
- Three-expert review: passed.
- `make pre-pr`: passed.
- `make pre-pr`: passed before rebasing from the stacked #3939 branch onto
merged `origin/main`.
- Post-rebase focused checks: cargo check, `cargo fmt --all --check`,
architecture migration guard, global facade scan, and diff hygiene passed.
- Issue #660 API-256 current slice:
- Branch freshness check: based on current `origin/main` after PR #3923
@@ -9465,6 +9476,21 @@ Notes:
public APIs, boxed public errors, production println/eprintln, or relaxed
ordering introduced in changed Rust files.
- Issue #660 GLOB-006 current slice:
- `cargo check -p rustfs-ecstore -p rustfs-heal -p rustfs-iam -p rustfs-notify -p rustfs-obs -p rustfs-protocols -p rustfs-s3select-api -p rustfs-scanner --lib`:
passed.
- `cargo fmt --all --check`: passed.
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `git diff --check`: passed.
- `make pre-pr`: passed.
- ECStore global facade scan: passed; direct
`rustfs_ecstore::api::global` import use is confined to
`rustfs/src/storage/storage_api.rs`.
- Rust risk scan: passed; no new unwrap/expect, numeric casts, string error
public APIs, boxed public errors, production println/eprintln, or relaxed
ordering introduced in changed Rust lines.
## Handoff Notes
- Continue with larger consumer-migration batches outside the cleaned
@@ -1526,14 +1526,6 @@ if [[ -s "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
fi
cat >"$GLOBAL_FACADE_BOUNDARY_EXPECTED_FILE" <<'EOF'
crates/heal/src/heal/storage_api.rs
crates/iam/src/storage_api.rs
crates/notify/src/storage_api.rs
crates/obs/src/metrics/storage_api.rs
crates/protocols/src/swift/storage_api.rs
crates/s3select-api/src/storage_api.rs
crates/scanner/src/storage_api.rs
crates/scanner/tests/storage_api/mod.rs
rustfs/src/storage/storage_api.rs
EOF