mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 04:16:38 +00:00
refactor: centralize ecstore locality runtime sources (#3808)
This commit is contained in:
@@ -15,8 +15,8 @@
|
|||||||
use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass};
|
use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass};
|
||||||
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::global::is_first_cluster_node_local;
|
|
||||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||||
|
use crate::runtime_sources;
|
||||||
use crate::storage_api_contracts::EcstoreObjectIO;
|
use crate::storage_api_contracts::EcstoreObjectIO;
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_config::audit::{
|
use rustfs_config::audit::{
|
||||||
@@ -1150,7 +1150,7 @@ where
|
|||||||
S: EcstoreObjectIO + StorageAdminApi,
|
S: EcstoreObjectIO + StorageAdminApi,
|
||||||
{
|
{
|
||||||
warn!("Configuration not found ({}): Start initializing new configuration", context);
|
warn!("Configuration not found ({}): Start initializing new configuration", context);
|
||||||
let cfg = if is_first_cluster_node_local().await {
|
let cfg = if runtime_sources::first_cluster_node_is_local().await {
|
||||||
new_and_save_server_config(api.clone()).await?
|
new_and_save_server_config(api.clone()).await?
|
||||||
} else {
|
} else {
|
||||||
let mut cfg = new_server_config();
|
let mut cfg = new_server_config();
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use super::{
|
|||||||
RebalanceBucketOutcome,
|
RebalanceBucketOutcome,
|
||||||
};
|
};
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::global::get_global_endpoints;
|
use crate::runtime_sources;
|
||||||
use crate::store::ECStore;
|
use crate::store::ECStore;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -120,11 +120,7 @@ impl ECStore {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if !get_global_endpoints()
|
if !runtime_sources::endpoint_pool_is_local(idx) {
|
||||||
.as_ref()
|
|
||||||
.get(idx)
|
|
||||||
.is_some_and(|v| v.endpoints.as_ref().first().is_some_and(|e| e.is_local))
|
|
||||||
{
|
|
||||||
debug!(
|
debug!(
|
||||||
event = EVENT_REBALANCE_STATE,
|
event = EVENT_REBALANCE_STATE,
|
||||||
component = LOG_COMPONENT_ECSTORE,
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
|||||||
@@ -28,7 +28,8 @@ use crate::{
|
|||||||
GLOBAL_BOOT_TIME, GLOBAL_EventNotifier, GLOBAL_IsErasureSD, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP,
|
GLOBAL_BOOT_TIME, GLOBAL_EventNotifier, GLOBAL_IsErasureSD, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP,
|
||||||
GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LifecycleSys, GLOBAL_LocalNodeName, GLOBAL_RootDiskThreshold, GLOBAL_TierConfigMgr,
|
GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LifecycleSys, GLOBAL_LocalNodeName, GLOBAL_RootDiskThreshold, GLOBAL_TierConfigMgr,
|
||||||
TypeLocalDiskSetDrives, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints,
|
TypeLocalDiskSetDrives, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints,
|
||||||
get_global_endpoints_opt, init_global_bucket_monitor, resolve_object_store_handle, set_global_deployment_id,
|
get_global_endpoints_opt, init_global_bucket_monitor, is_first_cluster_node_local, resolve_object_store_handle,
|
||||||
|
set_global_deployment_id,
|
||||||
},
|
},
|
||||||
notification_sys::{NotificationSys, get_global_notification_sys},
|
notification_sys::{NotificationSys, get_global_notification_sys},
|
||||||
store::ECStore,
|
store::ECStore,
|
||||||
@@ -58,6 +59,17 @@ pub(crate) fn endpoint_pools() -> Option<EndpointServerPools> {
|
|||||||
get_global_endpoints_opt()
|
get_global_endpoints_opt()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn endpoint_pool_is_local(pool_index: usize) -> bool {
|
||||||
|
get_global_endpoints()
|
||||||
|
.as_ref()
|
||||||
|
.get(pool_index)
|
||||||
|
.is_some_and(|pool| pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn first_cluster_node_is_local() -> bool {
|
||||||
|
is_first_cluster_node_local().await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn local_node_name() -> String {
|
pub(crate) async fn local_node_name() -> String {
|
||||||
GLOBAL_LOCAL_NODE_NAME.read().await.clone()
|
GLOBAL_LOCAL_NODE_NAME.read().await.clone()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,8 +48,8 @@ use crate::tier::{
|
|||||||
use crate::{
|
use crate::{
|
||||||
config::com::{CONFIG_PREFIX, read_config},
|
config::com::{CONFIG_PREFIX, read_config},
|
||||||
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
|
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
|
||||||
global::is_first_cluster_node_local,
|
|
||||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||||
|
runtime_sources,
|
||||||
storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations},
|
storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations},
|
||||||
store::ECStore,
|
store::ECStore,
|
||||||
};
|
};
|
||||||
@@ -1171,7 +1171,7 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
|
|||||||
}
|
}
|
||||||
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
|
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
|
||||||
warn!("config not found, start to init");
|
warn!("config not found, start to init");
|
||||||
if is_first_cluster_node_local().await {
|
if runtime_sources::first_cluster_node_is_local().await {
|
||||||
new_and_save_tiering_config(api).await.map_err(io::Error::other)
|
new_and_save_tiering_config(api).await.map_err(io::Error::other)
|
||||||
} else {
|
} else {
|
||||||
Ok(TierConfigMgr {
|
Ok(TierConfigMgr {
|
||||||
|
|||||||
@@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Current Context
|
## Current Context
|
||||||
|
|
||||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||||
- Branch: `overtrue/arch-ecstore-bucket-runtime-sources-batch`
|
- Branch: `overtrue/arch-owner-root-facade-source-batch`
|
||||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191`.
|
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192`.
|
||||||
- Based on: latest default branch after merge 3805.
|
- Based on: latest default branch after merge 3806.
|
||||||
- PR type for this branch: `consumer-migration`
|
- PR type for this branch: `consumer-migration`
|
||||||
- Runtime behavior changes: none.
|
- Runtime behavior changes: none.
|
||||||
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
||||||
@@ -22,7 +22,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
boot-time, init-time, root-disk threshold, and cached RPC channel reads,
|
boot-time, init-time, root-disk threshold, and cached RPC channel reads,
|
||||||
plus ECStore replication pool, replication stats, and event-host reads,
|
plus ECStore replication pool, replication stats, and event-host reads,
|
||||||
plus ECStore lifecycle queue state, tier config, lifecycle config, deployment
|
plus ECStore lifecycle queue state, tier config, lifecycle config, deployment
|
||||||
id, event-host reads, bucket monitor reads, and replication worker pool reads,
|
id, event-host reads, bucket monitor reads, replication worker pool reads,
|
||||||
|
config/tier first-node checks, and rebalance endpoint-locality checks,
|
||||||
through AppContext-first or owner-crate resolver boundaries.
|
through AppContext-first or owner-crate resolver boundaries.
|
||||||
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
||||||
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
||||||
@@ -32,7 +33,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
and storage owner thin bridge regressions, plus app context and notify
|
and storage owner thin bridge regressions, plus app context and notify
|
||||||
event-bridge thin module regressions; accept the reviewed AppContext resolver
|
event-bridge thin module regressions; accept the reviewed AppContext resolver
|
||||||
reverse dependencies in the layer baseline.
|
reverse dependencies in the layer baseline.
|
||||||
- Docs changes: record the API-136 through API-191 owner facade cleanup.
|
- Docs changes: record the API-136 through API-192 owner facade cleanup.
|
||||||
|
|
||||||
## Phase 0 Tasks
|
## Phase 0 Tasks
|
||||||
|
|
||||||
@@ -4779,6 +4780,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
scan, migration/layer guards, PR-before-push pre-commit quality gate, and
|
scan, migration/layer guards, PR-before-push pre-commit quality gate, and
|
||||||
three-expert review.
|
three-expert review.
|
||||||
|
|
||||||
|
- [x] `API-192` Centralize ECStore locality runtime source reads.
|
||||||
|
- Do: route rebalance endpoint-locality checks plus config and tier
|
||||||
|
first-cluster-node checks through the ECStore-owned runtime-source module.
|
||||||
|
- Acceptance: rebalance, config initialization, and tier config loading no
|
||||||
|
longer import global locality readers directly outside the runtime-source
|
||||||
|
boundary.
|
||||||
|
- Must preserve: rebalance local-pool worker selection, server-config
|
||||||
|
initialization ownership, non-first-node config lookup behavior, and tier
|
||||||
|
config initialization behavior.
|
||||||
|
- Verification: ECStore compile coverage, focused config/tier/rebalance
|
||||||
|
tests, formatting, diff hygiene, residual locality runtime-source scan,
|
||||||
|
Rust risk scan, migration/layer guards, PR-before-push pre-commit quality
|
||||||
|
gate, and three-expert review.
|
||||||
|
|
||||||
## Next PRs
|
## Next PRs
|
||||||
|
|
||||||
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
|
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
|
||||||
@@ -4787,6 +4802,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
|
|
||||||
| Expert | Status | Notes |
|
| Expert | Status | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
| Quality/architecture | pass | API-192 keeps ECStore locality reads behind the runtime-source boundary without adding new public APIs. |
|
||||||
|
| Migration preservation | pass | Rebalance local-pool selection plus config and tier first-node behavior preserve existing fallback semantics. |
|
||||||
|
| Testing/verification | pass | ECStore compile, focused config/tier/rebalance tests, formatting, migration/layer guards, residual scan, Rust risk scan, and pre-commit planned before PR. |
|
||||||
| Quality/architecture | pass | API-191 keeps bucket monitor and replication worker pool reads behind the ECStore runtime-source boundary without adding new public APIs. |
|
| Quality/architecture | pass | API-191 keeps bucket monitor and replication worker pool reads behind the ECStore runtime-source boundary without adding new public APIs. |
|
||||||
| Migration preservation | pass | Bandwidth limit updates, monitored resync readers, active worker statistics, and bucket replication bandwidth reports keep existing fallback behavior. |
|
| Migration preservation | pass | Bandwidth limit updates, monitored resync readers, active worker statistics, and bucket replication bandwidth reports keep existing fallback behavior. |
|
||||||
| Testing/verification | pass | ECStore compile, focused replication-state test, formatting, migration/layer guards, residual scan, Rust risk scan, and pre-commit planned before PR. |
|
| Testing/verification | pass | ECStore compile, focused replication-state test, formatting, migration/layer guards, residual scan, Rust risk scan, and pre-commit planned before PR. |
|
||||||
|
|||||||
Reference in New Issue
Block a user