gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: route external ECStore imports through owners (#3747)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-22 20:57:10 +08:00
committed by GitHub
parent 88a87b3e8f
commit 418b5d04f9
9 changed files with 76 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/config_manager.rs
+2 -4
View File
@@ -13,16 +13,14 @@
// limitations under the License.
use crate::{
Event, NotificationError, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade,
Event, NotificationError, ecstore_config, ecstore_global, ecstore_storage, registry::TargetRegistry,
rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade,
};
use rustfs_config::notify::{
NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::server_config::{Config, KVS};
use rustfs_ecstore::api::config as ecstore_config;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::storage as ecstore_storage;
use rustfs_targets::{Target, arn::TargetID};
use std::sync::Arc;
use tokio::sync::RwLock;
crates/notify/src/lib.rs
+4
View File
@@ -37,6 +37,10 @@ mod runtime_view;
mod services;
mod status_view;
pub(crate) use rustfs_ecstore::api::config as ecstore_config;
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
pub use bucket_config_manager::NotifyBucketConfigManager;
pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
pub use error::{LifecycleError, NotificationError};
crates/obs/src/metrics/mod.rs
+7
View File
@@ -19,6 +19,13 @@ pub mod scheduler;
pub mod schema;
pub mod stats_collector;
pub(crate) use rustfs_ecstore::api::bucket as ecstore_bucket;
pub(crate) use rustfs_ecstore::api::capacity as ecstore_capacity;
pub(crate) use rustfs_ecstore::api::data_usage as ecstore_data_usage;
pub(crate) use rustfs_ecstore::api::error as ecstore_error;
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
pub use collectors::*;
pub use config::*;
pub use report::{PrometheusMetric, report_metrics};
crates/obs/src/metrics/scheduler.rs
+1 -1
View File
@@ -70,6 +70,7 @@ use crate::metrics::config::{
ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
ENV_NODE_METRICS_INTERVAL, ENV_NOTIFICATION_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
};
use crate::metrics::ecstore_global;
use crate::metrics::report::{PrometheusMetric, report_metrics};
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, TARGET_ARN_L,
@@ -82,7 +83,6 @@ use crate::metrics::stats_collector::{
collect_scanner_metric_stats, collect_system_cpu_and_memory_stats_with,
};
use rustfs_audit::audit_target_metrics;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
use rustfs_utils::get_env_opt_u64;
use serde::Serialize;
crates/obs/src/metrics/stats_collector.rs
+1 -6
View File
@@ -26,15 +26,10 @@ use crate::metrics::collectors::{
DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats,
ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
};
use crate::metrics::{ecstore_bucket, ecstore_capacity, ecstore_data_usage, ecstore_error, ecstore_global, ecstore_storage};
use chrono::Utc;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::global_metrics;
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::capacity as ecstore_capacity;
use rustfs_ecstore::api::data_usage as ecstore_data_usage;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::storage as ecstore_storage;
use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
crates/s3select-api/src/lib.rs
+5
View File
@@ -16,6 +16,11 @@ use datafusion::{common::DataFusionError, sql::sqlparser::parser::ParserError};
use snafu::{Backtrace, Location, Snafu};
use std::fmt::Display;
pub(crate) use rustfs_ecstore::api::error as ecstore_error;
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
pub mod object_store;
pub mod query;
pub mod server;
crates/s3select-api/src/object_store.rs
+1 -4
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{ecstore_error, ecstore_global, ecstore_set_disk, ecstore_storage};
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Utc;
@@ -25,10 +26,6 @@ use object_store::{
};
use pin_project_lite::pin_project;
use rustfs_common::DEFAULT_DELIMITER;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::set_disk as ecstore_set_disk;
use rustfs_ecstore::api::storage as ecstore_storage;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
use s3s::S3Result;
use s3s::dto::SelectObjectContentInput;
docs/architecture/migration-progress.md
+41 -12
View File
@@ -5,17 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-external-ecstore-facade-aliases`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129`.
- Based on: API-129 slice.
- Branch: `overtrue/arch-external-owner-ecstore-boundaries`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130`.
- Based on: API-130 slice.
- PR type for this branch: `pure-move`
- Runtime behavior changes: none.
- Rust code changes: centralize non-ECStore grouped/raw ECStore facade imports
behind per-module `ecstore_*` aliases across external owner modules, tests,
and fuzz targets.
- CI/script changes: reject grouped `rustfs_ecstore::api::{...}` imports and
raw `rustfs_ecstore::api::<module>::...` subpaths outside ECStore.
- Docs changes: record the API-130 external ECStore facade alias cleanup.
- Rust code changes: move nested external production ECStore facade imports for
notify, observability, and S3 Select behind crate or module owner roots.
- CI/script changes: reject nested external production `rustfs_ecstore::api`
imports outside the approved crate or module owner roots.
- Docs changes: record the API-131 external owner boundary cleanup.
## Phase 0 Tasks
@@ -3862,6 +3861,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
scans, migration/layer guards, formatting, diff hygiene, Rust risk scan,
branch freshness check, pre-commit quality gate, and three-expert review.
- [x] `API-131` Route nested external production ECStore imports through owner roots.
- Do: expose notify, observability metrics, and S3 Select ECStore facade
aliases from their crate or module owner roots, and migrate nested
production files to import those local aliases instead of importing
`rustfs_ecstore::api` directly.
- Acceptance: nested production files under notify, observability, and S3
Select no longer import ECStore facade modules directly, while IAM,
scanner, heal, Swift, and owner root files remain the only approved
external production direct facade import points.
- Must preserve: notify config persistence, observability metrics collection
and scheduler bucket-monitor checks, S3 Select object-store error and
storage access behavior, and all public crate APIs.
- Verification: focused notify/obs/S3 Select compile, nested direct-import
residual scan, migration/layer guards, formatting, diff hygiene, Rust risk
scan, branch freshness check, pre-commit quality gate, and three-expert
review.
## Next PRs
1. `pure-move`: continue pruning remaining facade compatibility and owner boundaries.
@@ -3870,14 +3886,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | API-130 keeps external ECStore facade access explicit through per-module `ecstore_*` aliases without adding behavior wrappers. |
| Migration preservation | pass | IAM, notify, obs, Swift, S3 Select, e2e, heal/scanner test, and fuzz call paths keep the same ECStore symbols through local aliases. |
| Testing/verification | pass | Focused external crate compile, residual scans, migration/layer guards, formatting, diff hygiene, and diff-only Rust risk scan passed. |
| Quality/architecture | pass | API-131 keeps direct ECStore facade imports at external owner roots and leaves nested production modules on local aliases. |
| Migration preservation | pass | Notify, observability, and S3 Select call paths keep the same ECStore symbols through owner-root aliases only. |
| Testing/verification | pass | Focused notify/obs/S3 Select compile, nested direct-import residual scan, migration/layer guards, formatting, diff hygiene, and diff-only Rust risk scan passed. |
## Verification Notes
Passed before push:
- Issue #660 API-131 current slice:
- `cargo check --tests -p rustfs-notify -p rustfs-obs -p rustfs-s3select-api`: passed.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `make pre-commit`: passed, including clippy, script tests, nextest
`6518 passed, 111 skipped`, and doc-tests.
- Nested external production ECStore facade residual scan: passed.
- Rust risk scan: diff-only scan found new `as ecstore_*` import aliases
only; no new risky behavior added.
- Issue #660 API-130 current slice:
- `cargo check --tests -p rustfs-notify -p rustfs-obs -p rustfs-protocols -p rustfs-s3select-api -p e2e_test -p rustfs-heal -p rustfs-scanner -p rustfs-iam`: passed.
- `cargo fmt --all`: passed.
scripts/check_architecture_migration_rules.sh
+14 -1
View File
@@ -99,6 +99,7 @@ ALL_STORAGE_COMPAT_RAW_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_raw_
ALL_STORAGE_COMPAT_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_storage_compat_grouped_facade_import_hits.txt"
ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_ecstore_api_grouped_facade_import_hits.txt"
ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE="${TMP_DIR}/all_ecstore_api_raw_subpath_hits.txt"
EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/external_production_ecstore_import_hits.txt"
ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt"
RUSTFS_LOCAL_COMPAT_OWNER_SELF_PATH_HITS_FILE="${TMP_DIR}/rustfs_local_compat_owner_self_path_hits.txt"
RUSTFS_ROOT_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_compat_relative_consumer_hits.txt"
@@ -717,7 +718,7 @@ fi
--glob '!**/storage_compat.rs' \
--glob '!**/*storage_compat.rs' \
--glob '!target/**' \
| rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/lib\.rs|crates/notify/src/config_manager\.rs|crates/obs/src/metrics/(scheduler|stats_collector)\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/object_store\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
| rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
) |
cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE"
@@ -1066,6 +1067,18 @@ if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
report_failure "non-ECStore sources must keep raw ECStore facade subpaths behind local ecstore_* module aliases: $(paste -sd '; ' "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
crates/heal/src crates/iam/src crates/notify/src crates/obs/src crates/protocols/src crates/s3select-api/src crates/scanner/src \
--glob '*.rs' |
rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true
) >"$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE"
if [[ -s "$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" ]]; then
report_failure "external production ECStore facade imports must stay at crate or module owner roots: $(paste -sd '; ' "$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::(?:admin|app|storage)::storage_compat::ecstore_|crate::storage_compat::ecstore_' rustfs/src crates fuzz \