refactor: expose external ECStore owner symbols (#3748)

This commit is contained in:
Zhengchao An
2026-06-22 21:42:54 +08:00
committed by GitHub
parent 418b5d04f9
commit db7ff8f513
7 changed files with 127 additions and 79 deletions
+8 -22
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::{
Event, NotificationError, ecstore_config, ecstore_global, ecstore_storage, registry::TargetRegistry,
rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade,
Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
runtime_facade::NotifyRuntimeFacade,
};
use rustfs_config::notify::{
NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
@@ -31,8 +31,6 @@ const LOG_SUBSYSTEM_CONFIG: &str = "config";
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
type NotifyStore = ecstore_storage::ECStore;
#[derive(Debug)]
enum NotifyConfigStoreError {
StorageNotAvailable,
@@ -48,33 +46,21 @@ where
return Err(NotifyConfigStoreError::StorageNotAvailable);
};
let mut new_config = read_notify_server_config_without_migrate(store.clone()).await?;
let mut new_config = crate::read_notify_server_config_without_migrate(store.clone())
.await
.map_err(NotifyConfigStoreError::Read)?;
if !modifier(&mut new_config) {
return Ok(None);
}
save_notify_server_config(store, &new_config).await?;
crate::save_notify_server_config(store, &new_config)
.await
.map_err(NotifyConfigStoreError::Save)?;
Ok(Some(new_config))
}
fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
ecstore_global::resolve_object_store_handle()
}
async fn read_notify_server_config_without_migrate(store: Arc<NotifyStore>) -> Result<Config, NotifyConfigStoreError> {
ecstore_config::com::read_config_without_migrate(store)
.await
.map_err(|err| NotifyConfigStoreError::Read(err.to_string()))
}
async fn save_notify_server_config(store: Arc<NotifyStore>, config: &Config) -> Result<(), NotifyConfigStoreError> {
ecstore_config::com::save_server_config(store, config)
.await
.map_err(|err| NotifyConfigStoreError::Save(err.to_string()))
}
pub(crate) fn notify_configuration_hint() -> String {
let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT);
+24 -3
View File
@@ -18,6 +18,8 @@
//! It supports sending events to various targets
//! (like Webhook and MQTT) and includes features like event persistence and retry on failure.
use std::sync::Arc;
mod bucket_config_manager;
mod config_manager;
mod error;
@@ -37,9 +39,7 @@ mod runtime_view;
mod services;
mod status_view;
pub(crate) use rustfs_ecstore::api::config as ecstore_config;
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
pub use bucket_config_manager::NotifyBucketConfigManager;
pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
@@ -58,3 +58,24 @@ pub use runtime_facade::NotifyRuntimeFacade;
pub use runtime_view::NotifyRuntimeView;
pub use services::NotifyServices;
pub use status_view::NotifyStatusView;
pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
rustfs_ecstore::api::global::resolve_object_store_handle()
}
pub(crate) async fn read_notify_server_config_without_migrate(
store: Arc<NotifyStore>,
) -> Result<rustfs_config::server_config::Config, String> {
rustfs_ecstore::api::config::com::read_config_without_migrate(store)
.await
.map_err(|err| err.to_string())
}
pub(crate) async fn save_notify_server_config(
store: Arc<NotifyStore>,
config: &rustfs_config::server_config::Config,
) -> Result<(), String> {
rustfs_ecstore::api::config::com::save_server_config(store, config)
.await
.map_err(|err| err.to_string())
}
+9 -13
View File
@@ -35,10 +35,13 @@
use std::sync::Arc;
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::storage as ecstore_storage;
use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
};
use rustfs_ecstore::api::error::Result as SwiftStorageResult;
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_swift_object_store_handle;
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
pub mod account;
pub mod acl;
@@ -70,22 +73,15 @@ pub use router::{SwiftRoute, SwiftRouter};
#[allow(unused_imports)]
pub use types::{Container, Object, SwiftMetadata};
type SwiftBucketMetadata = ecstore_bucket::metadata::BucketMetadata;
type SwiftStorageResult<T> = ecstore_error::Result<T>;
type SwiftStore = ecstore_storage::ECStore;
pub type SwiftGetObjectReader = <SwiftStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
pub type SwiftObjectInfo = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub type SwiftObjectOptions = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub type SwiftPutObjReader = <SwiftStore as rustfs_storage_api::ObjectIO>::PutObjectReader;
fn resolve_swift_object_store_handle() -> Option<Arc<SwiftStore>> {
ecstore_global::resolve_object_store_handle()
}
async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResult<Arc<SwiftBucketMetadata>> {
ecstore_bucket::metadata_sys::get(bucket).await
get_swift_bucket_metadata_from_backend(bucket).await
}
async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, metadata).await
set_swift_bucket_metadata_in_backend(bucket, metadata).await
}
+25 -4
View File
@@ -15,11 +15,10 @@
use datafusion::{common::DataFusionError, sql::sqlparser::parser::ParserError};
use snafu::{Backtrace, Location, Snafu};
use std::fmt::Display;
use std::sync::Arc;
pub(crate) use rustfs_ecstore::api::error as ecstore_error;
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
pub(crate) use rustfs_ecstore::api::error::StorageError as SelectStorageError;
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
pub mod object_store;
pub mod query;
@@ -30,6 +29,28 @@ mod test;
pub type QueryResult<T> = Result<T, QueryError>;
pub(crate) type SelectGetObjectReader = <SelectStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
pub(crate) type SelectObjectInfo = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub(crate) type SelectObjectOptions = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub(crate) const SELECT_DEFAULT_READ_BUFFER_SIZE: usize = rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE;
pub(crate) fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
rustfs_ecstore::api::global::resolve_object_store_handle()
}
pub(crate) fn select_is_err_bucket_not_found(err: &SelectStorageError) -> bool {
rustfs_ecstore::api::error::is_err_bucket_not_found(err)
}
pub(crate) fn select_is_err_object_not_found(err: &SelectStorageError) -> bool {
rustfs_ecstore::api::error::is_err_object_not_found(err)
}
pub(crate) fn select_is_err_version_not_found(err: &SelectStorageError) -> bool {
rustfs_ecstore::api::error::is_err_version_not_found(err)
}
#[derive(Debug, Snafu)]
#[snafu(visibility(pub))]
pub enum QueryError {
+5 -25
View File
@@ -12,7 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{ecstore_error, ecstore_global, ecstore_set_disk, ecstore_storage};
use crate::{
SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions, SelectStorageError,
SelectStore, resolve_select_object_store_handle, select_is_err_bucket_not_found, select_is_err_object_not_found,
select_is_err_version_not_found,
};
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Utc;
@@ -45,34 +49,10 @@ use tokio::io::{AsyncRead, ReadBuf};
use tokio_util::io::ReaderStream;
use transform_stream::AsyncTryStream;
type SelectStorageError = ecstore_error::StorageError;
type SelectStore = ecstore_storage::ECStore;
type SelectGetObjectReader = <SelectStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
type SelectObjectInfo = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
type SelectObjectOptions = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
const SELECT_DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE;
fn select_default_read_buffer_size_u64() -> u64 {
u64::try_from(SELECT_DEFAULT_READ_BUFFER_SIZE).unwrap_or(u64::MAX)
}
fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
ecstore_global::resolve_object_store_handle()
}
fn select_is_err_bucket_not_found(err: &SelectStorageError) -> bool {
ecstore_error::is_err_bucket_not_found(err)
}
fn select_is_err_object_not_found(err: &SelectStorageError) -> bool {
ecstore_error::is_err_object_not_found(err)
}
fn select_is_err_version_not_found(err: &SelectStorageError) -> bool {
ecstore_error::is_err_version_not_found(err)
}
/// Maximum allowed object size for JSON DOCUMENT mode.
///
/// JSON DOCUMENT format requires loading the entire file into memory for DOM
+41 -11
View File
@@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-external-owner-ecstore-boundaries`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130`.
- Based on: API-130 slice.
- Branch: `overtrue/arch-external-owner-facade-symbols`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131`.
- Based on: API-131 slice.
- PR type for this branch: `pure-move`
- Runtime behavior changes: none.
- Rust code changes: move nested external production ECStore facade imports for
notify, observability, and S3 Select behind crate or module owner roots.
- CI/script changes: reject nested external production `rustfs_ecstore::api`
imports outside the approved crate or module owner roots.
- Docs changes: record the API-131 external owner boundary cleanup.
- Rust code changes: replace completed external owner root ECStore module
aliases for notify, Swift, and S3 Select with explicit local symbols and
wrapper functions.
- CI/script changes: reject restored `ecstore_*` module aliases in completed
external owner roots while allowing explicit owner-root facade symbols.
- Docs changes: record the API-132 external owner facade symbol cleanup.
## Phase 0 Tasks
@@ -3878,6 +3879,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
scan, branch freshness check, pre-commit quality gate, and three-expert
review.
- [x] `API-132` Replace completed external owner module aliases with symbols.
- Do: replace notify, Swift, and S3 Select owner-root `ecstore_*` module
aliases with explicit local ECStore symbols, type aliases, constants, and
wrapper functions.
- Acceptance: completed external owner roots no longer expose broad
`ecstore_*` module aliases, while nested modules keep using owner-local
symbols and the remaining larger observability, IAM, scanner, and heal
owner roots stay unchanged for later slices.
- Must preserve: notify config persistence, Swift bucket metadata access, S3
Select object-store error mapping, object reads, scan buffering, and all
public crate APIs.
- Verification: focused notify/Swift/S3 Select compile, completed-owner
alias residual scan, migration/layer guards, formatting, diff hygiene, Rust
risk scan, branch freshness check, pre-commit quality gate, and
three-expert review.
## Next PRs
1. `pure-move`: continue pruning remaining facade compatibility and owner boundaries.
@@ -3886,14 +3903,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | API-131 keeps direct ECStore facade imports at external owner roots and leaves nested production modules on local aliases. |
| Migration preservation | pass | Notify, observability, and S3 Select call paths keep the same ECStore symbols through owner-root aliases only. |
| Testing/verification | pass | Focused notify/obs/S3 Select compile, nested direct-import residual scan, migration/layer guards, formatting, diff hygiene, and diff-only Rust risk scan passed. |
| Quality/architecture | pass | API-132 replaces completed notify, Swift, and S3 Select owner-root ECStore module aliases with explicit local symbols and wrappers. |
| Migration preservation | pass | Notify config persistence, Swift bucket metadata access, and S3 Select object-store behavior keep the same ECStore call paths through owner roots. |
| Testing/verification | pass | Focused notify/Swift/S3 Select compile, completed-owner alias residual scan, migration/layer guards, formatting, diff hygiene, full pre-commit, and diff-only Rust risk scan passed. |
## Verification Notes
Passed before push:
- Issue #660 API-132 current slice:
- `cargo check --tests -p rustfs-notify -p rustfs-s3select-api -p rustfs-protocols --features rustfs-protocols/swift`: passed.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `make pre-commit`: passed, including clippy, script tests, nextest
`6518 passed, 111 skipped`, and doc-tests.
- Completed external owner module-alias residual scan: passed.
- Rust risk scan: diff-only scan found explicit `as` symbol imports only; no
new unwrap/expect, panic/todo/unsafe, or risky behavior added.
- Issue #660 API-131 current slice:
- `cargo check --tests -p rustfs-notify -p rustfs-obs -p rustfs-s3select-api`: passed.
- `cargo fmt --all`: passed.
+15 -1
View File
@@ -100,6 +100,7 @@ ALL_STORAGE_COMPAT_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_storage_compa
ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_ecstore_api_grouped_facade_import_hits.txt"
ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE="${TMP_DIR}/all_ecstore_api_raw_subpath_hits.txt"
EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/external_production_ecstore_import_hits.txt"
COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE="${TMP_DIR}/completed_external_owner_module_alias_hits.txt"
ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt"
RUSTFS_LOCAL_COMPAT_OWNER_SELF_PATH_HITS_FILE="${TMP_DIR}/rustfs_local_compat_owner_self_path_hits.txt"
RUSTFS_ROOT_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_compat_relative_consumer_hits.txt"
@@ -1060,13 +1061,26 @@ fi
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' rustfs/src crates fuzz \
--glob '*.rs' \
--glob '!crates/ecstore/**' || true
--glob '!crates/ecstore/**' |
rg -v '^(crates/notify/src/lib\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs):' || true
) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE"
if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
report_failure "non-ECStore sources must keep raw ECStore facade subpaths behind local ecstore_* module aliases: $(paste -sd '; ' "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
crates/notify/src/lib.rs \
crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs || true
) >"$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE"
if [[ -s "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE" ]]; then
report_failure "completed external owner roots must expose explicit ECStore symbols instead of ecstore_* module aliases: $(paste -sd '; ' "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \