mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: expose external ECStore owner symbols (#3748)
This commit is contained in:
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{
|
||||
Event, NotificationError, ecstore_config, ecstore_global, ecstore_storage, registry::TargetRegistry,
|
||||
rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade,
|
||||
Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
|
||||
runtime_facade::NotifyRuntimeFacade,
|
||||
};
|
||||
use rustfs_config::notify::{
|
||||
NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
|
||||
@@ -31,8 +31,6 @@ const LOG_SUBSYSTEM_CONFIG: &str = "config";
|
||||
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
|
||||
const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
|
||||
|
||||
type NotifyStore = ecstore_storage::ECStore;
|
||||
|
||||
#[derive(Debug)]
|
||||
enum NotifyConfigStoreError {
|
||||
StorageNotAvailable,
|
||||
@@ -48,33 +46,21 @@ where
|
||||
return Err(NotifyConfigStoreError::StorageNotAvailable);
|
||||
};
|
||||
|
||||
let mut new_config = read_notify_server_config_without_migrate(store.clone()).await?;
|
||||
let mut new_config = crate::read_notify_server_config_without_migrate(store.clone())
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Read)?;
|
||||
|
||||
if !modifier(&mut new_config) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
save_notify_server_config(store, &new_config).await?;
|
||||
crate::save_notify_server_config(store, &new_config)
|
||||
.await
|
||||
.map_err(NotifyConfigStoreError::Save)?;
|
||||
|
||||
Ok(Some(new_config))
|
||||
}
|
||||
|
||||
fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
|
||||
ecstore_global::resolve_object_store_handle()
|
||||
}
|
||||
|
||||
async fn read_notify_server_config_without_migrate(store: Arc<NotifyStore>) -> Result<Config, NotifyConfigStoreError> {
|
||||
ecstore_config::com::read_config_without_migrate(store)
|
||||
.await
|
||||
.map_err(|err| NotifyConfigStoreError::Read(err.to_string()))
|
||||
}
|
||||
|
||||
async fn save_notify_server_config(store: Arc<NotifyStore>, config: &Config) -> Result<(), NotifyConfigStoreError> {
|
||||
ecstore_config::com::save_server_config(store, config)
|
||||
.await
|
||||
.map_err(|err| NotifyConfigStoreError::Save(err.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn notify_configuration_hint() -> String {
|
||||
let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
|
||||
let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT);
|
||||
|
||||
@@ -18,6 +18,8 @@
|
||||
//! It supports sending events to various targets
|
||||
//! (like Webhook and MQTT) and includes features like event persistence and retry on failure.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
mod bucket_config_manager;
|
||||
mod config_manager;
|
||||
mod error;
|
||||
@@ -37,9 +39,7 @@ mod runtime_view;
|
||||
mod services;
|
||||
mod status_view;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::config as ecstore_config;
|
||||
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
|
||||
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
|
||||
|
||||
pub use bucket_config_manager::NotifyBucketConfigManager;
|
||||
pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
|
||||
@@ -58,3 +58,24 @@ pub use runtime_facade::NotifyRuntimeFacade;
|
||||
pub use runtime_view::NotifyRuntimeView;
|
||||
pub use services::NotifyServices;
|
||||
pub use status_view::NotifyStatusView;
|
||||
|
||||
pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
|
||||
rustfs_ecstore::api::global::resolve_object_store_handle()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_notify_server_config_without_migrate(
|
||||
store: Arc<NotifyStore>,
|
||||
) -> Result<rustfs_config::server_config::Config, String> {
|
||||
rustfs_ecstore::api::config::com::read_config_without_migrate(store)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub(crate) async fn save_notify_server_config(
|
||||
store: Arc<NotifyStore>,
|
||||
config: &rustfs_config::server_config::Config,
|
||||
) -> Result<(), String> {
|
||||
rustfs_ecstore::api::config::com::save_server_config(store, config)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
@@ -35,10 +35,13 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
|
||||
};
|
||||
use rustfs_ecstore::api::error::Result as SwiftStorageResult;
|
||||
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_swift_object_store_handle;
|
||||
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
|
||||
|
||||
pub mod account;
|
||||
pub mod acl;
|
||||
@@ -70,22 +73,15 @@ pub use router::{SwiftRoute, SwiftRouter};
|
||||
#[allow(unused_imports)]
|
||||
pub use types::{Container, Object, SwiftMetadata};
|
||||
|
||||
type SwiftBucketMetadata = ecstore_bucket::metadata::BucketMetadata;
|
||||
type SwiftStorageResult<T> = ecstore_error::Result<T>;
|
||||
type SwiftStore = ecstore_storage::ECStore;
|
||||
pub type SwiftGetObjectReader = <SwiftStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
|
||||
pub type SwiftObjectInfo = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
pub type SwiftObjectOptions = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
pub type SwiftPutObjReader = <SwiftStore as rustfs_storage_api::ObjectIO>::PutObjectReader;
|
||||
|
||||
fn resolve_swift_object_store_handle() -> Option<Arc<SwiftStore>> {
|
||||
ecstore_global::resolve_object_store_handle()
|
||||
}
|
||||
|
||||
async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResult<Arc<SwiftBucketMetadata>> {
|
||||
ecstore_bucket::metadata_sys::get(bucket).await
|
||||
get_swift_bucket_metadata_from_backend(bucket).await
|
||||
}
|
||||
|
||||
async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
|
||||
ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, metadata).await
|
||||
set_swift_bucket_metadata_in_backend(bucket, metadata).await
|
||||
}
|
||||
|
||||
@@ -15,11 +15,10 @@
|
||||
use datafusion::{common::DataFusionError, sql::sqlparser::parser::ParserError};
|
||||
use snafu::{Backtrace, Location, Snafu};
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::error as ecstore_error;
|
||||
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
|
||||
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
pub(crate) use rustfs_ecstore::api::error::StorageError as SelectStorageError;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
|
||||
|
||||
pub mod object_store;
|
||||
pub mod query;
|
||||
@@ -30,6 +29,28 @@ mod test;
|
||||
|
||||
pub type QueryResult<T> = Result<T, QueryError>;
|
||||
|
||||
pub(crate) type SelectGetObjectReader = <SelectStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
|
||||
pub(crate) const SELECT_DEFAULT_READ_BUFFER_SIZE: usize = rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
|
||||
pub(crate) fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
|
||||
rustfs_ecstore::api::global::resolve_object_store_handle()
|
||||
}
|
||||
|
||||
pub(crate) fn select_is_err_bucket_not_found(err: &SelectStorageError) -> bool {
|
||||
rustfs_ecstore::api::error::is_err_bucket_not_found(err)
|
||||
}
|
||||
|
||||
pub(crate) fn select_is_err_object_not_found(err: &SelectStorageError) -> bool {
|
||||
rustfs_ecstore::api::error::is_err_object_not_found(err)
|
||||
}
|
||||
|
||||
pub(crate) fn select_is_err_version_not_found(err: &SelectStorageError) -> bool {
|
||||
rustfs_ecstore::api::error::is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
#[derive(Debug, Snafu)]
|
||||
#[snafu(visibility(pub))]
|
||||
pub enum QueryError {
|
||||
|
||||
@@ -12,7 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{ecstore_error, ecstore_global, ecstore_set_disk, ecstore_storage};
|
||||
use crate::{
|
||||
SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions, SelectStorageError,
|
||||
SelectStore, resolve_select_object_store_handle, select_is_err_bucket_not_found, select_is_err_object_not_found,
|
||||
select_is_err_version_not_found,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use chrono::Utc;
|
||||
@@ -45,34 +49,10 @@ use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio_util::io::ReaderStream;
|
||||
use transform_stream::AsyncTryStream;
|
||||
|
||||
type SelectStorageError = ecstore_error::StorageError;
|
||||
type SelectStore = ecstore_storage::ECStore;
|
||||
type SelectGetObjectReader = <SelectStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
|
||||
type SelectObjectInfo = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
type SelectObjectOptions = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
|
||||
const SELECT_DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
|
||||
fn select_default_read_buffer_size_u64() -> u64 {
|
||||
u64::try_from(SELECT_DEFAULT_READ_BUFFER_SIZE).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
|
||||
ecstore_global::resolve_object_store_handle()
|
||||
}
|
||||
|
||||
fn select_is_err_bucket_not_found(err: &SelectStorageError) -> bool {
|
||||
ecstore_error::is_err_bucket_not_found(err)
|
||||
}
|
||||
|
||||
fn select_is_err_object_not_found(err: &SelectStorageError) -> bool {
|
||||
ecstore_error::is_err_object_not_found(err)
|
||||
}
|
||||
|
||||
fn select_is_err_version_not_found(err: &SelectStorageError) -> bool {
|
||||
ecstore_error::is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
/// Maximum allowed object size for JSON DOCUMENT mode.
|
||||
///
|
||||
/// JSON DOCUMENT format requires loading the entire file into memory for DOM
|
||||
|
||||
@@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-external-owner-ecstore-boundaries`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130`.
|
||||
- Based on: API-130 slice.
|
||||
- Branch: `overtrue/arch-external-owner-facade-symbols`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131`.
|
||||
- Based on: API-131 slice.
|
||||
- PR type for this branch: `pure-move`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: move nested external production ECStore facade imports for
|
||||
notify, observability, and S3 Select behind crate or module owner roots.
|
||||
- CI/script changes: reject nested external production `rustfs_ecstore::api`
|
||||
imports outside the approved crate or module owner roots.
|
||||
- Docs changes: record the API-131 external owner boundary cleanup.
|
||||
- Rust code changes: replace completed external owner root ECStore module
|
||||
aliases for notify, Swift, and S3 Select with explicit local symbols and
|
||||
wrapper functions.
|
||||
- CI/script changes: reject restored `ecstore_*` module aliases in completed
|
||||
external owner roots while allowing explicit owner-root facade symbols.
|
||||
- Docs changes: record the API-132 external owner facade symbol cleanup.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -3878,6 +3879,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
scan, branch freshness check, pre-commit quality gate, and three-expert
|
||||
review.
|
||||
|
||||
- [x] `API-132` Replace completed external owner module aliases with symbols.
|
||||
- Do: replace notify, Swift, and S3 Select owner-root `ecstore_*` module
|
||||
aliases with explicit local ECStore symbols, type aliases, constants, and
|
||||
wrapper functions.
|
||||
- Acceptance: completed external owner roots no longer expose broad
|
||||
`ecstore_*` module aliases, while nested modules keep using owner-local
|
||||
symbols and the remaining larger observability, IAM, scanner, and heal
|
||||
owner roots stay unchanged for later slices.
|
||||
- Must preserve: notify config persistence, Swift bucket metadata access, S3
|
||||
Select object-store error mapping, object reads, scan buffering, and all
|
||||
public crate APIs.
|
||||
- Verification: focused notify/Swift/S3 Select compile, completed-owner
|
||||
alias residual scan, migration/layer guards, formatting, diff hygiene, Rust
|
||||
risk scan, branch freshness check, pre-commit quality gate, and
|
||||
three-expert review.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `pure-move`: continue pruning remaining facade compatibility and owner boundaries.
|
||||
@@ -3886,14 +3903,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | pass | API-131 keeps direct ECStore facade imports at external owner roots and leaves nested production modules on local aliases. |
|
||||
| Migration preservation | pass | Notify, observability, and S3 Select call paths keep the same ECStore symbols through owner-root aliases only. |
|
||||
| Testing/verification | pass | Focused notify/obs/S3 Select compile, nested direct-import residual scan, migration/layer guards, formatting, diff hygiene, and diff-only Rust risk scan passed. |
|
||||
| Quality/architecture | pass | API-132 replaces completed notify, Swift, and S3 Select owner-root ECStore module aliases with explicit local symbols and wrappers. |
|
||||
| Migration preservation | pass | Notify config persistence, Swift bucket metadata access, and S3 Select object-store behavior keep the same ECStore call paths through owner roots. |
|
||||
| Testing/verification | pass | Focused notify/Swift/S3 Select compile, completed-owner alias residual scan, migration/layer guards, formatting, diff hygiene, full pre-commit, and diff-only Rust risk scan passed. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed before push:
|
||||
|
||||
- Issue #660 API-132 current slice:
|
||||
- `cargo check --tests -p rustfs-notify -p rustfs-s3select-api -p rustfs-protocols --features rustfs-protocols/swift`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `make pre-commit`: passed, including clippy, script tests, nextest
|
||||
`6518 passed, 111 skipped`, and doc-tests.
|
||||
- Completed external owner module-alias residual scan: passed.
|
||||
- Rust risk scan: diff-only scan found explicit `as` symbol imports only; no
|
||||
new unwrap/expect, panic/todo/unsafe, or risky behavior added.
|
||||
|
||||
- Issue #660 API-131 current slice:
|
||||
- `cargo check --tests -p rustfs-notify -p rustfs-obs -p rustfs-s3select-api`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
|
||||
@@ -100,6 +100,7 @@ ALL_STORAGE_COMPAT_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_storage_compa
|
||||
ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_ecstore_api_grouped_facade_import_hits.txt"
|
||||
ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE="${TMP_DIR}/all_ecstore_api_raw_subpath_hits.txt"
|
||||
EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/external_production_ecstore_import_hits.txt"
|
||||
COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE="${TMP_DIR}/completed_external_owner_module_alias_hits.txt"
|
||||
ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt"
|
||||
RUSTFS_LOCAL_COMPAT_OWNER_SELF_PATH_HITS_FILE="${TMP_DIR}/rustfs_local_compat_owner_self_path_hits.txt"
|
||||
RUSTFS_ROOT_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_compat_relative_consumer_hits.txt"
|
||||
@@ -1060,13 +1061,26 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' rustfs/src crates fuzz \
|
||||
--glob '*.rs' \
|
||||
--glob '!crates/ecstore/**' || true
|
||||
--glob '!crates/ecstore/**' |
|
||||
rg -v '^(crates/notify/src/lib\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs):' || true
|
||||
) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE"
|
||||
|
||||
if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
|
||||
report_failure "non-ECStore sources must keep raw ECStore facade subpaths behind local ecstore_* module aliases: $(paste -sd '; ' "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
|
||||
crates/notify/src/lib.rs \
|
||||
crates/protocols/src/swift/mod.rs \
|
||||
crates/s3select-api/src/lib.rs || true
|
||||
) >"$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE"
|
||||
|
||||
if [[ -s "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE" ]]; then
|
||||
report_failure "completed external owner roots must expose explicit ECStore symbols instead of ecstore_* module aliases: $(paste -sd '; ' "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
|
||||
|
||||
Reference in New Issue
Block a user