diff --git a/crates/e2e_test/src/storage_compat.rs b/crates/e2e_test/src/storage_compat.rs index 93a09ae6f..68b701de4 100644 --- a/crates/e2e_test/src/storage_compat.rs +++ b/crates/e2e_test/src/storage_compat.rs @@ -19,6 +19,27 @@ pub(crate) type TonicInterceptor = rustfs_ecstore::api::rpc::TonicInterceptor; pub(crate) type VolumeInfo = rustfs_ecstore::api::disk::VolumeInfo; pub(crate) type WalkDirOptions = rustfs_ecstore::api::disk::WalkDirOptions; -pub(crate) use rustfs_ecstore::api::rpc::{ - gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, -}; +pub(crate) use rustfs_ecstore::api::rpc::gen_tonic_signature_interceptor; + +pub(crate) async fn node_service_time_out_client( + addr: &String, + interceptor: TonicInterceptor, +) -> Result< + rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient< + tonic::service::interceptor::InterceptedService, + >, + Box, +> { + rustfs_ecstore::api::rpc::node_service_time_out_client(addr, interceptor).await +} + +pub(crate) async fn node_service_time_out_client_no_auth( + addr: &String, +) -> Result< + rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient< + tonic::service::interceptor::InterceptedService, + >, + Box, +> { + rustfs_ecstore::api::rpc::node_service_time_out_client_no_auth(addr).await +} diff --git a/crates/notify/src/storage_compat.rs b/crates/notify/src/storage_compat.rs index bed0215d8..5aa86e6bf 100644 --- a/crates/notify/src/storage_compat.rs +++ b/crates/notify/src/storage_compat.rs @@ -13,7 +13,9 @@ // limitations under the License. use rustfs_config::server_config::Config; -use rustfs_ecstore::api::{config, global}; +use std::sync::Arc; + +type NotifyStore = rustfs_ecstore::api::storage::ECStore; #[derive(Debug)] pub(crate) enum NotifyConfigStoreError { @@ -26,21 +28,33 @@ pub(crate) async fn update_server_config(mut modifier: F) -> Result bool, { - let Some(store) = global::resolve_object_store_handle() else { + let Some(store) = resolve_notify_object_store_handle() else { return Err(NotifyConfigStoreError::StorageNotAvailable); }; - let mut new_config = config::com::read_config_without_migrate(store.clone()) - .await - .map_err(|err| NotifyConfigStoreError::Read(err.to_string()))?; + let mut new_config = read_notify_server_config_without_migrate(store.clone()).await?; if !modifier(&mut new_config) { return Ok(None); } - config::com::save_server_config(store, &new_config) - .await - .map_err(|err| NotifyConfigStoreError::Save(err.to_string()))?; + save_notify_server_config(store, &new_config).await?; Ok(Some(new_config)) } + +fn resolve_notify_object_store_handle() -> Option> { + rustfs_ecstore::api::global::resolve_object_store_handle() +} + +async fn read_notify_server_config_without_migrate(store: Arc) -> Result { + rustfs_ecstore::api::config::com::read_config_without_migrate(store) + .await + .map_err(|err| NotifyConfigStoreError::Read(err.to_string())) +} + +async fn save_notify_server_config(store: Arc, config: &Config) -> Result<(), NotifyConfigStoreError> { + rustfs_ecstore::api::config::com::save_server_config(store, config) + .await + .map_err(|err| NotifyConfigStoreError::Save(err.to_string())) +} diff --git a/crates/obs/src/storage_compat.rs b/crates/obs/src/storage_compat.rs index 4fdc9f8cb..0bc30426f 100644 --- a/crates/obs/src/storage_compat.rs +++ b/crates/obs/src/storage_compat.rs @@ -17,14 +17,62 @@ use crate::metrics::collectors::{ ReplicationStats as MetricReplicationStats, }; use rustfs_storage_api::StorageAdminApi; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -pub(crate) use rustfs_ecstore::api::data_usage::load_data_usage_from_backend as load_obs_data_usage_from_backend; - pub(crate) type ObsStore = rustfs_ecstore::api::storage::ECStore; type ObsStorageInfo = ::StorageInfo; +pub(crate) struct ObsDataUsageInfo { + pub(crate) buckets_count: u64, + pub(crate) objects_total_count: u64, + pub(crate) versions_total_count: u64, + pub(crate) delete_markers_total_count: u64, + pub(crate) objects_total_size: u64, + pub(crate) buckets_usage: HashMap, +} + +pub(crate) struct ObsBucketUsageInfo { + pub(crate) size: u64, + pub(crate) objects_count: u64, + pub(crate) object_size_histogram: HashMap, + pub(crate) object_versions_histogram: HashMap, + pub(crate) versions_count: u64, + pub(crate) delete_markers_count: u64, +} + +pub(crate) async fn load_obs_data_usage_from_backend( + store: Arc, +) -> rustfs_ecstore::api::error::Result { + let data_usage = rustfs_ecstore::api::data_usage::load_data_usage_from_backend(store).await?; + + Ok(ObsDataUsageInfo { + buckets_count: data_usage.buckets_count, + objects_total_count: data_usage.objects_total_count, + versions_total_count: data_usage.versions_total_count, + delete_markers_total_count: data_usage.delete_markers_total_count, + objects_total_size: data_usage.objects_total_size, + buckets_usage: data_usage + .buckets_usage + .into_iter() + .map(|(bucket, usage)| { + ( + bucket, + ObsBucketUsageInfo { + size: usage.size, + objects_count: usage.objects_count, + object_size_histogram: usage.object_size_histogram, + object_versions_histogram: usage.object_versions_histogram, + versions_count: usage.versions_count, + delete_markers_count: usage.delete_markers_count, + }, + ) + }) + .collect(), + }) +} + #[derive(Debug, Clone, PartialEq)] pub(crate) struct ObsBucketReplicationBandwidthStats { pub(crate) bucket: String, diff --git a/crates/scanner/src/storage_compat.rs b/crates/scanner/src/storage_compat.rs index 3f2b1c6a5..f226472e3 100644 --- a/crates/scanner/src/storage_compat.rs +++ b/crates/scanner/src/storage_compat.rs @@ -31,19 +31,21 @@ pub(crate) type ListPathRawOptions = rustfs_ecstore::api::cache::ListPathRawOpti pub(crate) type SetDisks = rustfs_ecstore::api::set_disk::SetDisks; pub(crate) type StorageError = rustfs_ecstore::api::error::StorageError; -pub(crate) use rustfs_ecstore::api::bucket::{ - bucket_target_sys::BucketTargetSys, - lifecycle::{ - bucket_lifecycle_audit::LcEventSrc, - bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule, apply_transition_rule}, - evaluator::Evaluator, - lifecycle::{Event, Lifecycle, ObjectOpts}, - }, - metadata_sys::{get_lifecycle_config, get_object_lock_config, get_replication_config}, - replication::{ReplicationConfig, ReplicationConfigurationExt, ReplicationQueueAdmission, queue_replication_heal_internal}, - versioning::VersioningApi, - versioning_sys::BucketVersioningSys, +pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{ + GLOBAL_ExpiryState, apply_expiry_rule, apply_transition_rule, }; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{Event, Lifecycle, ObjectOpts}; +pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ + get_lifecycle_config, get_object_lock_config, get_replication_config, +}; +pub(crate) use rustfs_ecstore::api::bucket::replication::{ + ReplicationConfig, ReplicationConfigurationExt, ReplicationQueueAdmission, queue_replication_heal_internal, +}; +pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi; +pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskInfoOptions}; pub(crate) use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr; diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 41f62bee4..6b0e9a527 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -162,6 +162,13 @@ storage-class config contracts through explicit aliases. The storage compatibility boundary must not restore broad `metadata`, `metadata_sys`, `object_lock`, `policy_sys`, `replication`, `tagging`, `utils`, `versioning`, `versioning_sys`, `object_api_utils`, or `com` passthroughs. +Scanner, notify, observability, and e2e `storage_compat.rs` boundaries must +also stay narrow. Scanner must not restore grouped bucket compatibility exports +for target, lifecycle, metadata, replication, or versioning modules. Notify +must not restore broad `config`/`global` module imports. Observability must +consume data usage through a local DTO projection instead of re-exporting the +ECStore data-usage loader. The e2e harness must not restore grouped RPC +passthroughs. ECStore ClusterControlPlane read models must stay owned by the crate-private `cluster` module. Public access goes through `rustfs_ecstore::api::cluster` so diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 78d4d6752..d2e5a42bb 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-admin-app-compat-aliases` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082`. -- Stacked on: API-082 storage compatibility alias pruning. +- Branch: `overtrue/arch-edge-compat-alias-prune` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083`. +- Stacked on: API-083 admin/app compatibility alias pruning. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: prune admin/app bucket-facing compatibility passthroughs - into explicit local compatibility modules and aliases. -- CI/script changes: guard against restoring broad admin/app bucket, client, - and storage-class compatibility passthroughs. -- Docs changes: record the API-083 admin/app compatibility alias boundary. +- Rust code changes: prune scanner, notify, observability, and e2e edge + compatibility passthroughs into explicit aliases, wrappers, and local DTOs. +- CI/script changes: guard against restoring broad edge compatibility + passthroughs. +- Docs changes: record the API-084 edge compatibility alias boundary. ## Phase 0 Tasks @@ -187,6 +187,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block - Verification: RustFS compile coverage, admin/app compatibility residual scans, formatting, diff hygiene, risk scan, architecture guard, pre-commit quality gate, and three-expert review. +- [x] `API-084` Prune edge compatibility passthrough aliases. + - Completed slice: replace scanner grouped bucket compatibility exports, + notify broad config/global imports, observability data-usage passthroughs, + and e2e grouped RPC passthroughs with explicit edge-local aliases, + wrappers, and DTO projections. + - Acceptance: scanner bucket contracts stay explicitly named, notify config + persistence routes through local wrappers, observability metrics consume a + local data-usage DTO, and e2e RPC helper access stays narrow. + - Must preserve: scanner lifecycle and replication behavior, notification + server-config update semantics, observability cluster/bucket usage metrics, + and e2e RPC client/interceptor call sites. + - Verification: focused edge crate compile coverage, edge compatibility + residual scans, formatting, diff hygiene, risk scan, architecture guard, + pre-commit quality gate, and three-expert review. - [x] `G-012` Inventory placement and repair invariants. - Acceptance: [`placement-repair-invariants.md`](placement-repair-invariants.md) records @@ -3220,14 +3234,24 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | API-083 narrows admin/app bucket-facing compatibility passthroughs into explicit local aliases without adding new ECStore ownership cycles. | -| Migration preservation | passed | Admin replication, site-replication metadata, bucket targets, quota, tier stats, app bucket/object/multipart/lifecycle, ETag conversion, and storage-class behavior remain preserved. | -| Testing/verification | passed | RustFS compile coverage, admin/app compatibility residual scans, migration guard, formatting, diff hygiene, added-line risk scan, full pre-commit, and three-expert review passed. | +| Quality/architecture | passed | API-084 narrows scanner/notify/obs/e2e edge compatibility passthroughs without adding ECStore ownership cycles or new runtime dependencies. | +| Migration preservation | passed | Scanner lifecycle/replication helpers, notify config persistence, obs usage metrics, and e2e RPC helper behavior remain preserved behind local aliases/wrappers. | +| Testing/verification | passed | Focused edge compile coverage, compatibility residual scans, migration guard, formatting, diff hygiene, added-line risk scan, full pre-commit, and three-expert review passed. | ## Verification Notes Passed before push: +- Issue #660 API-084 current slice: + - `cargo check --tests -p rustfs-scanner -p rustfs-notify -p rustfs-obs -p e2e_test`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - Scanner/notify/obs/e2e broad compatibility residual scans: passed. + - Rust added-line risk scan on changed Rust files and guard script: passed. + - `make pre-commit`: passed. + - Issue #660 API-083 current slice: - `cargo check -p rustfs --lib`: passed. - `cargo fmt --all --check`: passed. diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 0ebcd5956..1bed791c7 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -84,6 +84,10 @@ RUSTFS_ADMIN_CONFIG_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_admin_con RUSTFS_STORAGE_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_storage_bucket_storage_compat_module_hits.txt" RUSTFS_ADMIN_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_admin_bucket_storage_compat_module_hits.txt" RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_app_bucket_storage_compat_module_hits.txt" +SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/scanner_bucket_storage_compat_module_hits.txt" +NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/notify_storage_compat_module_hits.txt" +OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/obs_storage_compat_passthrough_hits.txt" +E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE="${TMP_DIR}/e2e_storage_compat_rpc_passthrough_hits.txt" PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt" BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt" NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt" @@ -831,6 +835,46 @@ if [[ -s "$RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then report_failure "RustFS app storage compatibility must expose bucket/client/storageclass contracts as explicit aliases: $(paste -sd '; ' "$RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::bucket::\{[^}]*\b(?:bucket_target_sys|lifecycle|metadata_sys|replication|versioning|versioning_sys)\b[^}]*\}\s*;' \ + crates/scanner/src/storage_compat.rs || true +) >"$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" + +if [[ -s "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then + report_failure "scanner storage compatibility must expose bucket contracts as explicit aliases: $(paste -sd '; ' "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --no-heading 'use rustfs_ecstore::api::\{[^}]*\b(?:config|global)\b[^}]*\}\s*;' \ + crates/notify/src/storage_compat.rs || true +) >"$NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE" + +if [[ -s "$NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then + report_failure "notify storage compatibility must not import broad config/global modules: $(paste -sd '; ' "$NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::data_usage::load_data_usage_from_backend' \ + crates/obs/src/storage_compat.rs || true +) >"$OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE" + +if [[ -s "$OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE" ]]; then + report_failure "OBS storage compatibility must wrap data-usage access instead of re-exporting ECStore functions: $(paste -sd '; ' "$OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::rpc::\{[^}]*\b(?:gen_tonic_signature_interceptor|node_service_time_out_client|node_service_time_out_client_no_auth)\b[^}]*\}\s*;' \ + crates/e2e_test/src/storage_compat.rs || true +) >"$E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE" + +if [[ -s "$E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE" ]]; then + report_failure "e2e storage compatibility must wrap RPC helpers instead of re-exporting broad ECStore functions: $(paste -sd '; ' "$E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --no-heading '#!\[allow\(unused_imports\)\]' \