From 738b805ec045c512604d8bfb8c2097ed0f6c2b4c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 26 Jun 2026 14:45:05 +0800 Subject: [PATCH] refactor: segment app storage api boundary (#3902) --- docs/architecture/migration-progress.md | 59 +++++++++++++---- rustfs/src/app/admin_usecase.rs | 12 ++-- rustfs/src/app/bucket_usecase.rs | 24 +++---- rustfs/src/app/capacity_dirty_scope_test.rs | 10 +-- rustfs/src/app/context.rs | 10 +-- rustfs/src/app/context/global.rs | 4 +- rustfs/src/app/context/handles.rs | 6 +- rustfs/src/app/context/interfaces.rs | 6 +- rustfs/src/app/context/runtime_sources.rs | 8 +-- rustfs/src/app/context/startup.rs | 2 +- .../src/app/lifecycle_transition_api_test.rs | 14 ++-- rustfs/src/app/multipart_usecase.rs | 36 +++++----- rustfs/src/app/object_usecase.rs | 54 +++++++-------- rustfs/src/app/select_object.rs | 8 +-- rustfs/src/app/storage_api.rs | 65 +++++++++++++++++++ scripts/check_architecture_migration_rules.sh | 24 +++++-- 16 files changed, 230 insertions(+), 112 deletions(-) diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index a447796a9..d004b08b4 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,15 +5,13 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-root-storage-api-domain-boundary` +- Branch: `overtrue/arch-app-storage-api-domain-boundary` - Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/CTX-002`. -- Based on: PR #3899 has merged; branch segments the root-local and admin-local - storage API boundaries by consumer domain. +- Based on: rebased onto current `origin/main` after PR #3901 merged. - PR type for this branch: `consumer-migration` -- Runtime behavior changes: none expected for API-237/API-238; root, server, - startup, table, protocol, cluster, capacity, workload, config-test, error, - admin handler, admin service, and admin router consumers still use the same - owner symbols through narrower domain modules. +- Runtime behavior changes: none expected for API-239; app bucket, object, + multipart, admin, select, context, and test consumers still use the same owner + symbols through narrower domain modules. - Rust code changes: route replication pool, outbound TLS generation, runtime region, KMS encryption service, runtime support handles, S3 Select DB, internode RPC metrics, IAM authorization/handler reads, notification @@ -76,7 +74,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block table, protocols, capacity, workload, config tests, and error mapping, and admin storage API consumers through admin domain modules for access, bucket, cluster, config, contract, error, metrics, object, rebalance, runtime, and - tier boundaries. + tier boundaries, plus app storage API consumers through app domain modules + for admin, bucket, object, multipart, select, context, and test boundaries. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade @@ -86,10 +85,11 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block event-bridge thin module regressions, plus IAM runtime-source bypasses; accept the reviewed AppContext resolver reverse dependencies in the layer baseline, and block direct admin AppContext resolver consumers outside the - admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, reject storage owner direct ECStore/storage-api imports outside the owner-local `storage_api` boundary, reject ECStore internal direct storage-api imports outside the owner-local `storage_api_contracts` boundary, reject direct storage ECFS app usecase construction outside the storage S3 API boundary, reject flat root `storage_api` imports outside the new root-local domain modules, and reject flat admin `storage_api` imports outside the new admin domain modules. -- Docs changes: record the API-136 through API-238 owner facade, + admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, reject storage owner direct ECStore/storage-api imports outside the owner-local `storage_api` boundary, reject ECStore internal direct storage-api imports outside the owner-local `storage_api_contracts` boundary, reject direct storage ECFS app usecase construction outside the storage S3 API boundary, reject flat root `storage_api` imports outside the new root-local domain modules, reject flat admin `storage_api` imports outside the new admin domain modules, and reject flat app `storage_api` imports outside the new app consumer-domain modules. +- Docs changes: record the API-136 through API-239 owner facade, runtime-source, ECFS usecase, root storage API domain-boundary, and admin - storage API domain-boundary cleanup. + storage API domain-boundary cleanup, and app storage API consumer-domain + cleanup. ## Phase 0 Tasks @@ -5494,15 +5494,34 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block guards, diff hygiene, residual flat admin storage-api scan, Rust risk scan, and full PR gate passed before PR. +- [x] `API-239` Segment app storage API boundary by app consumer domain. + - Do: add app-local consumer-domain modules for admin, bucket, object, + multipart, select, context, and test storage API consumers, then migrate app + usecases, context modules, and app tests away from flat `app::storage_api` + imports. + - Acceptance: app consumers no longer import flat storage facade symbols, + storage contracts, runtime handles, request helpers, S3 helpers, bucket + helpers, IO helpers, SSE helpers, or test harness symbols directly from the + app storage API root; migration rules reject the old flat paths. + - Must preserve: app admin info/capacity behavior, bucket/object/multipart S3 + contracts, select object reads, AppContext runtime handles, test disk + setup, lifecycle transition test helpers, and capacity dirty-scope fixtures. + - Verification: focused RustFS compile, formatting, migration and layer + guards, diff hygiene, residual flat app storage-api scan, Rust risk scan, + and full PR gate passed before PR. + ## Next PRs 1. `consumer-migration`: continue larger root and owner boundary batches after - API-238. + API-239. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-239 segments the app-local storage API boundary into app consumer domain modules instead of a flat re-export surface. | +| Migration preservation | pass | App admin, bucket, object, multipart, select, context, lifecycle-transition, and capacity dirty-scope consumers keep the same owner symbols and call paths. | +| Testing/verification | pass | RustFS focused compile, formatting, migration/layer guards, residual flat app storage-api scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. | | Quality/architecture | pass | API-238 segments the admin-local storage API boundary into admin consumer domain modules instead of a flat re-export surface. | | Migration preservation | pass | Admin handlers, services, router, console paths, config persistence, bucket metadata, replication, quota, heal, metrics, rebalance, tier, and object zip consumers keep the same owner symbols and call paths. | | Testing/verification | pass | RustFS focused compile, formatting, migration/layer guards, residual flat admin storage-api scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. | @@ -5758,6 +5777,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: +- Issue #660 API-239 current slice: + - Branch freshness check: rebased onto current `origin/main` after PR #3901 + merged. + - `cargo check -p rustfs`: passed. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Flat app storage-api residual scan: passed; app consumers now use + `storage_api` consumer-domain modules instead of flat imports. + - Diff-added Rust risk scan: passed; no new production unwrap/expect, + numeric cast, String error, Box dyn Error, print macro, or relaxed atomic + ordering lines. + - `make pre-pr`: passed. + - Issue #660 API-238 current slice: - Branch freshness check: rebased onto current `origin/main` after PR #3899 merged. - `cargo check -p rustfs`: passed. diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index ce343470f..8a45b7628 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -14,13 +14,13 @@ //! Admin application use-case contracts. -use super::storage_api::StorageAdminApi; -use super::storage_api::admin::get_server_info; -use super::storage_api::capacity::{ +use super::storage_api::admin_usecase::StorageAdminApi; +use super::storage_api::admin_usecase::admin::get_server_info; +use super::storage_api::admin_usecase::capacity::{ PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free, }; -use super::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; -use super::storage_api::{ECStore, EndpointServerPools}; +use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; +use super::storage_api::admin_usecase::{ECStore, EndpointServerPools}; use crate::app::runtime_sources::{ AppContext, current_app_context, resolve_endpoints_handle, resolve_object_store_handle_for_context, }; @@ -597,7 +597,7 @@ impl DefaultAdminUsecase { #[cfg(test)] mod tests { - use super::super::storage_api::capacity::{PoolDecommissionInfo, PoolStatus}; + use super::super::storage_api::admin_usecase::capacity::{PoolDecommissionInfo, PoolStatus}; use super::*; use time::OffsetDateTime; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index b8442f884..44f97d5e7 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -14,12 +14,12 @@ //! Bucket application use-case contracts. -use super::storage_api::ECStore; -use super::storage_api::StorageObjectInfo as ObjectInfo; -use super::storage_api::access::{ReqInfo, authorize_request, req_info_ref}; +use super::storage_api::bucket_usecase::ECStore; +use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo; +use super::storage_api::bucket_usecase::access::{ReqInfo, authorize_request, req_info_ref}; #[cfg(test)] -use super::storage_api::bucket::target::BucketTarget; -use super::storage_api::bucket::{ +use super::storage_api::bucket_usecase::bucket::target::BucketTarget; +use super::storage_api::bucket_usecase::bucket::{ ObjectLockConfigExt as _, VersioningConfigExt as _, bucket_target_sys::BucketTargetSys, lifecycle::bucket_lifecycle_ops::{ @@ -37,20 +37,20 @@ use super::storage_api::bucket::{ utils::serialize, versioning_sys::BucketVersioningSys, }; -use super::storage_api::data_usage::remove_bucket_usage_from_backend; -use super::storage_api::error::StorageError; -use super::storage_api::helper::{OperationHelper, spawn_background_with_context}; -use super::storage_api::object_utils::to_s3s_etag; -use super::storage_api::s3_api::bucket::{ +use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend; +use super::storage_api::bucket_usecase::error::StorageError; +use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context}; +use super::storage_api::bucket_usecase::object_utils::to_s3s_etag; +use super::storage_api::bucket_usecase::s3_api::bucket::{ ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, rustfs_owner, }; -use super::storage_api::{ +use super::storage_api::bucket_usecase::{ BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, MakeBucketOptions, }; -use super::storage_api::{ +use super::storage_api::bucket_usecase::{ get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, request_context, validate_list_object_unordered_with_delimiter, }; diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index 00634b3be..f67672766 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_api::bucket::metadata_sys; -use super::storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; -use super::storage_api::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; +use super::storage_api::test::bucket::metadata_sys; +use super::storage_api::test::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; +use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; use serial_test::serial; @@ -30,7 +30,7 @@ use tokio::fs; use tokio_util::sync::CancellationToken; use uuid::Uuid; -use super::storage_api::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; +use super::storage_api::test::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; static CAPACITY_DIRTY_SCOPE_ENV: OnceLock<(Vec, Arc, TempDir)> = OnceLock::new(); static CAPACITY_DIRTY_SCOPE_INIT: Once = Once::new(); @@ -78,7 +78,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec, Arc) { }; let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); - super::storage_api::runtime::init_local_disks(endpoint_pools.clone()) + super::storage_api::test::runtime::init_local_disks(endpoint_pools.clone()) .await .unwrap(); diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index e4ab21c3c..82e64044d 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -26,12 +26,12 @@ pub use global::*; pub use handles::*; pub use interfaces::*; -use super::storage_api::bucket::metadata_sys::BucketMetadataSys; -use super::storage_api::runtime::{ +use super::storage_api::context::bucket::metadata_sys::BucketMetadataSys; +use super::storage_api::context::runtime::{ BucketBandwidthMonitor, DailyAllTierStats, DynReplicationPool, ExpiryState, NotificationSys, ReplicationStats, ScannerMetricsReport, StorageClassConfig, TierConfigMgr, }; -use super::storage_api::{ECStore, EndpointServerPools}; +use super::storage_api::context::{ECStore, EndpointServerPools}; use crate::config::RustFSBufferConfig; use rustfs_config::server_config::Config; use rustfs_credentials::Credentials; @@ -566,8 +566,8 @@ fn resolve_buffer_config_with( #[cfg(test)] mod tests { - use super::super::storage_api::runtime::init_local_disks; - use super::super::storage_api::{Endpoint, Endpoints, PoolEndpoints}; + use super::super::storage_api::context::runtime::init_local_disks; + use super::super::storage_api::context::{Endpoint, Endpoints, PoolEndpoints}; use super::*; use crate::app::context::global::AppContextTestInterfaces; use crate::app::context::handles::{ diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index e3d0a3909..b37a1bf4e 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_api::ECStore; -use super::super::storage_api::runtime::set_object_store_resolver; +use super::super::storage_api::context::ECStore; +use super::super::storage_api::context::runtime::set_object_store_resolver; use super::handles::{ IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface, default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface, diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 8be06c376..5f8b46432 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_api::EndpointServerPools; -use super::super::storage_api::bucket::metadata_sys::BucketMetadataSys; -use super::super::storage_api::runtime::{ +use super::super::storage_api::context::EndpointServerPools; +use super::super::storage_api::context::bucket::metadata_sys::BucketMetadataSys; +use super::super::storage_api::context::runtime::{ BucketBandwidthMonitor, DailyAllTierStats, DynReplicationPool, ExpiryState, NotificationSys, ReplicationStats, ScannerMetricsReport, StorageClassConfig, TierConfigMgr, }; diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index 722f5771a..8f1687e19 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_api::EndpointServerPools; -use super::super::storage_api::bucket::metadata_sys::BucketMetadataSys; -use super::super::storage_api::runtime::{ +use super::super::storage_api::context::EndpointServerPools; +use super::super::storage_api::context::bucket::metadata_sys::BucketMetadataSys; +use super::super::storage_api::context::runtime::{ BucketBandwidthMonitor, DailyAllTierStats, DynReplicationPool, ExpiryState, NotificationSys, ReplicationStats, ScannerMetricsReport, StorageClassConfig, TierConfigMgr, }; diff --git a/rustfs/src/app/context/runtime_sources.rs b/rustfs/src/app/context/runtime_sources.rs index a1cc8a52f..9ad6408d6 100644 --- a/rustfs/src/app/context/runtime_sources.rs +++ b/rustfs/src/app/context/runtime_sources.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_api::bucket::metadata_sys::BucketMetadataSys; -use super::super::storage_api::runtime::{ +use super::super::storage_api::context::bucket::metadata_sys::BucketMetadataSys; +use super::super::storage_api::context::runtime::{ BucketBandwidthMonitor, DailyAllTierStats, DynReplicationPool, ExpiryState, NotificationSys, ReplicationStats, ScannerMetricsReport, StorageClassConfig, TierConfigMgr, collect_scanner_metrics_report, get_daily_all_tier_stats, get_global_boot_time, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_expiry_state, get_global_lock_client, get_global_lock_clients, get_global_notification_sys, get_global_region, get_global_replication_pool, get_global_replication_stats, get_global_tier_config_mgr, global_rustfs_port, new_object_layer_fn, set_global_storage_class, }; -use super::super::storage_api::{ECStore, EndpointServerPools}; +use super::super::storage_api::context::{ECStore, EndpointServerPools}; use crate::config::{RustFSBufferConfig, get_global_buffer_config}; use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config}; use rustfs_credentials::{Credentials, get_global_action_cred}; @@ -108,7 +108,7 @@ pub fn notification_system() -> Option<&'static NotificationSys> { } pub fn bucket_metadata() -> Option>> { - super::super::storage_api::bucket::metadata_sys::get_global_bucket_metadata_sys() + super::super::storage_api::context::bucket::metadata_sys::get_global_bucket_metadata_sys() } pub fn bucket_monitor() -> Option> { diff --git a/rustfs/src/app/context/startup.rs b/rustfs/src/app/context/startup.rs index 0bb78d262..f726c82be 100644 --- a/rustfs/src/app/context/startup.rs +++ b/rustfs/src/app/context/startup.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_api::ECStore; +use super::super::storage_api::context::ECStore; use super::global::{AppContext, get_global_app_context, init_global_app_context}; use super::runtime_sources; use rustfs_kms::KmsServiceManager; diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index b6e10d693..130254c1e 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_api::bucket::{ +use super::storage_api::test::bucket::{ lifecycle, metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG}, metadata_sys, transition_api::{ReadCloser, ReaderImpl}, }; -use super::storage_api::ecfs::FS; -use super::storage_api::object_utils::to_s3s_etag; -use super::storage_api::runtime::{AppWarmBackend, TierConfig, TierType, WarmBackendGetOpts}; -use super::storage_api::{ +use super::storage_api::test::ecfs::FS; +use super::storage_api::test::object_utils::to_s3s_etag; +use super::storage_api::test::runtime::{AppWarmBackend, TierConfig, TierType, WarmBackendGetOpts}; +use super::storage_api::test::{ BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _, }; -use super::storage_api::{ +use super::storage_api::test::{ ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, }; @@ -109,7 +109,7 @@ async fn setup_test_env() -> (Vec, Arc) { let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); - super::storage_api::runtime::init_local_disks(endpoint_pools.clone()) + super::storage_api::test::runtime::init_local_disks(endpoint_pools.clone()) .await .unwrap(); diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 28b024a93..2aab93695 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -14,45 +14,45 @@ //! Multipart application use-case contracts. -use super::storage_api::ECStore; +use super::storage_api::multipart_usecase::ECStore; #[cfg(test)] -use super::storage_api::HTTPPreconditions; -use super::storage_api::access::has_bypass_governance_header; -use super::storage_api::bucket::quota::checker::QuotaChecker; -use super::storage_api::bucket::{ +use super::storage_api::multipart_usecase::HTTPPreconditions; +use super::storage_api::multipart_usecase::access::has_bypass_governance_header; +use super::storage_api::multipart_usecase::bucket::quota::checker::QuotaChecker; +use super::storage_api::multipart_usecase::bucket::{ lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate}, metadata_sys, quota::QuotaOperation, replication::{get_must_replicate_options, must_replicate, schedule_replication}, versioning_sys::BucketVersioningSys, }; -use super::storage_api::compression::is_disk_compressible; -use super::storage_api::data_usage::record_bucket_object_write_memory; -use super::storage_api::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; -use super::storage_api::helper::OperationHelper; +use super::storage_api::multipart_usecase::compression::is_disk_compressible; +use super::storage_api::multipart_usecase::data_usage::record_bucket_object_write_memory; +use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; +use super::storage_api::multipart_usecase::helper::OperationHelper; #[cfg(test)] -use super::storage_api::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; -use super::storage_api::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value}; -use super::storage_api::object_utils::to_s3s_etag; -use super::storage_api::options::{ +use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; +use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value}; +use super::storage_api::multipart_usecase::object_utils::to_s3s_etag; +use super::storage_api::multipart_usecase::options::{ copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts, parse_copy_source_range, put_opts, validate_archive_content_encoding, }; -use super::storage_api::s3_api::multipart::{ +use super::storage_api::multipart_usecase::s3_api::multipart::{ ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, }; -use super::storage_api::set_disk::is_valid_storage_class; -use super::storage_api::sse::{ +use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class; +use super::storage_api::multipart_usecase::sse::{ DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention, build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error, mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption, }; -use super::storage_api::{ +use super::storage_api::multipart_usecase::{ CompletePart, HTTPRangeSpec, MultipartOperations as _, MultipartUploadResult, ObjectIO as _, ObjectOperations as _, }; -use super::storage_api::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; +use super::storage_api::multipart_usecase::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write}; use crate::app::runtime_sources::{AppContext, current_app_context, resolve_object_store_handle_for_context}; use crate::capacity::record_capacity_write; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index ab1ae83d0..484c97429 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -15,12 +15,14 @@ //! Object application use-case contracts. // Performance metrics recording (with zero-copy-metrics integration) -use super::storage_api::ECStore; +use super::storage_api::object_usecase::ECStore; #[cfg(test)] -use super::storage_api::HTTPPreconditions; -use super::storage_api::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut}; -use super::storage_api::bucket::quota::checker::QuotaChecker; -use super::storage_api::bucket::{ +use super::storage_api::object_usecase::HTTPPreconditions; +use super::storage_api::object_usecase::access::{ + PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut, +}; +use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker; +use super::storage_api::object_usecase::bucket::{ ReplicationConfigExt as _, VersioningConfigExt as _, lifecycle::{ bucket_lifecycle_audit::LcEventSrc, @@ -43,38 +45,38 @@ use super::storage_api::bucket::{ validate_restore_request, versioning_sys::BucketVersioningSys, }; -use super::storage_api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; -use super::storage_api::concurrency::{ +use super::storage_api::object_usecase::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; +use super::storage_api::object_usecase::concurrency::{ self, ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; -use super::storage_api::data_usage::{record_bucket_object_delete_memory, record_bucket_object_write_memory}; -use super::storage_api::deadlock_detector; -use super::storage_api::ecfs::FS; -use super::storage_api::error::{ +use super::storage_api::object_usecase::data_usage::{record_bucket_object_delete_memory, record_bucket_object_write_memory}; +use super::storage_api::object_usecase::deadlock_detector; +use super::storage_api::object_usecase::ecfs::FS; +use super::storage_api::object_usecase::error::{ DiskError, Error as EcstoreError, StorageError, is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; -use super::storage_api::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; -use super::storage_api::helper::{OperationHelper, spawn_background_with_context}; -use super::storage_api::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader}; -use super::storage_api::object_utils::to_s3s_etag; -use super::storage_api::options::{ +use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; +use super::storage_api::object_usecase::helper::{OperationHelper, spawn_background_with_context}; +use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader}; +use super::storage_api::object_usecase::object_utils::to_s3s_etag; +use super::storage_api::object_usecase::options::{ copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts, }; -use super::storage_api::request_context::{self, spawn_traced}; -use super::storage_api::s3_api::multipart::parse_list_parts_params; -use super::storage_api::set_disk::{get_lock_acquire_timeout, is_valid_storage_class}; -use super::storage_api::sse::{ +use super::storage_api::object_usecase::request_context::{self, spawn_traced}; +use super::storage_api::object_usecase::s3_api::multipart::parse_list_parts_params; +use super::storage_api::object_usecase::set_disk::{get_lock_acquire_timeout, is_valid_storage_class}; +use super::storage_api::object_usecase::sse::{ DecryptionRequest, EncryptionRequest, SSEType, apply_bucket_default_lock_retention, build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error, sse_decryption, sse_encryption, }; -use super::storage_api::storage_class as storageclass; -use super::storage_api::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; -use super::storage_api::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _}; -use super::storage_api::{ +use super::storage_api::object_usecase::storage_class as storageclass; +use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; +use super::storage_api::object_usecase::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _}; +use super::storage_api::object_usecase::{ RFC1123, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read, @@ -162,7 +164,7 @@ use tokio_util::io::{ReaderStream, StreamReader}; use tracing::{debug, error, instrument, warn}; use uuid::Uuid; -use super::storage_api::{ +use super::storage_api::object_usecase::{ StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, }; @@ -786,7 +788,7 @@ fn should_buffer_get_object_in_memory_with_threshold( #[cfg(test)] mod deadlock_request_guard_tests { use super::DeadlockRequestGuard; - use crate::app::storage_api::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy}; + use crate::app::storage_api::object_usecase::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy}; use std::sync::Arc; #[test] diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index 095918374..5a0ab3c3d 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -1,7 +1,7 @@ -use super::storage_api::ObjectOperations as _; -use super::storage_api::options::get_opts; -use super::storage_api::request_context::spawn_traced; -use super::storage_api::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; +use super::storage_api::select_object::ObjectOperations as _; +use super::storage_api::select_object::options::get_opts; +use super::storage_api::select_object::request_context::spawn_traced; +use super::storage_api::select_object::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; use crate::app::runtime_sources::resolve_s3select_db; use crate::error::ApiError; use bytes::Bytes; diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 0c1071a53..70bd3511b 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -736,3 +736,68 @@ pub(crate) use rustfs_storage_api::{ }; #[cfg(test)] pub(crate) use rustfs_storage_api::{HTTPPreconditions, HealOperations}; + +pub(crate) mod admin_usecase { + pub(crate) use super::{ECStore, EndpointServerPools, StorageAdminApi}; + pub(crate) use super::{admin, capacity, data_usage}; +} + +pub(crate) mod bucket_usecase { + pub(crate) use super::{ + BucketOperations, BucketOptions, DeleteBucketOptions, ECStore, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, + MakeBucketOptions, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations, + process_topic_configurations, validate_list_object_unordered_with_delimiter, + }; + pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api}; +} + +pub(crate) mod object_usecase { + #[cfg(test)] + pub(crate) use super::HTTPPreconditions; + pub(crate) use super::{ + ECStore, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations, RFC1123, StorageDeletedObject, StorageObjectInfo, + StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, bucket, check_preconditions, get_validated_store, + has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, + remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, + validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, + wrap_response_with_cors, + }; + pub(crate) use super::{ + access, compression, concurrency, data_usage, deadlock_detector, ecfs, error, head_prefix, helper, io, object_utils, + options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper, + }; +} + +pub(crate) mod multipart_usecase { + #[cfg(test)] + pub(crate) use super::HTTPPreconditions; + pub(crate) use super::{ + CompletePart, ECStore, HTTPRangeSpec, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectOperations, + StorageObjectOptions, StoragePutObjReader, access, bucket, compression, data_usage, error, helper, io, object_utils, + options, s3_api, set_disk, sse, + }; +} + +pub(crate) mod select_object { + pub(crate) use super::ObjectOperations; + pub(crate) use super::get_validated_store; + pub(crate) use super::{options, request_context}; + pub(crate) use super::{validate_sse_headers_for_read, validate_ssec_for_read}; +} + +pub(crate) mod context { + pub(crate) use super::bucket; + pub(crate) use super::{ECStore, EndpointServerPools, runtime}; + #[cfg(test)] + pub(crate) use super::{Endpoint, Endpoints, PoolEndpoints}; +} + +#[cfg(test)] +pub(crate) mod test { + pub(crate) use super::{ + BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, HealOperations, ListOperations, + MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, PoolEndpoints, StorageObjectInfo, + StorageObjectOptions, StoragePutObjReader, + }; + pub(crate) use super::{bucket, ecfs, object_utils, runtime}; +} diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index e0e0e3f04..7b1d18129 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -185,6 +185,7 @@ RUSTFS_ROOT_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_byp RUSTFS_ROOT_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_contract_bypass_hits.txt" RUSTFS_ROOT_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_domain_bypass_hits.txt" RUSTFS_ADMIN_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_admin_storage_api_domain_bypass_hits.txt" +RUSTFS_APP_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_storage_api_domain_bypass_hits.txt" RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_storage_api_contract_bypass_hits.txt" RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_admin_storage_api_contract_bypass_hits.txt" RUSTFS_STORAGE_DIRECT_APP_CONTEXT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_storage_direct_app_context_bypass_hits.txt" @@ -1643,12 +1644,14 @@ fi { rg -n --with-filename \ 'use super::(?:\{[^}]*\b(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b|(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b)' \ - rustfs/src/app/*.rs rustfs/src/app/context/*.rs \ - -g '!storage_api.rs' || true + rustfs/src/app \ + --glob '*.rs' \ + --glob '!rustfs/src/app/storage_api.rs' || true rg -n --with-filename \ 'super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)|super::super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)' \ - rustfs/src/app/*.rs rustfs/src/app/context/*.rs \ - -g '!storage_api.rs' || true + rustfs/src/app \ + --glob '*.rs' \ + --glob '!rustfs/src/app/storage_api.rs' || true rg -n --with-filename \ 'crate::app::(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|get_global_|set_object_store_resolver)' \ rustfs/src --glob '*.rs' || true @@ -1748,6 +1751,19 @@ if [[ -s "$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" ]]; then report_failure "RustFS app storage contracts must stay behind rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename \ + '(?:crate::app|super)::storage_api::(?:ECStore|Endpoint|EndpointServerPools|Endpoints|PoolEndpoints|HTTPPreconditions|HealOperations|BucketOperations|BucketOptions|CompletePart|DeleteBucketOptions|HTTPRangeSpec|ListObjectVersionsInfo|ListObjectsV2Info|ListOperations|MakeBucketOptions|MultipartOperations|MultipartUploadResult|NamespaceLocking|ObjectIO|ObjectOperations|StorageAdminApi|StorageObjectInfo|StorageObjectOptions|StoragePutObjReader|access|admin|bucket|capacity|compression|concurrency|data_usage|deadlock_detector|ecfs|error|head_prefix|helper|io|object_utils|options|request_context|runtime|s3_api|set_disk|sse|storage_class|timeout_wrapper)\b|use super::storage_api::\{' \ + rustfs/src/app \ + --glob '*.rs' \ + --glob '!rustfs/src/app/storage_api.rs' || true +) >"$RUSTFS_APP_STORAGE_API_DOMAIN_BYPASS_HITS_FILE" + +if [[ -s "$RUSTFS_APP_STORAGE_API_DOMAIN_BYPASS_HITS_FILE" ]]; then + report_failure "RustFS app storage-api consumers must use app domain modules from rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_STORAGE_API_DOMAIN_BYPASS_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename '^use rustfs_storage_api|rustfs_storage_api::' \