diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index d5725da36..14abc5afc 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,13 +5,13 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-storage-api-residual-flat-consumers` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/CTX-002`. -- Based on: rebased onto current `origin/main` after PR #3905 merged. +- Branch: `overtrue/arch-rustfs-storage-contract-domain-imports` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/CTX-002`. +- Based on: rebased onto current `origin/main` after PR #3908 merged. - PR type for this branch: `consumer-migration` -- Runtime behavior changes: none expected for API-242; external/test local - storage API boundaries still expose the same storage-api contracts through - consumer-domain modules. +- Runtime behavior changes: none expected for API-243; RustFS storage, admin, + and app storage API boundaries still expose the same storage contracts and + helper APIs through domain modules. - Rust code changes: route replication pool, outbound TLS generation, runtime region, KMS encryption service, runtime support handles, S3 Select DB, internode RPC metrics, IAM authorization/handler reads, notification @@ -82,7 +82,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block domain modules, and local ECStore type aliases for admin config and storage RPC paths, plus external/test local storage API contract imports through local `storage_contracts` aliases and consumer-domain modules instead of - root re-exports. + root re-exports, plus RustFS storage owner, admin, and app local storage API + contract imports through local `storage_contracts` aliases and domain-module + exports instead of root re-exports, plus RustFS app/admin storage helper + imports through domain-module exports instead of root re-exports. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade @@ -92,13 +95,14 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block event-bridge thin module regressions, plus IAM runtime-source bypasses; accept the reviewed AppContext resolver reverse dependencies in the layer baseline, and block direct admin AppContext resolver consumers outside the - admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, reject storage owner direct ECStore/storage-api imports outside the owner-local `storage_api` boundary, reject ECStore internal direct storage-api imports outside the owner-local `storage_api_contracts` boundary, reject direct storage ECFS app usecase construction outside the storage S3 API boundary, reject flat root `storage_api` imports outside the new root-local domain modules, reject flat admin `storage_api` imports outside the new admin domain modules, reject flat app `storage_api` imports outside the new app consumer-domain modules, reject flat external crate-local `storage_api` imports outside the new consumer-domain modules, reject residual flat local `storage_api` imports from notify, ECStore tests/benches, and remaining RustFS app storage API type references, and reject external/test local storage API root contract re-exports. + admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, reject storage owner direct ECStore/storage-api imports outside the owner-local `storage_api` boundary, reject ECStore internal direct storage-api imports outside the owner-local `storage_api_contracts` boundary, reject direct storage ECFS app usecase construction outside the storage S3 API boundary, reject flat root `storage_api` imports outside the new root-local domain modules, reject flat admin `storage_api` imports outside the new admin domain modules, reject flat app `storage_api` imports outside the new app consumer-domain modules, reject flat external crate-local `storage_api` imports outside the new consumer-domain modules, reject residual flat local `storage_api` imports from notify, ECStore tests/benches, and remaining RustFS app storage API type references, reject external/test local storage API root contract re-exports, reject RustFS local storage API root contract re-exports, and reject RustFS app/admin storage helper root re-exports. - Docs changes: record the API-136 through API-242 owner facade, runtime-source, ECFS usecase, root storage API domain-boundary, and admin storage API domain-boundary cleanup, and app storage API consumer-domain cleanup, plus external crate-local and residual local storage API - consumer-domain cleanup, plus external/test storage contract root re-export - cleanup. + consumer-domain cleanup, external/test storage contract root re-export + cleanup, RustFS local storage contract root re-export cleanup, and RustFS + app/admin storage helper root re-export cleanup. ## Phase 0 Tasks @@ -5564,14 +5568,32 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block and layer guards, diff hygiene, external root storage-api re-export scan, Rust risk scan, and full PR gate before PR. +- [x] `API-243` Segment RustFS storage API root exports by domain module. + - Do: replace RustFS storage owner, admin, and app local storage API root + contract re-exports with local `storage_contracts` aliases and domain-module + exports, and move app/admin storage helper root re-exports into their + consumer-domain modules. + - Acceptance: RustFS storage owner, admin, app, and root storage API + boundaries no longer expose storage-api contracts from their roots, and + app/admin boundaries no longer expose storage helpers from their roots; + migration rules reject both root re-export regressions. + - Must preserve: storage ECFS, RPC, S3 output mapping, admin handlers, app + usecases, AppContext/test fixtures, and storage contract call paths. + - Verification: focused RustFS compile, formatting, migration and layer + guards, diff hygiene, RustFS root storage-api/helper re-export scans, Rust + risk scan, and full PR gate before PR. + ## Next PRs -1. `consumer-migration`: continue larger owner boundary batches after API-242. +1. `consumer-migration`: continue larger owner boundary batches after API-243. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-243 removes RustFS local storage API root contract/helper re-exports and keeps exposure inside domain modules. | +| Migration preservation | pass | Storage ECFS/RPC/S3 helpers, admin handlers, app usecases, AppContext fixtures, and test harness consumers keep the same underlying storage contracts and helpers. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, root storage-api/helper re-export scans, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. | | Quality/architecture | pass | API-242 removes external/test local storage API root contract re-exports and keeps contract exposure inside consumer-domain modules. | | Migration preservation | pass | ECStore tests, heal, IAM, OBS metrics, Swift, S3 Select, and scanner consumers keep the same storage-api contracts and call paths. | | Testing/verification | pass | Focused external/test compile, formatting, migration/layer guards, external root storage-api re-export scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. | @@ -5839,6 +5861,26 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: +- Issue #660 API-243 current slice: + - Branch freshness check: rebased onto current `origin/main` after PR #3908 + merged. + - `cargo check --tests --benches -p rustfs -F rustfs-protocols/swift`: + passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - RustFS storage-api root re-export scan: passed; storage owner, admin, app, + and root storage API boundaries no longer expose `rustfs_storage_api` + contracts from their roots. + - RustFS app/admin storage helper root re-export scan: passed; app/admin + storage API boundaries now expose storage helpers from domain modules. + - Diff-added Rust risk scan: passed; no new production unwrap/expect, + numeric cast, String error, Box dyn Error, print macro, or relaxed atomic + ordering lines. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `make pre-pr`: passed. + - Issue #660 API-242 current slice: - Branch freshness check: rebased onto current `origin/main` after PR #3905 merged. diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index c2f8dba2a..53f3f16bf 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -15,6 +15,8 @@ use std::ops::Deref; use std::sync::Arc; +use rustfs_storage_api as storage_contracts; + mod ecstore_bucket { pub(crate) use crate::storage::ecstore_bucket::{ bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning, @@ -408,17 +410,6 @@ pub(crate) static ERR_TIER_NAME_NOT_UPPERCASE: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_NAME_NOT_UPPERCASE); pub(crate) static ERR_TIER_NOT_FOUND: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_NOT_FOUND); -pub(crate) use crate::storage::StorageObjectOptions; -pub(crate) use crate::storage::access::{ReqInfo, authorize_request}; -pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced}; -pub(crate) use rustfs_storage_api::{ - BucketOperations, BucketOptions, CapabilitySnapshotError, CapabilityState, CapabilityStatus, DeleteBucketOptions, - HealOperations, ListOperations, MakeBucketOptions, ObjectIO, ObjectOperations, ObservabilitySnapshot, - ObservabilitySnapshotProvider, SRBucketDeleteOp, StorageAdminApi, TopologySnapshot, TopologySnapshotProvider, -}; -#[cfg(test)] -pub(crate) use rustfs_storage_api::{MemorySamplingState, PlatformSupport, TopologyCapabilities, UserspaceProfilingCapability}; - pub(crate) mod data_usage { use std::sync::Arc; @@ -430,7 +421,8 @@ pub(crate) mod data_usage { } pub(crate) mod access { - pub(crate) use super::{ReqInfo, RequestContext, authorize_request, spawn_traced}; + pub(crate) use crate::storage::access::{ReqInfo, authorize_request}; + pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced}; } pub(crate) mod bucket { @@ -457,13 +449,15 @@ pub(crate) mod cluster { ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, }; - pub(crate) use super::{ + pub(crate) use super::storage_contracts::{ CapabilitySnapshotError, CapabilityState, CapabilityStatus, ObservabilitySnapshot, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider, }; #[cfg(test)] - pub(crate) use super::{MemorySamplingState, PlatformSupport, TopologyCapabilities, UserspaceProfilingCapability}; + pub(crate) use super::storage_contracts::{ + MemorySamplingState, PlatformSupport, TopologyCapabilities, UserspaceProfilingCapability, + }; } pub(crate) mod config { @@ -475,7 +469,7 @@ pub(crate) mod config { } pub(crate) mod contract { - pub(crate) use super::{ + pub(crate) use super::storage_contracts::{ BucketOperations, BucketOptions, DeleteBucketOptions, HealOperations, ListOperations, MakeBucketOptions, ObjectIO, ObjectOperations, SRBucketDeleteOp, StorageAdminApi, }; @@ -490,7 +484,7 @@ pub(crate) mod metrics { } pub(crate) mod object { - pub(crate) use super::StorageObjectOptions; + pub(crate) use crate::storage::StorageObjectOptions; } pub(crate) mod rebalance { diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 70bd3511b..483974d61 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -14,15 +14,11 @@ //! App-local boundary for storage-layer helper APIs used by S3 use cases. -pub(crate) use crate::storage::ECStore; -#[cfg(test)] -pub(crate) use crate::storage::{Endpoint, Endpoints, PoolEndpoints}; -pub(crate) use rustfs_storage_api::{ObjectToDelete, TransitionedObject}; -pub(crate) type EndpointServerPools = crate::storage::EndpointServerPools; +use rustfs_storage_api as storage_contracts; #[cfg(test)] #[allow(non_snake_case)] -pub(crate) fn EndpointServerPools(pools: Vec) -> EndpointServerPools { +pub(crate) fn EndpointServerPools(pools: Vec) -> crate::storage::EndpointServerPools { crate::storage::EndpointServerPools::from(pools) } @@ -54,7 +50,7 @@ pub(crate) mod data_usage { } pub(crate) async fn load_data_usage_from_backend( - store: Arc, + store: Arc, ) -> Result { crate::storage::ecstore_data_usage::load_data_usage_from_backend(store).await } @@ -69,7 +65,7 @@ pub(crate) mod data_usage { } pub(crate) async fn remove_bucket_usage_from_backend( - store: Arc, + store: Arc, bucket: &str, ) -> Result<(), crate::storage::StorageError> { crate::storage::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await @@ -103,7 +99,7 @@ pub(crate) mod runtime { crate::storage::ecstore_config::set_global_storage_class(cfg); } - pub(crate) fn get_global_endpoints_opt() -> Option { + pub(crate) fn get_global_endpoints_opt() -> Option { crate::storage::get_global_endpoints_opt() } @@ -135,7 +131,7 @@ pub(crate) mod runtime { crate::storage::get_global_expiry_state() } - pub(crate) fn new_object_layer_fn() -> Option> { + pub(crate) fn new_object_layer_fn() -> Option> { crate::storage::new_object_layer_fn() } @@ -172,7 +168,9 @@ pub(crate) mod runtime { } #[cfg(test)] - pub(crate) async fn init_local_disks(endpoint_pools: super::EndpointServerPools) -> Result<(), crate::storage::StorageError> { + pub(crate) async fn init_local_disks( + endpoint_pools: crate::storage::EndpointServerPools, + ) -> Result<(), crate::storage::StorageError> { crate::storage::init_local_disks(endpoint_pools).await } } @@ -348,7 +346,7 @@ pub(crate) mod bucket { version_id: Option, versioned: bool, suspended: bool, - transitioned: &crate::app::storage_api::TransitionedObject, + transitioned: &super::super::super::storage_contracts::TransitionedObject, ) -> Option { crate::storage::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( version_id, @@ -359,7 +357,7 @@ pub(crate) mod bucket { } pub(crate) fn transitioned_force_delete_journal_entry( - transitioned: &crate::app::storage_api::TransitionedObject, + transitioned: &super::super::super::storage_contracts::TransitionedObject, ) -> Option { crate::storage::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(transitioned) } @@ -544,7 +542,7 @@ pub(crate) mod bucket { pub(crate) async fn check_replicate_delete( bucket: &str, - dobj: &crate::app::storage_api::ObjectToDelete, + dobj: &super::super::storage_contracts::ObjectToDelete, oi: &crate::storage::StorageObjectInfo, del_opts: &crate::storage::StorageObjectOptions, gerr: Option, @@ -721,83 +719,78 @@ pub(crate) mod s3_api { } } -pub(crate) use crate::storage::{ - RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, - check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, - parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, - validate_list_object_unordered_with_delimiter, validate_object_key, validate_sse_headers_for_read, - validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, -}; -pub(crate) use rustfs_storage_api::{ - BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, HTTPRangeSpec, ListObjectVersionsInfo, ListObjectsV2Info, - ListOperations, MakeBucketOptions, MultipartOperations, MultipartUploadResult, NamespaceLocking, ObjectIO, ObjectOperations, - StorageAdminApi, -}; -#[cfg(test)] -pub(crate) use rustfs_storage_api::{HTTPPreconditions, HealOperations}; - pub(crate) mod admin_usecase { - pub(crate) use super::{ECStore, EndpointServerPools, StorageAdminApi}; + pub(crate) use super::storage_contracts::StorageAdminApi; pub(crate) use super::{admin, capacity, data_usage}; + pub(crate) use crate::storage::{ECStore, EndpointServerPools}; } pub(crate) mod bucket_usecase { - pub(crate) use super::{ - BucketOperations, BucketOptions, DeleteBucketOptions, ECStore, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, - MakeBucketOptions, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations, - process_topic_configurations, validate_list_object_unordered_with_delimiter, + pub(crate) use super::storage_contracts::{ + BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, + MakeBucketOptions, }; pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api}; + pub(crate) use crate::storage::{ + ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations, + process_topic_configurations, validate_list_object_unordered_with_delimiter, + }; } pub(crate) mod object_usecase { #[cfg(test)] - pub(crate) use super::HTTPPreconditions; + pub(crate) use super::storage_contracts::HTTPPreconditions; + pub(crate) use super::storage_contracts::{HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations}; pub(crate) use super::{ - ECStore, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations, RFC1123, StorageDeletedObject, StorageObjectInfo, - StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, bucket, check_preconditions, get_validated_store, - has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, - validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, - wrap_response_with_cors, + access, bucket, compression, concurrency, data_usage, deadlock_detector, ecfs, error, head_prefix, helper, io, + object_utils, options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper, }; - pub(crate) use super::{ - access, compression, concurrency, data_usage, deadlock_detector, ecfs, error, head_prefix, helper, io, object_utils, - options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper, + pub(crate) use crate::storage::{ + ECStore, RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, + StoragePutObjReader, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, + parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, + strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, validate_object_key, + validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, }; } pub(crate) mod multipart_usecase { #[cfg(test)] - pub(crate) use super::HTTPPreconditions; - pub(crate) use super::{ - CompletePart, ECStore, HTTPRangeSpec, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectOperations, - StorageObjectOptions, StoragePutObjReader, access, bucket, compression, data_usage, error, helper, io, object_utils, - options, s3_api, set_disk, sse, + pub(crate) use super::storage_contracts::HTTPPreconditions; + pub(crate) use super::storage_contracts::{ + CompletePart, HTTPRangeSpec, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectOperations, }; + pub(crate) use super::{ + access, bucket, compression, data_usage, error, helper, io, object_utils, options, s3_api, set_disk, sse, + }; + pub(crate) use crate::storage::{ECStore, StorageObjectOptions, StoragePutObjReader}; } pub(crate) mod select_object { - pub(crate) use super::ObjectOperations; - pub(crate) use super::get_validated_store; + pub(crate) use super::storage_contracts::ObjectOperations; pub(crate) use super::{options, request_context}; - pub(crate) use super::{validate_sse_headers_for_read, validate_ssec_for_read}; + pub(crate) use crate::storage::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; } pub(crate) mod context { - pub(crate) use super::bucket; - pub(crate) use super::{ECStore, EndpointServerPools, runtime}; #[cfg(test)] - pub(crate) use super::{Endpoint, Endpoints, PoolEndpoints}; + pub(crate) use super::EndpointServerPools; + pub(crate) use super::bucket; + pub(crate) use super::runtime; + pub(crate) use crate::storage::{ECStore, EndpointServerPools}; + #[cfg(test)] + pub(crate) use crate::storage::{Endpoint, Endpoints, PoolEndpoints}; } #[cfg(test)] pub(crate) mod test { - pub(crate) use super::{ - BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, HealOperations, ListOperations, - MakeBucketOptions, MultipartOperations, ObjectIO, ObjectOperations, PoolEndpoints, StorageObjectInfo, - StorageObjectOptions, StoragePutObjReader, + pub(crate) use super::EndpointServerPools; + pub(crate) use super::storage_contracts::{ + BucketOperations, BucketOptions, HealOperations, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO, + ObjectOperations, }; pub(crate) use super::{bucket, ecfs, object_utils, runtime}; + pub(crate) use crate::storage::{ + ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, + }; } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 638ae701d..3cf37e484 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::BucketOperations; use super::ECStore; use super::ecfs::FS; use super::resolve_object_store_handle; @@ -23,6 +22,7 @@ use crate::auth::{check_key_valid, get_condition_values_with_query_and_client_in use crate::error::ApiError; use crate::license::license_check; use crate::server::RemoteAddr; +use crate::storage::contract::BucketOperations; use crate::storage::request_context::RequestContext; use crate::storage::runtime_sources; use metrics::counter; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index d9356f389..e55b07a07 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -14,9 +14,8 @@ use super::{ BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, - BUCKET_WEBSITE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, OBJECT_LOCK_CONFIG, ObjectLockRetentionOptions, - ObjectOperations as _, StorageError, check_retention_for_modification, decode_tags, decode_tags_to_map, - delete_bucket_metadata_config, encode_tags, get_bucket_accelerate_config, get_bucket_logging_config, + BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, check_retention_for_modification, decode_tags, + decode_tags_to_map, delete_bucket_metadata_config, encode_tags, get_bucket_accelerate_config, get_bucket_logging_config, get_bucket_object_lock_config, get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_website_config, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, record_replication_proxy, serialize, update_bucket_metadata_config, @@ -24,6 +23,7 @@ use super::{ use super::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; use crate::error::ApiError; use crate::storage::access::has_bypass_governance_header; +use crate::storage::contract::{BucketOperations, BucketOptions, ObjectLockRetentionOptions, ObjectOperations as _}; use crate::storage::helper::OperationHelper; use crate::storage::options::get_opts; use crate::storage::runtime_sources; diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 9a60d70a6..c1c653449 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -14,12 +14,13 @@ use super::StorageReplicationConfigExt as _; use super::{ - BucketOperations, BucketOptions, StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, - get_bucket_replication_config, resolve_object_store_handle, + StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, get_bucket_replication_config, + resolve_object_store_handle, }; use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabled}; use crate::error::ApiError; use crate::server::cors; +use crate::storage::contract::{BucketOperations, BucketOptions, ObjectToDelete}; use crate::storage::ecfs::ListObjectUnorderedQuery; use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; use http::{HeaderMap, HeaderValue, StatusCode}; @@ -45,8 +46,8 @@ use time::format_description::well_known::Rfc3339; use time::{format_description::FormatItem, macros::format_description}; use tracing::{debug, warn}; +use crate::storage::StorageObjectInfo as ObjectInfo; use crate::storage::runtime_sources; -use crate::storage::{StorageObjectInfo as ObjectInfo, StorageObjectToDelete as ObjectToDelete}; const LOG_COMPONENT_STORAGE: &str = "storage"; const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 46b602b53..4aab1ba58 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::ECStore; -use super::ListOperations as _; +use crate::storage::contract::ListOperations as _; use std::sync::Arc; /// Determines if the key "looks like a prefix" (ends with `/`). diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index d76479a53..fc0acb0bd 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{BucketVersioningSys, HTTPPreconditions, HTTPRangeSpec, Result, StorageError}; +use super::{BucketVersioningSys, Result, StorageError}; +use crate::storage::contract::{HTTPPreconditions, HTTPRangeSpec}; use http::header::{IF_MATCH, IF_NONE_MATCH}; use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index cf4e62e6a..5b7a47be6 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -13,17 +13,17 @@ // limitations under the License. use super::super::{ - BucketOptions, CollectMetricsOpts, DeleteBucketOptions, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, - FileInfoVersions, LocalPeerS3Client, MakeBucketOptions, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, - ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageAdminApi, - StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, - find_local_disk_by_ref, get_local_server_property, load_bucket_metadata, reload_transition_tier_config, - resolve_object_store_handle, set_bucket_metadata, + CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions, + LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, + UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property, + load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata, }; use crate::admin::service::{ config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot}, site_replication::reload_site_replication_runtime_state, }; +use crate::storage::contract::{BucketOptions, DeleteBucketOptions, MakeBucketOptions, StorageAdminApi}; use crate::storage::runtime_sources; use bytes::Bytes; use futures::Stream; diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index eb75d30cd..1ffae1f65 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::s3_api::common::rustfs_owner; -use crate::storage::to_s3s_etag; -use crate::storage::{ +use crate::storage::contract::{ BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, }; +use crate::storage::s3_api::common::rustfs_owner; +use crate::storage::to_s3s_etag; use percent_encoding::percent_decode_str; use s3s::dto::{ Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput, @@ -394,8 +394,8 @@ mod tests { build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, }; - use crate::storage::BucketInfo; use crate::storage::StorageObjectInfo as ObjectInfo; + use crate::storage::contract::BucketInfo; use crate::storage::s3_api::common::rustfs_owner; use s3s::S3ErrorCode; use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object}; diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index ad5637cd2..0db320294 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::storage::contract::{ListMultipartsInfo, ListPartsInfo}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use crate::storage::to_s3s_etag; -use crate::storage::{ListMultipartsInfo, ListPartsInfo}; use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp}; use s3s::{S3Error, S3ErrorCode}; @@ -195,9 +195,9 @@ mod tests { MAX_MULTIPART_UPLOADS_LIST, build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, }; + use crate::storage::contract::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use crate::storage::to_s3s_etag; - use crate::storage::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo}; use s3s::S3ErrorCode; use s3s::dto::Timestamp; use time::OffsetDateTime; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 7309052f8..70b547057 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -16,20 +16,23 @@ use std::sync::Arc; -pub(crate) use rustfs_storage_api::{ - BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, DiskCapabilities, HTTPPreconditions, - HTTPRangeSpec, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ListPartsInfo, - MakeBucketOptions, ObjectIO, ObjectLockRetentionOptions, ObjectOperations, ObjectToDelete, StorageAdminApi, - TopologyCapabilities, TopologySnapshot, -}; -#[cfg(test)] -pub(crate) use rustfs_storage_api::{MultipartInfo, PartInfo}; +use rustfs_storage_api as storage_contracts; -pub(crate) type StorageDeletedObject = DeletedObject; +pub(crate) mod contract { + pub(crate) use rustfs_storage_api::{ + BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, HTTPPreconditions, HTTPRangeSpec, ListMultipartsInfo, + ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ListPartsInfo, MakeBucketOptions, ObjectLockRetentionOptions, + ObjectOperations, ObjectToDelete, StorageAdminApi, + }; + #[cfg(test)] + pub(crate) use rustfs_storage_api::{MultipartInfo, PartInfo}; +} + +pub(crate) type StorageDeletedObject = storage_contracts::DeletedObject; pub(crate) type StorageGetObjectReader = super::GetObjectReader; pub(crate) type StorageObjectInfo = super::ObjectInfo; pub(crate) type StorageObjectOptions = super::ObjectOptions; -pub(crate) type StorageObjectToDelete = ObjectToDelete; +pub(crate) type StorageObjectToDelete = storage_contracts::ObjectToDelete; pub(crate) type StoragePutObjReader = super::PutObjReader; pub(crate) mod ecstore_admin { @@ -624,10 +627,14 @@ pub(crate) trait StoragePeerS3ClientExt { bucket: &str, opts: &rustfs_common::heal_channel::HealOpts, ) -> DiskResult; - async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> DiskResult<()>; - async fn list_bucket(&self, opts: &BucketOptions) -> DiskResult>; - async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> DiskResult<()>; - async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> DiskResult; + async fn make_bucket(&self, bucket: &str, opts: &storage_contracts::MakeBucketOptions) -> DiskResult<()>; + async fn list_bucket(&self, opts: &storage_contracts::BucketOptions) -> DiskResult>; + async fn delete_bucket(&self, bucket: &str, opts: &storage_contracts::DeleteBucketOptions) -> DiskResult<()>; + async fn get_bucket_info( + &self, + bucket: &str, + opts: &storage_contracts::BucketOptions, + ) -> DiskResult; } impl StoragePeerS3ClientExt for LocalPeerS3Client { @@ -639,19 +646,23 @@ impl StoragePeerS3ClientExt for LocalPeerS3Client { ecstore_rpc::PeerS3Client::heal_bucket(self, bucket, opts).await } - async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> DiskResult<()> { + async fn make_bucket(&self, bucket: &str, opts: &storage_contracts::MakeBucketOptions) -> DiskResult<()> { ecstore_rpc::PeerS3Client::make_bucket(self, bucket, opts).await } - async fn list_bucket(&self, opts: &BucketOptions) -> DiskResult> { + async fn list_bucket(&self, opts: &storage_contracts::BucketOptions) -> DiskResult> { ecstore_rpc::PeerS3Client::list_bucket(self, opts).await } - async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> DiskResult<()> { + async fn delete_bucket(&self, bucket: &str, opts: &storage_contracts::DeleteBucketOptions) -> DiskResult<()> { ecstore_rpc::PeerS3Client::delete_bucket(self, bucket, opts).await } - async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> DiskResult { + async fn get_bucket_info( + &self, + bucket: &str, + opts: &storage_contracts::BucketOptions, + ) -> DiskResult { ecstore_rpc::PeerS3Client::get_bucket_info(self, bucket, opts).await } } @@ -906,9 +917,9 @@ where pub(crate) fn topology_snapshot_from_endpoint_pools_with_capabilities( endpoint_pools: &EndpointServerPools, - capabilities: TopologyCapabilities, - disk_capabilities: DiskCapabilities, -) -> TopologySnapshot { + capabilities: storage_contracts::TopologyCapabilities, + disk_capabilities: storage_contracts::DiskCapabilities, +) -> storage_contracts::TopologySnapshot { ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(endpoint_pools, capabilities, disk_capabilities) } @@ -946,7 +957,7 @@ impl StorageVersioningConfigExt for s3s::dto::VersioningConfiguration { } } -pub(crate) type GetObjectReader = ::GetObjectReader; -pub(crate) type ObjectInfo = ::ObjectInfo; -pub(crate) type ObjectOptions = ::ObjectOptions; -pub(crate) type PutObjReader = ::PutObjectReader; +pub(crate) type GetObjectReader = ::GetObjectReader; +pub(crate) type ObjectInfo = ::ObjectInfo; +pub(crate) type ObjectOptions = ::ObjectOptions; +pub(crate) type PutObjReader = ::PutObjectReader; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index dea227fa9..6a1d05724 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -120,6 +120,8 @@ EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_runtime_ec EXTERNAL_RUNTIME_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/external_runtime_storage_api_bypass_hits.txt" EXTERNAL_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/external_storage_api_domain_bypass_hits.txt" EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE="${TMP_DIR}/external_storage_api_root_reexport_hits.txt" +RUSTFS_STORAGE_API_ROOT_REEXPORT_HITS_FILE="${TMP_DIR}/rustfs_storage_api_root_reexport_hits.txt" +RUSTFS_APP_ADMIN_STORAGE_HELPER_ROOT_REEXPORT_HITS_FILE="${TMP_DIR}/rustfs_app_admin_storage_helper_root_reexport_hits.txt" EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_compat_bypass_hits.txt" FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt" ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt" @@ -1464,6 +1466,30 @@ if [[ -s "$EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE" ]]; then report_failure "external local storage_api boundaries must expose storage-api contracts from consumer-domain modules, not root re-exports: $(paste -sd '; ' "$EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename '^pub(?:\(crate\))? use rustfs_storage_api' \ + rustfs/src/storage_api.rs \ + rustfs/src/storage/storage_api.rs \ + rustfs/src/admin/storage_api.rs \ + rustfs/src/app/storage_api.rs || true +) >"$RUSTFS_STORAGE_API_ROOT_REEXPORT_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_API_ROOT_REEXPORT_HITS_FILE" ]]; then + report_failure "RustFS local storage_api boundaries must expose storage-api contracts from domain modules, not root re-exports: $(paste -sd '; ' "$RUSTFS_STORAGE_API_ROOT_REEXPORT_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename '^pub(?:\(crate\))? use crate::storage::' \ + rustfs/src/admin/storage_api.rs \ + rustfs/src/app/storage_api.rs || true +) >"$RUSTFS_APP_ADMIN_STORAGE_HELPER_ROOT_REEXPORT_HITS_FILE" + +if [[ -s "$RUSTFS_APP_ADMIN_STORAGE_HELPER_ROOT_REEXPORT_HITS_FILE" ]]; then + report_failure "RustFS app/admin storage_api boundaries must expose storage helpers from domain modules, not root re-exports: $(paste -sd '; ' "$RUSTFS_APP_ADMIN_STORAGE_HELPER_ROOT_REEXPORT_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename 'get_global_(?:action_cred|server_config|notification_sys)' \