From 19c925c48056cae78aaabf3dad6483d1e3b58593 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 26 Jun 2026 04:19:02 +0800 Subject: [PATCH] refactor: expand storage api boundaries (#3886) * refactor: route root storage contracts * refactor: expand storage api boundaries --- docs/architecture/migration-progress.md | 73 +++++++++++++++++-- rustfs/src/admin/handlers/account_info.rs | 2 +- rustfs/src/admin/handlers/bucket_meta.rs | 2 +- rustfs/src/admin/handlers/cluster_snapshot.rs | 6 +- rustfs/src/admin/handlers/config_admin.rs | 2 +- rustfs/src/admin/handlers/extensions.rs | 5 +- rustfs/src/admin/handlers/heal.rs | 2 +- .../src/admin/handlers/object_zip_download.rs | 2 +- rustfs/src/admin/handlers/rebalance.rs | 2 +- rustfs/src/admin/handlers/replication.rs | 2 +- rustfs/src/admin/handlers/site_replication.rs | 2 +- rustfs/src/admin/handlers/system.rs | 20 ++--- rustfs/src/admin/router.rs | 2 +- rustfs/src/admin/service/config.rs | 2 +- rustfs/src/admin/storage_api.rs | 7 ++ rustfs/src/app/admin_usecase.rs | 2 +- rustfs/src/app/bucket_usecase.rs | 18 ++--- rustfs/src/app/capacity_dirty_scope_test.rs | 2 +- .../src/app/lifecycle_transition_api_test.rs | 8 +- rustfs/src/app/mod.rs | 1 - rustfs/src/app/multipart_usecase.rs | 18 ++--- rustfs/src/app/object_usecase.rs | 8 +- rustfs/src/app/s3_api.rs | 30 -------- rustfs/src/app/select_object.rs | 2 +- rustfs/src/app/storage_api.rs | 25 +++++++ rustfs/src/cluster_snapshot.rs | 8 +- rustfs/src/error.rs | 3 +- rustfs/src/runtime_capabilities.rs | 12 +-- rustfs/src/server/event.rs | 3 +- rustfs/src/server/readiness.rs | 3 +- rustfs/src/startup_bucket_metadata.rs | 5 +- rustfs/src/storage_api.rs | 9 +++ rustfs/src/table_catalog.rs | 13 ++-- scripts/check_architecture_migration_rules.sh | 46 +++++++++++- scripts/layer-dependency-baseline.txt | 30 ++++---- 35 files changed, 242 insertions(+), 135 deletions(-) delete mode 100644 rustfs/src/app/s3_api.rs diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index e9a779de5..349c1a030 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-root-server-storage-facade-cleanup` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217`. -- Based on: API-217 merged; branch routes root/server/startup storage facade consumers through a root-local `storage_api` boundary on latest `main`. +- Branch: `overtrue/arch-storage-api-boundary-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218`. +- Based on: API-218 branch; branch routes root/server/startup, app, and admin storage contract consumers through local `storage_api` boundaries and folds app S3 helper forwarding into the app boundary. - PR type for this branch: `consumer-migration` - Runtime behavior changes: none. - Rust code changes: route replication pool, outbound TLS generation, runtime @@ -53,7 +53,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block storage, bucket metadata, notification, services, readiness, HTTP/layer request context, module switches, event dispatch, cluster/runtime snapshots, capacity, workload admission, table catalog, init, protocol clients, and - config tests through a root-local storage_api boundary. + config tests through a root-local storage_api boundary, plus root/server/startup + `rustfs_storage_api` contract imports through the same boundary, app/admin + `rustfs_storage_api` contract imports through their local boundaries, and app + S3 helper forwarding through `app::storage_api`. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade @@ -63,8 +66,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block event-bridge thin module regressions, plus IAM runtime-source bypasses; accept the reviewed AppContext resolver reverse dependencies in the layer baseline, and block direct admin AppContext resolver consumers outside the - admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, and reject root/server/startup direct storage facade regressions from migrated outer consumers. -- Docs changes: record the API-136 through API-218 owner facade and lifecycle + admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, and keep app S3 helper imports routed through `app::storage_api`. +- Docs changes: record the API-136 through API-221 owner facade and lifecycle runtime-source cleanup. ## Phase 0 Tasks @@ -5161,14 +5164,70 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block layer guards, diff hygiene, residual migrated facade scan, Rust risk scan, fast PR gate, and full PR gate before PR. +- [x] `API-219` Route root/server/startup storage contracts through storage_api boundary. + - Do: re-export the remaining root/server/startup `rustfs_storage_api` + DTOs and traits from the root-local `storage_api` boundary, then route + cluster/runtime snapshots, bucket metadata startup, table catalog, + readiness, error mapping, and event tests through that boundary. + - Acceptance: migrated outer runtime consumers no longer import + `rustfs_storage_api` directly, and migration rules reject direct contract + import regressions outside storage/app/admin owner boundaries. + - Must preserve: cluster and runtime topology snapshots, bucket metadata + list contracts, HTTP range error mapping, table catalog object/list/lock + contracts, readiness storage inventory checks, and event transition test + DTO shape. + - Verification: focused RustFS compile/tests, formatting, migration and + layer guards, diff hygiene, residual migrated contract scan, Rust risk + scan, fast PR gate, and full PR gate before PR. + +- [x] `API-220` Route app/admin storage contracts through local storage_api boundaries. + - Do: re-export app and admin `rustfs_storage_api` DTOs and traits from + their local `storage_api` boundaries, then route app usecases/tests and + admin handlers/router/services through those boundaries. + - Acceptance: migrated app/admin consumers no longer import + `rustfs_storage_api` directly, and migration rules reject direct contract + import regressions outside their owner boundaries. + - Must preserve: app bucket/object/multipart/select contract types, app + capacity and transition test contracts, admin storage/admin/list/object + traits, cluster snapshot DTOs, heal/config/bucket metadata, replication, + rebalance, site replication, extension, object zip, and system admin + contract behavior. + - Verification: focused RustFS compile/tests, formatting, migration and + layer guards, diff hygiene, residual migrated contract scan, Rust risk + scan, fast PR gate, and full PR gate before PR. + +- [x] `API-221` Fold app S3 helper forwarding into app storage_api boundary. + - Do: move the app-local S3 response/parameter helper forwarding surface + into `app::storage_api`, remove the standalone app `s3_api` forwarding + module, and route bucket/object/multipart usecases through the storage + boundary. + - Acceptance: app usecases consume S3 helper forwarding through + `app::storage_api`, and migration rules reject direct storage S3 helper + and legacy app `s3_api` bypasses. + - Must preserve: list-buckets, list-objects, list-object-versions, multipart + upload/list-parts response builders, query parsers, upload part-number + parsing, and RustFS owner metadata helper behavior. + - Verification: focused RustFS app compile/tests, formatting, migration and + layer guards, diff hygiene, residual app S3 helper scan, Rust risk scan, + fast PR gate, and full PR gate before PR. + ## Next PRs -1. `consumer-migration`: continue larger outer/owner facade batches after API-218. +1. `consumer-migration`: continue larger outer/owner facade batches after API-221. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-221 folds app S3 helper forwarding into app storage_api and removes the standalone app s3_api forwarding module. | +| Migration preservation | pass | Bucket/object/multipart list and multipart helper parsers/builders keep the same storage S3 API implementations and owner metadata helper. | +| Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual app S3 helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | +| Quality/architecture | pass | API-220 routes app/admin storage contract DTO and trait imports through their local storage_api boundaries. | +| Migration preservation | pass | App usecase/test contracts and admin handler/router/service contracts keep the same rustfs_storage_api DTOs and traits. | +| Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual migrated contract scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | +| Quality/architecture | pass | API-219 routes root/server/startup storage contract DTO and trait imports through the root-local storage_api boundary. | +| Migration preservation | pass | Cluster/runtime snapshots, metadata startup, table catalog, readiness, error mapping, and event tests keep the same rustfs_storage_api contracts. | +| Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual migrated contract scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | | Quality/architecture | pass | API-218 adds a root-local storage_api boundary and routes root/server/startup storage facade consumers through it. | | Migration preservation | pass | Startup storage, metadata migration, notification, readiness, HTTP request context, module switches, event dispatch, snapshots, capacity, workload admission, table catalog, and config tests keep the same storage implementations. | | Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual migrated facade scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index be4538fd8..455777684 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -16,6 +16,7 @@ use crate::admin::auth::authenticate_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{resolve_action_credentials, resolve_object_store_handle}; use crate::admin::storage_api::versioning_sys::BucketVersioningSys; +use crate::admin::storage_api::{BucketOperations, BucketOptions, StorageAdminApi}; use crate::auth::get_condition_values; use crate::server::{ADMIN_PREFIX, RemoteAddr}; use http::{HeaderMap, HeaderValue}; @@ -24,7 +25,6 @@ use matchit::Params; use rustfs_policy::policy::BucketPolicy; use rustfs_policy::policy::default::DEFAULT_POLICIES; use rustfs_policy::policy::{Args, action::Action, action::S3Action}; -use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::Serialize; diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index f3c9d7aa9..ff5318317 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::admin::storage_api::ecstore_utils::{deserialize, serialize}; +use crate::admin::storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; use crate::admin::storage_api::{ StorageError, metadata::{ @@ -41,7 +42,6 @@ use rustfs_policy::policy::{ BucketPolicy, action::{Action, AdminAction}, }; -use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use s3s::{ Body, S3Request, S3Response, S3Result, diff --git a/rustfs/src/admin/handlers/cluster_snapshot.rs b/rustfs/src/admin/handlers/cluster_snapshot.rs index 7771bdce4..28f8e4f8f 100644 --- a/rustfs/src/admin/handlers/cluster_snapshot.rs +++ b/rustfs/src/admin/handlers/cluster_snapshot.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::storage_api::{CapabilityState, CapabilityStatus, ObservabilitySnapshot, TopologySnapshot}; use crate::admin::{ auth::validate_admin_request, router::{AdminOperation, Operation, S3Router}, @@ -34,7 +35,6 @@ use matchit::Params; use rustfs_concurrency::AdmissionState as WorkloadAdmissionState; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass}; use rustfs_policy::policy::action::{Action, AdminAction}; -use rustfs_storage_api::{CapabilityState, CapabilityStatus, ObservabilitySnapshot, TopologySnapshot}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::Serialize; @@ -590,16 +590,16 @@ fn summarize_named_capability_statuses( #[cfg(test)] mod tests { use super::{ClusterSnapshotResponse, ClusterSnapshotSummary, ClusterSnapshotView}; + use crate::admin::storage_api::CapabilityState; use crate::admin::storage_api::ecstore_cluster::{ ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, }; + use crate::admin::storage_api::{CapabilityStatus, ObservabilitySnapshot, TopologySnapshot}; use crate::cluster_snapshot::{ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot}; use crate::server::{DependencyReadiness, ReadinessDegradedReason}; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass}; - use rustfs_storage_api::CapabilityState; - use rustfs_storage_api::{CapabilityStatus, ObservabilitySnapshot, TopologySnapshot}; #[test] fn cluster_snapshot_handler_requires_server_info_admin_permission() { diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index 751ce1a0c..8060bdac4 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -19,6 +19,7 @@ use crate::admin::service::config::{ apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload, validate_server_config, }; +use crate::admin::storage_api::ListOperations as _; use crate::admin::storage_api::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use crate::admin::storage_api::{ RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, @@ -73,7 +74,6 @@ use rustfs_config::{ }; use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; -use rustfs_storage_api::ListOperations as _; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::Serialize; diff --git a/rustfs/src/admin/handlers/extensions.rs b/rustfs/src/admin/handlers/extensions.rs index 35635f2d9..1f4baae49 100644 --- a/rustfs/src/admin/handlers/extensions.rs +++ b/rustfs/src/admin/handlers/extensions.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::storage_api::CapabilityStatus; use crate::admin::{ auth::validate_admin_request, handlers::{cluster_snapshot, plugins_instances, system}, @@ -31,7 +32,6 @@ use rustfs_extension_schema::{ OPS_PROFILER_CAPABILITY, OpsDiagnosticsContract, OpsProfilerContract, }; use rustfs_policy::policy::action::{Action, AdminAction}; -use rustfs_storage_api::CapabilityStatus; use rustfs_targets::{ OpsDiagnosticsRegistry, OpsProfilerRegistry, TargetPluginExternalFlowGate, TargetPluginExternalFlowGateStatus, builtin_extension_schemas, builtin_ops_diagnostics_contract, builtin_ops_diagnostics_extension_schema, @@ -118,7 +118,8 @@ pub(crate) struct ExtensionInstancesResponse { pub next_marker: Option, } -async fn build_extension_catalog_response() -> Result { +async fn build_extension_catalog_response() -> Result +{ let mut extensions = builtin_extension_schemas(); let example = example_external_webhook_plugin(); extensions.push(target_marketplace_extension_schema(&example.manifest)); diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 03ccdd532..258a18e0d 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -15,6 +15,7 @@ use crate::admin::auth::{authenticate_request, validate_admin_request}; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::resolve_object_store_handle; +use crate::admin::storage_api::HealOperations as _; use crate::admin::storage_api::ecstore_utils::is_valid_object_prefix; use crate::admin::storage_api::is_reserved_or_invalid_bucket; use crate::admin::storage_api::spawn_traced; @@ -29,7 +30,6 @@ use rustfs_config::MAX_HEAL_REQUEST_SIZE; use rustfs_heal::heal::utils::format_set_disk_id; use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_scanner::scanner::{BackgroundHealInfo, read_background_heal_info}; -use rustfs_storage_api::HealOperations as _; use rustfs_utils::path::path_join; use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE}; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index 7c1166d1d..b8f8337c9 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -14,6 +14,7 @@ use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{resolve_action_credentials, resolve_object_store_handle, resolve_region}; +use crate::admin::storage_api::{BucketOperations, BucketOptions, ListOperations as _, ObjectIO as _, ObjectOperations as _}; use crate::admin::storage_api::{ReqInfo, StorageObjectOptions as ObjectOptions, authorize_request}; use crate::auth::{check_key_valid, get_session_token}; use crate::error::ApiError; @@ -31,7 +32,6 @@ use matchit::Params; use rand::RngExt; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_storage_api::{BucketOperations, ListOperations as _, ObjectIO as _, ObjectOperations as _, bucket::BucketOptions}; use rustfs_trusted_proxies::{ClientInfo, ValidationMode}; use rustfs_utils::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad}; use s3s::{Body, S3Request, S3Response, S3Result, dto::StreamingBlob, header::CONTENT_TYPE, s3_error}; diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 0484da879..f8271250b 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::storage_api::{BucketOperations, BucketOptions, StorageAdminApi}; use crate::admin::storage_api::{ DiskStat, ECStore, NotificationSys, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStopPropagationRecord, StorageError, decode_rebalance_stop_propagation_record, @@ -29,7 +30,6 @@ use http::{HeaderMap, HeaderValue, StatusCode, Uri}; use hyper::Method; use matchit::Params; use rustfs_policy::policy::action::{Action, AdminAction}; -use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi}; use rustfs_utils::{ MaskedAccessKey, http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}, diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 53c5a8fd0..191a7282c 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -23,6 +23,7 @@ use crate::admin::storage_api::metadata_sys; use crate::admin::storage_api::metadata_sys::get_replication_config; use crate::admin::storage_api::replication::BucketStats; use crate::admin::storage_api::target::BucketTarget; +use crate::admin::storage_api::{BucketOperations, BucketOptions}; use crate::admin::utils::read_compatible_admin_body; use crate::auth::{check_key_valid, get_session_token}; use crate::error::ApiError; @@ -33,7 +34,6 @@ use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; -use rustfs_storage_api::{BucketOperations, BucketOptions}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use std::collections::HashMap; diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 0da7461a9..9464467e4 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -34,6 +34,7 @@ use crate::admin::storage_api::metadata_sys; use crate::admin::storage_api::replication::ResyncOpts; use crate::admin::storage_api::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_api::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; +use crate::admin::storage_api::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}; use crate::admin::storage_api::{delete_admin_config, read_admin_config, save_admin_config}; use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body}; use crate::auth::{check_key_valid, get_session_token}; @@ -70,7 +71,6 @@ use rustfs_policy::policy::{ }; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; -use rustfs_storage_api::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}; use rustfs_tls_runtime::GlobalPublishedOutboundTlsState; use rustfs_utils::http::get_source_scheme; use rustls_pki_types::pem::PemObject; diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 0a1b05dba..90c437fb7 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -16,6 +16,9 @@ use super::{cluster_snapshot, metrics}; use crate::admin::auth::validate_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::resolve_endpoints_handle; +use crate::admin::storage_api::{ + CapabilityState, CapabilityStatus, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider, +}; use crate::app::admin_usecase::{DefaultAdminUsecase, QueryServerInfoRequest}; use crate::auth::{check_key_valid, get_session_token}; use crate::runtime_capabilities::{EndpointTopologySnapshotProvider, RustFsObservabilitySnapshotProvider}; @@ -27,9 +30,6 @@ use matchit::Params; use rustfs_concurrency::WorkloadAdmissionRegistrySnapshot; use rustfs_madmin::{InfoMessage, StorageInfo}; use rustfs_policy::policy::action::{Action, AdminAction, S3Action}; -use rustfs_storage_api::{ - CapabilityState, CapabilityStatus, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider, -}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::Serialize; @@ -297,7 +297,7 @@ pub struct RuntimeCapabilitiesResponse { pub summary: RuntimeCapabilitiesSummary, pub cluster_snapshot_path: String, pub cluster_snapshot_summary: Option, - pub observability: rustfs_storage_api::ObservabilitySnapshot, + pub observability: crate::admin::storage_api::ObservabilitySnapshot, pub workload_admission: WorkloadAdmissionRegistrySnapshot, pub topology: Option, pub topology_status: CapabilityStatus, @@ -306,7 +306,7 @@ pub struct RuntimeCapabilitiesResponse { pub struct RuntimeCapabilitiesHandler {} pub(crate) async fn build_runtime_capabilities_response() --> Result { +-> Result { let usecase = DefaultAdminUsecase::from_global(); let observability_provider = RustFsObservabilitySnapshotProvider; let observability = observability_provider.observability_snapshot().await?; @@ -339,7 +339,7 @@ pub(crate) async fn build_runtime_capabilities_response() } fn build_runtime_capabilities_summary( - observability: &rustfs_storage_api::ObservabilitySnapshot, + observability: &crate::admin::storage_api::ObservabilitySnapshot, topology: Option<&TopologySnapshot>, topology_status: &CapabilityStatus, cluster_snapshot_summary: Option<&CapabilityStatus>, @@ -515,13 +515,13 @@ mod tests { OBSERVABILITY_SUMMARY_RESOLVED, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED, build_runtime_capabilities_response, build_runtime_capabilities_summary, system_admin_discovery, }; - use crate::app::admin_usecase::DefaultAdminUsecase; - use rustfs_concurrency::{AdmissionState, WorkloadClass}; - use rustfs_madmin::{InfoMessage, StorageInfo}; - use rustfs_storage_api::{ + use crate::admin::storage_api::{ CapabilityState, CapabilityStatus, MemorySamplingState, ObservabilitySnapshot, PlatformSupport, TopologyCapabilities, TopologySnapshot, UserspaceProfilingCapability, }; + use crate::app::admin_usecase::DefaultAdminUsecase; + use rustfs_concurrency::{AdmissionState, WorkloadClass}; + use rustfs_madmin::{InfoMessage, StorageInfo}; #[tokio::test] async fn runtime_capabilities_response_reports_missing_topology_before_storage_init() { diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index c923cb52e..7ca2a5f09 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -31,6 +31,7 @@ use crate::admin::runtime_sources::{ resolve_object_store_handle, resolve_region, resolve_replication_pool_handle, resolve_replication_stats_handle, resolve_server_config, }; +use crate::admin::storage_api::{BucketOperations, BucketOptions}; use crate::admin::storage_api::{ReqInfo, authorize_request, spawn_traced}; use crate::app::object_usecase::DefaultObjectUsecase; use crate::auth::{check_key_valid, get_session_token}; @@ -64,7 +65,6 @@ use rustfs_notify::{Event as NotificationEvent, notification_system}; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_types::EventName; use rustfs_signer::pre_sign_v4; -use rustfs_storage_api::{BucketOperations, BucketOptions}; use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::http::{ SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index 6123178e7..374975651 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -16,6 +16,7 @@ use crate::admin::runtime_sources::{ AppContext, get_global_app_context, publish_server_config, publish_storage_class_config, resolve_notification_system, resolve_object_store_handle, resolve_object_store_handle_for_context, }; +use crate::admin::storage_api::StorageAdminApi; use crate::admin::storage_api::storageclass; use crate::admin::storage_api::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate}; use rustfs_audit::reload_audit_config; @@ -27,7 +28,6 @@ use rustfs_config::{AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR}; use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState}; use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; use rustfs_iam::oidc::load_oidc_provider_configs_from_server_config; -use rustfs_storage_api::StorageAdminApi; use rustfs_targets::config::{ validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 462ba74cb..43a78db8b 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -411,6 +411,13 @@ pub(crate) static ERR_TIER_NOT_FOUND: AdminErrorRef = AdminErrorRef(|| &ecstore_ pub(crate) use crate::storage::StorageObjectOptions; pub(crate) use crate::storage::access::{ReqInfo, authorize_request}; pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced}; +pub(crate) use rustfs_storage_api::{ + BucketOperations, BucketOptions, CapabilitySnapshotError, CapabilityState, CapabilityStatus, DeleteBucketOptions, + HealOperations, ListOperations, MakeBucketOptions, ObjectIO, ObjectOperations, ObservabilitySnapshot, + ObservabilitySnapshotProvider, SRBucketDeleteOp, StorageAdminApi, TopologySnapshot, TopologySnapshotProvider, +}; +#[cfg(test)] +pub(crate) use rustfs_storage_api::{MemorySamplingState, PlatformSupport, TopologyCapabilities, UserspaceProfilingCapability}; pub(crate) mod data_usage { use std::sync::Arc; diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index 1f061b076..b4236b396 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -14,6 +14,7 @@ //! Admin application use-case contracts. +use super::storage_api::StorageAdminApi; use super::storage_api::admin::get_server_info; use super::storage_api::capacity::{ PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free, @@ -29,7 +30,6 @@ use crate::error::ApiError; use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness}; use rustfs_data_usage::DataUsageInfo; use rustfs_madmin::{InfoMessage, StorageInfo}; -use rustfs_storage_api::StorageAdminApi; use s3s::S3ErrorCode; use std::sync::Arc; use tracing::{debug, error, info, warn}; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index f382c6ba6..1aec3d7b1 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -14,11 +14,6 @@ //! Bucket application use-case contracts. -use super::s3_api::bucket::{ - ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, - build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, - rustfs_owner, -}; use super::storage_api::ECStore; use super::storage_api::StorageObjectInfo as ObjectInfo; use super::storage_api::access::{ReqInfo, authorize_request, req_info_ref}; @@ -46,6 +41,15 @@ use super::storage_api::data_usage::remove_bucket_usage_from_backend; use super::storage_api::error::StorageError; use super::storage_api::helper::{OperationHelper, spawn_background_with_context}; use super::storage_api::object_utils::to_s3s_etag; +use super::storage_api::s3_api::bucket::{ + ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, + build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, + rustfs_owner, +}; +use super::storage_api::{ + BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo as StorageListObjectVersionsInfo, + ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, MakeBucketOptions, +}; use super::storage_api::{ get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, request_context, validate_list_object_unordered_with_delimiter, @@ -70,10 +74,6 @@ use rustfs_policy::policy::{ {BucketPolicy, BucketPolicyArgs, Effect, Validator}, }; use rustfs_s3_ops::S3Operation; -use rustfs_storage_api::{ - BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo as StorageListObjectVersionsInfo, - ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, MakeBucketOptions, -}; use rustfs_targets::{ EventName, arn::{ARN, TargetIDError}, diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index 850fad8ed..00634b3be 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -13,10 +13,10 @@ // limitations under the License. use super::storage_api::bucket::metadata_sys; +use super::storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; use super::storage_api::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; -use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; use serial_test::serial; use std::{ collections::HashSet, diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index fa965d8d0..b6e10d693 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -21,6 +21,10 @@ use super::storage_api::bucket::{ use super::storage_api::ecfs::FS; use super::storage_api::object_utils::to_s3s_etag; use super::storage_api::runtime::{AppWarmBackend, TierConfig, TierType, WarmBackendGetOpts}; +use super::storage_api::{ + BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, + ObjectOperations as _, +}; use super::storage_api::{ ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, @@ -34,10 +38,6 @@ use futures::stream; use http::{Extensions, HeaderMap, HeaderValue, Method, Uri, header::IF_NONE_MATCH}; use rustfs_config::{ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; -use rustfs_storage_api::{ - BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, - ObjectOperations as _, -}; use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header}; use s3s::{S3Request, dto::*}; use serial_test::serial; diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index 7ae97616a..8f282ea64 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -21,7 +21,6 @@ pub mod context; pub mod multipart_usecase; pub mod object_usecase; pub(crate) mod runtime_sources; -pub(crate) mod s3_api; mod select_object; pub(crate) mod storage_api; diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index dab7d7318..ab817b621 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -14,11 +14,9 @@ //! Multipart application use-case contracts. -use super::s3_api::multipart::{ - ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, - parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, -}; use super::storage_api::ECStore; +#[cfg(test)] +use super::storage_api::HTTPPreconditions; use super::storage_api::access::has_bypass_governance_header; use super::storage_api::bucket::quota::checker::QuotaChecker; use super::storage_api::bucket::{ @@ -40,6 +38,10 @@ use super::storage_api::options::{ copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts, parse_copy_source_range, put_opts, validate_archive_content_encoding, }; +use super::storage_api::s3_api::multipart::{ + ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, + parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, +}; use super::storage_api::set_disk::is_valid_storage_class; use super::storage_api::sse::{ DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention, @@ -47,6 +49,9 @@ use super::storage_api::sse::{ extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error, mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption, }; +use super::storage_api::{ + CompletePart, HTTPRangeSpec, MultipartOperations as _, MultipartUploadResult, ObjectIO as _, ObjectOperations as _, +}; use super::storage_api::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write}; use crate::app::runtime_sources::{AppContext, get_global_app_context, resolve_object_store_handle_for_context}; @@ -58,11 +63,6 @@ use futures::StreamExt; use http::{HeaderMap, Uri}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_s3_ops::S3Operation; -#[cfg(test)] -use rustfs_storage_api::HTTPPreconditions; -use rustfs_storage_api::{ - CompletePart, HTTPRangeSpec, MultipartOperations as _, MultipartUploadResult, ObjectIO as _, ObjectOperations as _, -}; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index e9fca1507..f7ea29d97 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -15,8 +15,9 @@ //! Object application use-case contracts. // Performance metrics recording (with zero-copy-metrics integration) -use super::s3_api::multipart::parse_list_parts_params; use super::storage_api::ECStore; +#[cfg(test)] +use super::storage_api::HTTPPreconditions; use super::storage_api::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut}; use super::storage_api::bucket::quota::checker::QuotaChecker; use super::storage_api::bucket::{ @@ -63,6 +64,7 @@ use super::storage_api::options::{ filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts, }; use super::storage_api::request_context::{self, spawn_traced}; +use super::storage_api::s3_api::multipart::parse_list_parts_params; use super::storage_api::set_disk::{get_lock_acquire_timeout, is_valid_storage_class}; use super::storage_api::sse::{ DecryptionRequest, EncryptionRequest, SSEType, apply_bucket_default_lock_retention, build_ssec_read_headers, @@ -71,6 +73,7 @@ use super::storage_api::sse::{ }; use super::storage_api::storage_class as storageclass; use super::storage_api::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; +use super::storage_api::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _}; use super::storage_api::{ RFC1123, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, @@ -107,9 +110,6 @@ use rustfs_object_capacity::capacity_manager::get_capacity_manager; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object}; use rustfs_s3select_api::object_store::bytes_stream; -#[cfg(test)] -use rustfs_storage_api::HTTPPreconditions; -use rustfs_storage_api::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _}; use rustfs_targets::{ EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, }; diff --git a/rustfs/src/app/s3_api.rs b/rustfs/src/app/s3_api.rs deleted file mode 100644 index 0f1bbc0f1..000000000 --- a/rustfs/src/app/s3_api.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! App-local S3 API response helpers. - -pub(crate) mod bucket { - pub(crate) use crate::storage::s3_api::bucket::{ - ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, - build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, - }; - pub(crate) use crate::storage::s3_api::common::rustfs_owner; -} - -pub(crate) mod multipart { - pub(crate) use crate::storage::s3_api::multipart::{ - ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, - parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, - }; -} diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index cda5ae119..095918374 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -1,3 +1,4 @@ +use super::storage_api::ObjectOperations as _; use super::storage_api::options::get_opts; use super::storage_api::request_context::spawn_traced; use super::storage_api::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; @@ -16,7 +17,6 @@ use rustfs_s3select_api::{ object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds}, query::{Context, Query}, }; -use rustfs_storage_api::ObjectOperations as _; use s3s::dto::{ CSVOutput, CompressionType, ContinuationEvent, EndEvent, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput, JSONType, OutputSerialization, Progress, ProgressEvent, QuoteFields, RecordsEvent, SelectObjectContentEvent, diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 5bcac944a..85218ec0c 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -702,6 +702,24 @@ pub(crate) mod timeout_wrapper { pub(crate) use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; } +pub(crate) mod s3_api { + pub(crate) mod bucket { + pub(crate) use crate::storage::s3_api::bucket::{ + ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, + build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, + parse_list_objects_v2_params, + }; + pub(crate) use crate::storage::s3_api::common::rustfs_owner; + } + + pub(crate) mod multipart { + pub(crate) use crate::storage::s3_api::multipart::{ + ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, + parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, + }; + } +} + pub(crate) use crate::storage::{ RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, @@ -710,3 +728,10 @@ pub(crate) use crate::storage::{ validate_list_object_unordered_with_delimiter, validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, }; +pub(crate) use rustfs_storage_api::{ + BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, HTTPRangeSpec, ListObjectVersionsInfo, ListObjectsV2Info, + ListOperations, MakeBucketOptions, MultipartOperations, MultipartUploadResult, NamespaceLocking, ObjectIO, ObjectOperations, + StorageAdminApi, +}; +#[cfg(test)] +pub(crate) use rustfs_storage_api::{HTTPPreconditions, HealOperations}; diff --git a/rustfs/src/cluster_snapshot.rs b/rustfs/src/cluster_snapshot.rs index dbf83c6f6..2b7de0fd5 100644 --- a/rustfs/src/cluster_snapshot.rs +++ b/rustfs/src/cluster_snapshot.rs @@ -16,14 +16,13 @@ use crate::runtime_capabilities::runtime_observability_snapshot; use crate::server::{ DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason, snapshot_dependency_readiness_report, }; -use crate::storage_api::EndpointServerPools; use crate::storage_api::ecstore_cluster::{ ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterPeerHealthSnapshot, ClusterPoolStateSnapshot, }; +use crate::storage_api::{EndpointServerPools, ObservabilitySnapshot, TopologySnapshot}; use crate::workload_admission::workload_admission_registry_snapshot; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot}; -use rustfs_storage_api::{ObservabilitySnapshot, TopologySnapshot}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ClusterReadOnlySnapshot { @@ -116,9 +115,10 @@ pub fn cluster_has_actionable_pressure(snapshot: &ClusterReadOnlySnapshot) -> bo #[cfg(test)] mod tests { use super::*; - use crate::storage_api::{Endpoint, Endpoints, PoolEndpoints}; + use crate::storage_api::{ + CapabilityState, CapabilityStatus, DiskCapabilities, Endpoint, Endpoints, PoolEndpoints, TopologyCapabilities, + }; use rustfs_concurrency::{WorkloadAdmissionSnapshot, WorkloadClass}; - use rustfs_storage_api::{CapabilityState, CapabilityStatus, DiskCapabilities, TopologyCapabilities}; #[test] fn runtime_status_snapshot_maps_ready_and_degraded_reports() { diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 7660474e7..bd20be6c0 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::{QuotaError, StorageError}; -use rustfs_storage_api::HTTPRangeError; +use crate::storage_api::{HTTPRangeError, QuotaError, StorageError}; use s3s::{S3Error, S3ErrorCode}; #[derive(Debug)] diff --git a/rustfs/src/runtime_capabilities.rs b/rustfs/src/runtime_capabilities.rs index a760a78e8..068d98aa7 100644 --- a/rustfs/src/runtime_capabilities.rs +++ b/rustfs/src/runtime_capabilities.rs @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities}; -use rustfs_storage_api::{ - CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, MemorySamplingState, ObservabilitySnapshot, +use crate::storage_api::{ + CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, EndpointServerPools, MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider, - UserspaceProfilingCapability, + UserspaceProfilingCapability, topology_snapshot_from_endpoint_pools_with_capabilities, }; const NOT_WIRED_INTO_RUNTIME: &str = "not wired into runtime"; @@ -185,8 +184,9 @@ fn cgroup_memory_status() -> CapabilityStatus { #[cfg(test)] mod tests { use super::*; - use crate::storage_api::{Endpoint, Endpoints, PoolEndpoints}; - use rustfs_storage_api::{CapabilityState, ObservabilitySnapshotProvider, TopologySnapshotProvider}; + use crate::storage_api::{ + CapabilityState, Endpoint, Endpoints, ObservabilitySnapshotProvider, PoolEndpoints, TopologySnapshotProvider, + }; #[tokio::test] async fn observability_provider_returns_platform_snapshot() { diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index 0093b6fa5..6c5f02d18 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -234,9 +234,8 @@ pub async fn init_event_notifier() { #[cfg(test)] mod tests { use super::{convert_ecstore_object_info, parse_host_and_port}; - use crate::storage_api::StorageObjectInfo; + use crate::storage_api::{StorageObjectInfo, TransitionedObject}; use chrono::{DateTime, Utc}; - use rustfs_storage_api::TransitionedObject; use std::{collections::HashMap, sync::Arc}; use time::{Duration, OffsetDateTime}; diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index ab107c695..9bcbb4066 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -15,7 +15,7 @@ use crate::server::runtime_sources; use crate::server::{ServiceState, ServiceStateManager}; use crate::server::{has_path_prefix, is_table_catalog_path}; -use crate::storage_api::{Endpoint, EndpointServerPools, is_dist_erasure}; +use crate::storage_api::{Endpoint, EndpointServerPools, StorageAdminApi, is_dist_erasure}; #[cfg(test)] use crate::storage_api::{Endpoints, PoolEndpoints}; use bytes::Bytes; @@ -26,7 +26,6 @@ use hyper::body::Incoming; use metrics::{counter, gauge}; use rustfs_common::GlobalReadiness; use rustfs_madmin::{Disk, StorageInfo}; -use rustfs_storage_api::StorageAdminApi; use std::future::Future; use std::pin::Pin; use std::sync::Arc; diff --git a/rustfs/src/startup_bucket_metadata.rs b/rustfs/src/startup_bucket_metadata.rs index 6c9808236..d62fe1c26 100644 --- a/rustfs/src/startup_bucket_metadata.rs +++ b/rustfs/src/startup_bucket_metadata.rs @@ -13,8 +13,9 @@ // limitations under the License. use crate::startup_runtime_sources; -use crate::storage_api::{ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config}; -use rustfs_storage_api::{BucketOperations, BucketOptions}; +use crate::storage_api::{ + BucketOperations, BucketOptions, ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config, +}; use std::{ io::{Error, Result}, sync::Arc, diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 0d35c3253..71976fb8f 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -29,9 +29,18 @@ pub(crate) use crate::storage::{ topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, try_migrate_server_config, update_erasure_type, verify_rpc_signature, }; +pub(crate) use rustfs_storage_api::{ + BucketOperations, BucketOptions, CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, HTTPPreconditions, + HTTPRangeError, HTTPRangeSpec, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, MemorySamplingState, + NamespaceLocking, ObjectIO, ObjectInfoOrErr, ObjectOperations, ObservabilitySnapshot, ObservabilitySnapshotProvider, + PlatformSupport, StorageAdminApi, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider, + UserspaceProfilingCapability, WalkOptions, +}; #[cfg(test)] pub(crate) use crate::storage::{DisksLayout, Endpoints, PoolEndpoints}; +#[cfg(test)] +pub(crate) use rustfs_storage_api::{CapabilityState, TransitionedObject}; pub(crate) mod access { pub(crate) use crate::storage::access::ReqInfo; diff --git a/rustfs/src/table_catalog.rs b/rustfs/src/table_catalog.rs index 334a86142..2c344fb22 100644 --- a/rustfs/src/table_catalog.rs +++ b/rustfs/src/table_catalog.rs @@ -29,8 +29,11 @@ use std::{ use crate::storage_api::{ BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, - BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata, - get_lock_acquire_timeout, table_catalog_path_hash, + BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, HTTPPreconditions, HTTPRangeSpec, + ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, + ListOperations as StorageListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, + ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as StorageObjectOperations, RUSTFS_META_BUCKET, StorageError, + WalkOptions as StorageWalkOptions, get_bucket_metadata, get_lock_acquire_timeout, table_catalog_path_hash, }; use bytes::Bytes; use datafusion::{ @@ -40,12 +43,6 @@ use datafusion::{ use http::HeaderMap; use metrics::{counter, histogram}; use rustfs_filemeta::FileInfo; -use rustfs_storage_api::{ - HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, - ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as StorageListOperations, - NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr, - ObjectOperations as StorageObjectOperations, WalkOptions as StorageWalkOptions, -}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use time::{Duration, OffsetDateTime}; use tokio::io::AsyncReadExt; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 9794f45a7..8ac7da8a5 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -167,6 +167,9 @@ RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_s RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_admin_storage_api_bypass_hits.txt" RUSTFS_ADMIN_STORAGE_API_ROOT_FACADE_HITS_FILE="${TMP_DIR}/rustfs_admin_storage_api_root_facade_hits.txt" RUSTFS_ROOT_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_bypass_hits.txt" +RUSTFS_ROOT_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_contract_bypass_hits.txt" +RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_storage_api_contract_bypass_hits.txt" +RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_admin_storage_api_contract_bypass_hits.txt" RUSTFS_STORAGE_DIRECT_APP_CONTEXT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_storage_direct_app_context_bypass_hits.txt" awk ' @@ -1436,11 +1439,11 @@ fi ( cd "$ROOT_DIR" - rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api' rustfs/src/app --glob '*_usecase.rs' || true + rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' || true ) >"$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" if [[ -s "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" ]]; then - report_failure "RustFS app usecases must consume S3 API helpers through rustfs/src/app/s3_api.rs: $(paste -sd '; ' "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE")" + report_failure "RustFS app usecases must consume S3 API helpers through rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE")" fi ( @@ -1535,6 +1538,45 @@ if [[ -s "$RUSTFS_ROOT_STORAGE_API_BYPASS_HITS_FILE" ]]; then report_failure "RustFS root/server/startup storage facades must stay behind rustfs/src/storage_api.rs: $(paste -sd '; ' "$RUSTFS_ROOT_STORAGE_API_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename '^use rustfs_storage_api|rustfs_storage_api::' \ + rustfs/src \ + --glob '*.rs' \ + --glob '!rustfs/src/admin/**' \ + --glob '!rustfs/src/app/**' \ + --glob '!rustfs/src/storage/**' \ + --glob '!rustfs/src/storage_api.rs' || true +) >"$RUSTFS_ROOT_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" + +if [[ -s "$RUSTFS_ROOT_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" ]]; then + report_failure "RustFS root/server/startup storage contracts must stay behind rustfs/src/storage_api.rs: $(paste -sd '; ' "$RUSTFS_ROOT_STORAGE_API_CONTRACT_BYPASS_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename '^use rustfs_storage_api|rustfs_storage_api::' \ + rustfs/src/app \ + --glob '*.rs' \ + --glob '!rustfs/src/app/storage_api.rs' || true +) >"$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" + +if [[ -s "$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" ]]; then + report_failure "RustFS app storage contracts must stay behind rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename '^use rustfs_storage_api|rustfs_storage_api::' \ + rustfs/src/admin \ + --glob '*.rs' \ + --glob '!rustfs/src/admin/storage_api.rs' || true +) >"$RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" + +if [[ -s "$RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" ]]; then + report_failure "RustFS admin storage contracts must stay behind rustfs/src/admin/storage_api.rs: $(paste -sd '; ' "$RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename 'crate::app::context::|use crate::app::context|app::context::' rustfs/src/storage --glob '*.rs' | diff --git a/scripts/layer-dependency-baseline.txt b/scripts/layer-dependency-baseline.txt index 35bee3976..bd4d3fb07 100644 --- a/scripts/layer-dependency-baseline.txt +++ b/scripts/layer-dependency-baseline.txt @@ -22,21 +22,21 @@ dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::ecfs::FS -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectVersionsParams -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectsV2Params -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_buckets_output -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_object_versions_output -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_objects_output -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_objects_v2_output -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::parse_list_object_versions_params -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::parse_list_objects_v2_params -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::common::rustfs_owner -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::multipart::ListMultipartUploadsParams -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::multipart::build_list_multipart_uploads_output -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::multipart::build_list_parts_output -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::multipart::parse_list_multipart_uploads_params -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::multipart::parse_list_parts_params -dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::multipart::parse_upload_part_number +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectVersionsParams +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectsV2Params +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_buckets_output +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_object_versions_output +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_objects_output +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_objects_v2_output +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::parse_list_object_versions_params +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::parse_list_objects_v2_params +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::common::rustfs_owner +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::ListMultipartUploadsParams +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::build_list_multipart_uploads_output +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::build_list_parts_output +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::parse_list_multipart_uploads_params +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::parse_list_parts_params +dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::parse_upload_part_number dep|rustfs/src/init.rs|infra->interface|crate::admin dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context dep|rustfs/src/server/http.rs|infra->interface|crate::admin