From 76af78385b18cdd00a3ae01834b26010cb186039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Mon, 22 Jun 2026 07:19:22 +0800 Subject: [PATCH] refactor: localize owner storage compat consumers (#3709) --- docs/architecture/migration-progress.md | 44 +++++++++++---- rustfs/src/admin/handlers/account_info.rs | 2 +- .../admin/handlers/audit_runtime_config.rs | 2 +- rustfs/src/admin/handlers/bucket_meta.rs | 4 +- rustfs/src/admin/handlers/config_admin.rs | 30 +++++------ rustfs/src/admin/handlers/heal.rs | 10 ++-- rustfs/src/admin/handlers/kms_dynamic.rs | 2 +- rustfs/src/admin/handlers/metrics.rs | 2 +- rustfs/src/admin/handlers/mod.rs | 1 + .../src/admin/handlers/object_zip_download.rs | 4 +- rustfs/src/admin/handlers/oidc.rs | 2 +- rustfs/src/admin/handlers/pools.rs | 2 +- rustfs/src/admin/handlers/quota.rs | 8 +-- rustfs/src/admin/handlers/rebalance.rs | 10 ++-- rustfs/src/admin/handlers/replication.rs | 18 +++---- rustfs/src/admin/handlers/site_replication.rs | 38 ++++++------- rustfs/src/admin/handlers/storage_compat.rs | 15 ++++++ rustfs/src/admin/handlers/sts.rs | 2 +- rustfs/src/admin/handlers/table_catalog.rs | 2 +- rustfs/src/admin/handlers/tier.rs | 6 +-- rustfs/src/admin/handlers/trace.rs | 2 +- rustfs/src/admin/mod.rs | 1 + rustfs/src/admin/router.rs | 52 +++++++++--------- rustfs/src/admin/router_storage_compat.rs | 15 ++++++ rustfs/src/admin/service/config.rs | 18 +++---- rustfs/src/admin/service/mod.rs | 1 + rustfs/src/admin/service/site_replication.rs | 4 +- rustfs/src/admin/service/storage_compat.rs | 15 ++++++ rustfs/src/app/admin_usecase.rs | 14 ++--- rustfs/src/app/bucket_usecase.rs | 14 ++--- rustfs/src/app/capacity_dirty_scope_test.rs | 4 +- rustfs/src/app/context.rs | 1 + rustfs/src/app/context/compat.rs | 18 +++---- rustfs/src/app/context/global.rs | 2 +- rustfs/src/app/context/handles.rs | 8 +-- rustfs/src/app/context/interfaces.rs | 6 +-- rustfs/src/app/context/startup.rs | 2 +- rustfs/src/app/context/storage_compat.rs | 15 ++++++ .../src/app/lifecycle_transition_api_test.rs | 10 ++-- rustfs/src/app/mod.rs | 1 + rustfs/src/app/multipart_usecase.rs | 50 ++++++++++------- rustfs/src/app/object_usecase.rs | 54 ++++++++++--------- rustfs/src/app/usecase_storage_compat.rs | 15 ++++++ rustfs/src/storage/rpc/http_service.rs | 10 ++-- rustfs/src/storage/rpc/mod.rs | 1 + rustfs/src/storage/rpc/node_service.rs | 8 +-- rustfs/src/storage/rpc/storage_compat.rs | 15 ++++++ rustfs/src/storage/s3_api/bucket.rs | 2 +- rustfs/src/storage/s3_api/mod.rs | 1 + rustfs/src/storage/s3_api/multipart.rs | 4 +- rustfs/src/storage/s3_api/storage_compat.rs | 17 ++++++ rustfs/src/storage/storage_compat.rs | 9 +--- scripts/check_architecture_migration_rules.sh | 20 +++++++ 53 files changed, 396 insertions(+), 217 deletions(-) create mode 100644 rustfs/src/admin/handlers/storage_compat.rs create mode 100644 rustfs/src/admin/router_storage_compat.rs create mode 100644 rustfs/src/admin/service/storage_compat.rs create mode 100644 rustfs/src/app/context/storage_compat.rs create mode 100644 rustfs/src/app/usecase_storage_compat.rs create mode 100644 rustfs/src/storage/rpc/storage_compat.rs create mode 100644 rustfs/src/storage/s3_api/storage_compat.rs diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index b08d5a1e3..a9fba588f 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,17 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-runtime-compat-consumer-batch` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100`. -- Stacked on: `overtrue/arch-disk-rpc-method-wrappers` pending API-097 merge. +- Branch: `overtrue/arch-owner-compat-consumer-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101`. +- Based on: `origin/main` after API-098/API-099/API-100 merge (#3708). - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: move capacity, server, startup, table catalog, - runtime-capability, workload-admission, error, and config-test ECStore - compatibility consumers from the root runtime facade into local boundaries. -- CI/script changes: guard against restoring consumer wrappers in the root - runtime compatibility facade and tolerate retiring the empty root facade. -- Docs changes: record the API-098/API-099/API-100 runtime consumer boundary cleanup. +- Rust code changes: move admin handler/service/router, app usecase/context, + and storage RPC/S3 API ECStore compatibility consumers behind owner-local + compatibility boundaries. +- CI/script changes: guard against restoring direct owner compatibility + consumers outside local compatibility boundary modules. +- Docs changes: record the API-101 owner compatibility consumer cleanup. ## Phase 0 Tasks @@ -435,6 +435,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block - Verification: RustFS test-target compile coverage, direct root compatibility consumer residual scan, migration and layer guards, formatting, diff hygiene, Rust risk scan, pre-commit quality gate, and three-expert review. +- [x] `API-101` Localize owner compatibility consumers. + - Completed slice: route admin handler/service/router, app usecase/context, + and storage RPC/S3 API compatibility consumers through local owner + boundary modules instead of their root owner `storage_compat.rs` facades. + - Acceptance: selected admin, app, and storage owner consumers no longer + import `crate::admin::storage_compat`, `crate::app::storage_compat`, or + `crate::storage::storage_compat` directly outside local compatibility + boundary modules; migration rules reject regressions. + - Must preserve: admin config and bucket metadata behavior, replication and + heal status mapping, app runtime context wiring, RPC verification and disk + lookup behavior, and S3 API ETag conversion. + - Verification: RustFS test-target compile coverage, owner compatibility + consumer residual scan, migration and layer guards, formatting, diff + hygiene, Rust risk scan, pre-commit quality gate, and three-expert review. - [x] `G-012` Inventory placement and repair invariants. - Acceptance: [`placement-repair-invariants.md`](placement-repair-invariants.md) records @@ -3476,6 +3490,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: +- Issue #660 API-101 current slice: + - `cargo check -p rustfs --tests`: passed. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Owner compatibility consumer residual scan: passed. + - Rust risk scan on changed Rust files and guard script: passed. + - `make pre-commit`: passed. + - Issue #660 API-098 current slice: - `cargo check -p rustfs --tests`: passed. - `cargo fmt --all`: passed. diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index 106c82d24..72dce0147 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -13,8 +13,8 @@ // limitations under the License. use crate::admin::auth::authenticate_request; +use crate::admin::handlers::storage_compat::versioning_sys::BucketVersioningSys; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::versioning_sys::BucketVersioningSys; use crate::app::context::resolve_object_store_handle; use crate::auth::get_condition_values; use crate::server::{ADMIN_PREFIX, RemoteAddr}; diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index 1e913d42d..7b0f9f321 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::handlers::storage_compat::{read_admin_config_without_migrate, save_admin_server_config}; use crate::admin::handlers::target_descriptor::AdminTargetSpec; -use crate::admin::storage_compat::{read_admin_config_without_migrate, save_admin_server_config}; use crate::app::context::resolve_object_store_handle; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; use rustfs_config::DEFAULT_DELIMITER; diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index f9a74bab0..fbcdfb34b 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::storage_compat::utils::{deserialize, serialize}; -use crate::admin::storage_compat::{ +use crate::admin::handlers::storage_compat::utils::{deserialize, serialize}; +use crate::admin::handlers::storage_compat::{ StorageError, metadata::{ BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index 47b380bc3..dd30c59fa 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -13,16 +13,16 @@ // limitations under the License. use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; +use crate::admin::handlers::storage_compat::{ + RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, + save_admin_config, save_admin_server_config, +}; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::service::config::{ apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload, validate_server_config, }; -use crate::admin::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; -use crate::admin::storage_compat::{ - RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, - save_admin_config, save_admin_server_config, -}; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; use crate::app::context::resolve_object_store_handle; use crate::auth::{check_key_valid, get_session_token}; @@ -710,7 +710,7 @@ fn success_response(config_applied: bool) -> S3Result S3Result> { +fn object_store() -> S3Result> { resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized")) } @@ -755,7 +755,7 @@ fn config_update_sub_system(directives: &[ConfigDirective]) -> S3Result S3Result<()> { if DEFAULT_KVS.get().is_none() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); } let Some(defaults) = DEFAULT_KVS.get() else { return Err(s3_error!(InternalError, "config defaults are not initialized")); @@ -1409,7 +1409,7 @@ fn env_help_key(sub_system: &str, key: &str) -> String { fn default_help_postfix(sub_system: &str, key: &str) -> String { if DEFAULT_KVS.get().is_none() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); } DEFAULT_KVS @@ -1896,7 +1896,7 @@ mod tests { #[test] fn full_config_export_can_be_reapplied() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); let mut original = ServerConfig::new(); apply_set_directives( &mut original, @@ -1943,7 +1943,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn build_help_response_appends_default_value_postfix() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response"); assert_eq!(response.keys_help.len(), 2); @@ -2056,7 +2056,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_includes_env_override_lines() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); temp_env::with_vars( [ ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")), @@ -2092,7 +2092,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_lists_env_only_targets() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || { let config = ServerConfig::new(); let rendered = String::from_utf8( @@ -2115,7 +2115,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_supports_specific_env_only_target_queries() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || { let config = ServerConfig::new(); let rendered = String::from_utf8( @@ -2138,7 +2138,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_orders_default_before_named_targets() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || { let mut config = ServerConfig::new(); apply_set_directives( @@ -2319,7 +2319,7 @@ identity_openid client_id="existing-client""#, #[test] fn storage_class_get_target_none_matches_full_export() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::handlers::storage_compat::init_admin_config_defaults(); let mut config = ServerConfig::new(); apply_set_directives( &mut config, diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 664017a15..20cfcd14d 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -13,9 +13,9 @@ // limitations under the License. use crate::admin::auth::{authenticate_request, validate_admin_request}; +use crate::admin::handlers::storage_compat::is_reserved_or_invalid_bucket; +use crate::admin::handlers::storage_compat::utils::is_valid_object_prefix; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::is_reserved_or_invalid_bucket; -use crate::admin::storage_compat::utils::is_valid_object_prefix; use crate::app::context::resolve_object_store_handle; use crate::server::ADMIN_PREFIX; use crate::server::RemoteAddr; @@ -362,10 +362,10 @@ fn should_handle_root_heal_directly(hip: &HealInitParams) -> bool { && hip.hs.set.is_none() } -fn map_root_heal_status(heal_err: Option) -> S3Result<()> { +fn map_root_heal_status(heal_err: Option) -> S3Result<()> { match heal_err { None => Ok(()), - Some(crate::admin::storage_compat::StorageError::NoHealRequired) => { + Some(crate::admin::handlers::storage_compat::StorageError::NoHealRequired) => { info!( event = EVENT_ADMIN_RESPONSE_EMITTED, component = LOG_COMPONENT_ADMIN_API, @@ -720,7 +720,7 @@ mod tests { encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response, map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target, }; - use crate::admin::storage_compat::StorageError; + use crate::admin::handlers::storage_compat::StorageError; use bytes::Bytes; use http::StatusCode; use http::Uri; diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index ca829ddc5..31b1585f7 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -15,8 +15,8 @@ //! KMS dynamic configuration admin API handlers use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::storage_compat::{read_admin_config, save_admin_config}; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::{read_admin_config, save_admin_config}; use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; diff --git a/rustfs/src/admin/handlers/metrics.rs b/rustfs/src/admin/handlers/metrics.rs index 3d3ab9c5f..a42b43021 100644 --- a/rustfs/src/admin/handlers/metrics.rs +++ b/rustfs/src/admin/handlers/metrics.rs @@ -19,8 +19,8 @@ //! exposition endpoint. use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics}; use crate::admin::router::Operation; -use crate::admin::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::RemoteAddr; use crate::storage::request_context::spawn_traced; diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 7e0c35caa..235041dcc 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -45,6 +45,7 @@ pub mod replication; pub mod scanner; pub mod service_account; pub mod site_replication; +pub(crate) mod storage_compat; pub mod sts; pub mod system; pub mod table_catalog; diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index 507acf32e..50c14dca6 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::handlers::storage_compat::get_global_region; use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::get_global_region; use crate::app::context::resolve_object_store_handle; use crate::auth::{check_key_valid, get_session_token}; use crate::error::ApiError; @@ -649,7 +649,7 @@ async fn preflight_zip_items(request: &CreateObjectZipDownloadRequest, items: &[ Ok(()) } -fn storage_error_to_s3(err: crate::admin::storage_compat::Error) -> s3s::S3Error { +fn storage_error_to_s3(err: crate::admin::handlers::storage_compat::Error) -> s3s::S3Error { ApiError::from(err).into() } diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index 33d1b07c9..56ffa4a96 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -14,8 +14,8 @@ use super::sts::create_oidc_sts_credentials; use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::storage_compat::{read_admin_config_without_migrate, save_admin_server_config}; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::{read_admin_config_without_migrate, save_admin_server_config}; use crate::app::context::resolve_object_store_handle; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr}; diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index 6f5057527..5e4cecd32 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -99,7 +99,7 @@ macro_rules! log_pool_response_emitted { }; } -fn endpoints_from_context() -> Option { +fn endpoints_from_context() -> Option { resolve_endpoints_handle() } diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index ca8061e6b..35283de67 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -15,10 +15,10 @@ //! Quota admin handlers for HTTP API use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket}; +use crate::admin::handlers::storage_compat::metadata_sys::BucketMetadataSys; +use crate::admin::handlers::storage_compat::quota::checker::QuotaChecker; +use crate::admin::handlers::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::metadata_sys::BucketMetadataSys; -use crate::admin::storage_compat::quota::checker::QuotaChecker; -use crate::admin::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::ADMIN_PREFIX; @@ -175,7 +175,7 @@ async fn current_usage_from_context(bucket: &str) -> u64 { return 0; }; - match crate::admin::storage_compat::load_data_usage_from_backend(store).await { + match crate::admin::handlers::storage_compat::load_data_usage_from_backend(store).await { Ok(data_usage_info) => data_usage_info .buckets_usage .get(bucket) diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index e07f35ed6..58d1989a8 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::storage_compat::{ +use crate::admin::handlers::storage_compat::{ DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys, }; use crate::{ @@ -148,7 +148,7 @@ fn build_rebalance_pool_progress( now: OffsetDateTime, stop_time: Option, percent_free_goal: f64, - ps: &crate::admin::storage_compat::RebalanceStats, + ps: &crate::admin::handlers::storage_compat::RebalanceStats, ) -> Option { let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64; let terminal_time = ps.info.end_time.or(stop_time); @@ -191,7 +191,7 @@ fn build_rebalance_pool_statuses( now: OffsetDateTime, stop_time: Option, percent_free_goal: f64, - pool_stats: &[crate::admin::storage_compat::RebalanceStats], + pool_stats: &[crate::admin::handlers::storage_compat::RebalanceStats], disk_stats: &[DiskStat], ) -> Vec { pool_stats @@ -561,7 +561,9 @@ mod rebalance_handler_tests { RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used, rebalance_remaining_buckets, rebalance_used_pct, }; - use crate::admin::storage_compat::{DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats}; + use crate::admin::handlers::storage_compat::{ + DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats, + }; use time::OffsetDateTime; #[test] diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 8d662fa16..9c13a6da9 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -14,16 +14,16 @@ use crate::admin::auth::validate_admin_request; use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint; +use crate::admin::handlers::storage_compat::StorageError; +use crate::admin::handlers::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys}; +use crate::admin::handlers::storage_compat::global_rustfs_port; +use crate::admin::handlers::storage_compat::metadata::BUCKET_TARGETS_FILE; +use crate::admin::handlers::storage_compat::metadata_sys; +use crate::admin::handlers::storage_compat::metadata_sys::get_replication_config; +use crate::admin::handlers::storage_compat::replication::BucketStats; +use crate::admin::handlers::storage_compat::replication::GLOBAL_REPLICATION_STATS; +use crate::admin::handlers::storage_compat::target::BucketTarget; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::storage_compat::StorageError; -use crate::admin::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys}; -use crate::admin::storage_compat::global_rustfs_port; -use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE; -use crate::admin::storage_compat::metadata_sys; -use crate::admin::storage_compat::metadata_sys::get_replication_config; -use crate::admin::storage_compat::replication::BucketStats; -use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS; -use crate::admin::storage_compat::target::BucketTarget; use crate::admin::utils::read_compatible_admin_body; use crate::app::context::resolve_object_store_handle; use crate::auth::{check_key_valid, get_session_token}; diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index ec53b175b..9ff508674 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -13,25 +13,27 @@ // limitations under the License. use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::storage_compat::Error as StorageError; +use crate::admin::handlers::storage_compat::bucket_target_sys::BucketTargetSys; +use crate::admin::handlers::storage_compat::metadata::{ + BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, + BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, +}; +use crate::admin::handlers::storage_compat::metadata_sys; +use crate::admin::handlers::storage_compat::replication::GLOBAL_REPLICATION_STATS; +use crate::admin::handlers::storage_compat::replication::{ResyncOpts, get_global_replication_pool}; +use crate::admin::handlers::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; +use crate::admin::handlers::storage_compat::utils::{deserialize, serialize}; +use crate::admin::handlers::storage_compat::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; +use crate::admin::handlers::storage_compat::{delete_admin_config, read_admin_config, save_admin_config}; +use crate::admin::handlers::storage_compat::{ + get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port, +}; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::site_replication_identity::{ canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint, site_identity_key, }; -use crate::admin::storage_compat::Error as StorageError; -use crate::admin::storage_compat::bucket_target_sys::BucketTargetSys; -use crate::admin::storage_compat::metadata::{ - BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, - BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, -}; -use crate::admin::storage_compat::metadata_sys; -use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS; -use crate::admin::storage_compat::replication::{ResyncOpts, get_global_replication_pool}; -use crate::admin::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; -use crate::admin::storage_compat::utils::{deserialize, serialize}; -use crate::admin::storage_compat::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; -use crate::admin::storage_compat::{delete_admin_config, read_admin_config, save_admin_config}; -use crate::admin::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port}; use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body}; use crate::app::context::resolve_object_store_handle; use crate::auth::{check_key_valid, get_session_token}; @@ -652,7 +654,7 @@ async fn site_replication_peer_client() -> S3Result { built } -fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::EndpointServerPools>) -> bool { +fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::handlers::storage_compat::EndpointServerPools>) -> bool { if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() { return true; } @@ -3353,7 +3355,7 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option } fn bucket_meta_local_updated_at( - bucket_meta: &crate::admin::storage_compat::metadata::BucketMetadata, + bucket_meta: &crate::admin::handlers::storage_compat::metadata::BucketMetadata, config_file: &str, ) -> OffsetDateTime { match config_file { @@ -4574,8 +4576,8 @@ impl Operation for SRRotateServiceAccountHandler { #[cfg(test)] mod tests { use super::*; - use crate::admin::storage_compat::Endpoint; - use crate::admin::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::admin::handlers::storage_compat::Endpoint; + use crate::admin::handlers::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints}; use http::{HeaderMap, HeaderValue, Uri}; use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation}; use rustfs_policy::policy::action::S3Action; diff --git a/rustfs/src/admin/handlers/storage_compat.rs b/rustfs/src/admin/handlers/storage_compat.rs new file mode 100644 index 000000000..36fec6af3 --- /dev/null +++ b/rustfs/src/admin/handlers/storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::admin::storage_compat::*; diff --git a/rustfs/src/admin/handlers/sts.rs b/rustfs/src/admin/handlers/sts.rs index 058a2c40a..e3fb92dad 100644 --- a/rustfs/src/admin/handlers/sts.rs +++ b/rustfs/src/admin/handlers/sts.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::is_admin::IsAdminHandler; -use crate::admin::storage_compat::utils::serialize; +use crate::admin::handlers::storage_compat::utils::serialize; use crate::{ admin::{ handlers::site_replication::site_replication_iam_change_hook, diff --git a/rustfs/src/admin/handlers/table_catalog.rs b/rustfs/src/admin/handlers/table_catalog.rs index bf9a01ab9..bd4066bf9 100644 --- a/rustfs/src/admin/handlers/table_catalog.rs +++ b/rustfs/src/admin/handlers/table_catalog.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys}; +use crate::admin::handlers::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys}; use crate::admin::{ auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object}, router::{AdminOperation, Operation, S3Router}, diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 60043d771..b493d99ee 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -13,8 +13,8 @@ // limitations under the License. #![allow(unused_variables, unused_mut, unused_must_use)] -use crate::admin::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState; -use crate::admin::storage_compat::{ +use crate::admin::handlers::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState; +use crate::admin::handlers::storage_compat::{ AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, TierConfig, TierCreds, TierType, get_global_notification_sys, storageclass, @@ -906,7 +906,7 @@ impl Operation for ClearTier { #[cfg(test)] mod tests { use super::*; - use crate::admin::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats; + use crate::admin::handlers::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats; use http::Uri; use matchit::Router; diff --git a/rustfs/src/admin/handlers/trace.rs b/rustfs/src/admin/handlers/trace.rs index 3fa47bd9e..bbcd8ccf6 100644 --- a/rustfs/src/admin/handlers/trace.rs +++ b/rustfs/src/admin/handlers/trace.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::handlers::storage_compat::PeerRestClient; use crate::admin::router::Operation; -use crate::admin::storage_compat::PeerRestClient; use crate::app::context::resolve_endpoints_handle; use http::StatusCode; use hyper::Uri; diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index c678b2155..35fa3374a 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -21,6 +21,7 @@ mod plugin_contract; #[allow(dead_code)] pub(crate) mod route_policy; pub mod router; +pub(crate) mod router_storage_compat; pub mod service; pub mod site_replication_identity; pub(crate) mod storage_compat; diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 3590888ec..348102398 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -14,23 +14,23 @@ use crate::admin::console::{is_console_path, make_console_server}; use crate::admin::handlers::oidc::is_oidc_path; -use crate::admin::storage_compat::GLOBAL_BOOT_TIME; -use crate::admin::storage_compat::PeerRestClient; -use crate::admin::storage_compat::bandwidth::monitor::BandwidthDetails; -use crate::admin::storage_compat::bucket_target_sys::{ +use crate::admin::router_storage_compat::GLOBAL_BOOT_TIME; +use crate::admin::router_storage_compat::PeerRestClient; +use crate::admin::router_storage_compat::bandwidth::monitor::BandwidthDetails; +use crate::admin::router_storage_compat::bucket_target_sys::{ BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient, }; -use crate::admin::storage_compat::get_global_notification_sys; -use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE; -use crate::admin::storage_compat::metadata_sys; -use crate::admin::storage_compat::read_admin_config_without_migrate; -use crate::admin::storage_compat::replication::{ +use crate::admin::router_storage_compat::get_global_notification_sys; +use crate::admin::router_storage_compat::metadata::BUCKET_TARGETS_FILE; +use crate::admin::router_storage_compat::metadata_sys; +use crate::admin::router_storage_compat::read_admin_config_without_migrate; +use crate::admin::router_storage_compat::replication::{ BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ResyncOpts, get_global_replication_pool, }; -use crate::admin::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets}; -use crate::admin::storage_compat::versioning_sys::BucketVersioningSys; -use crate::admin::storage_compat::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; -use crate::admin::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region}; +use crate::admin::router_storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets}; +use crate::admin::router_storage_compat::versioning_sys::BucketVersioningSys; +use crate::admin::router_storage_compat::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; +use crate::admin::router_storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region}; use crate::app::context::resolve_object_store_handle; use crate::app::object_usecase::DefaultObjectUsecase; use crate::auth::{check_key_valid, get_session_token}; @@ -1419,7 +1419,9 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> { async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> { match metadata_sys::get_replication_config(bucket).await { Ok(_) => Ok(()), - Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)), + Err(crate::admin::router_storage_compat::StorageError::ConfigNotFound) => { + Err(s3_error!(ReplicationConfigurationNotFoundError)) + } Err(err) => Err(ApiError::from(err).into()), } } @@ -1944,7 +1946,7 @@ async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget) fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions { PutObjectOptions { - internal: crate::admin::storage_compat::bucket_target_sys::AdvancedPutOptions { + internal: crate::admin::router_storage_compat::bucket_target_sys::AdvancedPutOptions { source_version_id: Uuid::new_v4().to_string(), replication_status: ReplicationStatusType::Replica, source_mtime: now, @@ -2059,7 +2061,7 @@ async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result { .object_lock_enabled .as_ref() .is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)), - Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Ok(false), + Err(crate::admin::router_storage_compat::StorageError::ConfigNotFound) => Ok(false), Err(err) => Err(ApiError::from(err).into()), } } @@ -2773,7 +2775,7 @@ mod tests { #[test] fn apply_replication_reset_to_targets_updates_matching_target() { let mut targets = BucketTargets { - targets: vec![crate::admin::storage_compat::target::BucketTarget { + targets: vec![crate::admin::router_storage_compat::target::BucketTarget { arn: "arn:target".to_string(), ..Default::default() }], @@ -2795,10 +2797,10 @@ mod tests { let mut status = BucketReplicationResyncStatus::new(); status.targets_map.insert( "arn:z".to_string(), - crate::admin::storage_compat::replication::TargetReplicationResyncStatus { + crate::admin::router_storage_compat::replication::TargetReplicationResyncStatus { resync_id: "rid-z".to_string(), last_update: Some(datetime!(2025-01-03 00:00 UTC)), - resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed, + resync_status: crate::admin::router_storage_compat::replication::ResyncStatusType::ResyncFailed, failed_count: 2, failed_size: 4, bucket: "bucket-z".to_string(), @@ -2808,10 +2810,10 @@ mod tests { ); status.targets_map.insert( "arn:a".to_string(), - crate::admin::storage_compat::replication::TargetReplicationResyncStatus { + crate::admin::router_storage_compat::replication::TargetReplicationResyncStatus { resync_id: "rid-a".to_string(), last_update: Some(datetime!(2025-01-02 00:00 UTC)), - resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted, + resync_status: crate::admin::router_storage_compat::replication::ResyncStatusType::ResyncCompleted, replicated_count: 3, replicated_size: 9, bucket: "bucket-a".to_string(), @@ -2841,10 +2843,10 @@ mod tests { let mut status = BucketReplicationResyncStatus::new(); status.targets_map.insert( "arn:z".to_string(), - crate::admin::storage_compat::replication::TargetReplicationResyncStatus { + crate::admin::router_storage_compat::replication::TargetReplicationResyncStatus { resync_id: "rid-z".to_string(), last_update: Some(datetime!(2025-02-03 00:00 UTC)), - resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed, + resync_status: crate::admin::router_storage_compat::replication::ResyncStatusType::ResyncFailed, failed_count: 2, failed_size: 4, bucket: "bucket-z".to_string(), @@ -2854,10 +2856,10 @@ mod tests { ); status.targets_map.insert( "arn:a".to_string(), - crate::admin::storage_compat::replication::TargetReplicationResyncStatus { + crate::admin::router_storage_compat::replication::TargetReplicationResyncStatus { resync_id: "rid-a".to_string(), last_update: Some(datetime!(2025-02-02 00:00 UTC)), - resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted, + resync_status: crate::admin::router_storage_compat::replication::ResyncStatusType::ResyncCompleted, replicated_count: 3, replicated_size: 9, bucket: "bucket-a".to_string(), diff --git a/rustfs/src/admin/router_storage_compat.rs b/rustfs/src/admin/router_storage_compat.rs new file mode 100644 index 000000000..36fec6af3 --- /dev/null +++ b/rustfs/src/admin/router_storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::admin::storage_compat::*; diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index d259347d4..368895aa3 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::storage_compat::get_global_notification_sys; -use crate::admin::storage_compat::set_global_storage_class; -use crate::admin::storage_compat::storageclass; -use crate::admin::storage_compat::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate}; +use crate::admin::service::storage_compat::get_global_notification_sys; +use crate::admin::service::storage_compat::set_global_storage_class; +use crate::admin::service::storage_compat::storageclass; +use crate::admin::service::storage_compat::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate}; use crate::app::context::resolve_object_store_handle; use rustfs_audit::reload_audit_config; use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS}; @@ -371,7 +371,7 @@ pub async fn signal_config_snapshot_reload() { #[cfg(test)] mod tests { use super::*; - use crate::admin::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; + use crate::admin::service::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES}; use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; @@ -427,7 +427,7 @@ mod tests { #[test] fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::service::storage_compat::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); @@ -441,7 +441,7 @@ mod tests { #[test] fn validate_audit_subsystem_config_rejects_relative_queue_dir() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::service::storage_compat::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); @@ -456,7 +456,7 @@ mod tests { #[test] fn validate_identity_openid_config_rejects_missing_openid_scope() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::service::storage_compat::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); @@ -473,7 +473,7 @@ mod tests { #[test] fn validate_identity_openid_config_rejects_invalid_named_provider_id() { - crate::admin::storage_compat::init_admin_config_defaults(); + crate::admin::service::storage_compat::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults"); let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target"); diff --git a/rustfs/src/admin/service/mod.rs b/rustfs/src/admin/service/mod.rs index 5af8b3f7e..097e35a41 100644 --- a/rustfs/src/admin/service/mod.rs +++ b/rustfs/src/admin/service/mod.rs @@ -14,3 +14,4 @@ pub mod config; pub mod site_replication; +pub(crate) mod storage_compat; diff --git a/rustfs/src/admin/service/site_replication.rs b/rustfs/src/admin/service/site_replication.rs index ddad748aa..aaab96bac 100644 --- a/rustfs/src/admin/service/site_replication.rs +++ b/rustfs/src/admin/service/site_replication.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::admin::service::storage_compat::Error as StorageError; +use crate::admin::service::storage_compat::{read_admin_config, save_admin_config}; use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with}; -use crate::admin::storage_compat::Error as StorageError; -use crate::admin::storage_compat::{read_admin_config, save_admin_config}; use crate::app::context::resolve_object_store_handle; use rustfs_madmin::PeerInfo; use s3s::{S3Error, S3ErrorCode, S3Result}; diff --git a/rustfs/src/admin/service/storage_compat.rs b/rustfs/src/admin/service/storage_compat.rs new file mode 100644 index 000000000..36fec6af3 --- /dev/null +++ b/rustfs/src/admin/service/storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::admin::storage_compat::*; diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index 2707802e3..753433b4b 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -15,11 +15,13 @@ //! Admin application use-case contracts. use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context}; -use crate::app::storage_compat::ECStore; -use crate::app::storage_compat::EndpointServerPools; -use crate::app::storage_compat::get_server_info; -use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free}; -use crate::app::storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; +use crate::app::usecase_storage_compat::ECStore; +use crate::app::usecase_storage_compat::EndpointServerPools; +use crate::app::usecase_storage_compat::get_server_info; +use crate::app::usecase_storage_compat::{ + PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, +}; +use crate::app::usecase_storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; use crate::capacity::resolve_admin_used_capacity; use crate::error::ApiError; use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness}; @@ -316,7 +318,7 @@ impl DefaultAdminUsecase { #[cfg(test)] mod tests { use super::*; - use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus}; + use crate::app::usecase_storage_compat::{PoolDecommissionInfo, PoolStatus}; use time::OffsetDateTime; #[tokio::test] diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index f49549905..dd7c91a7e 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -20,12 +20,12 @@ use crate::admin::handlers::site_replication::{ use crate::app::context::{ AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context, }; -use crate::app::storage_compat::ECStore; -use crate::app::storage_compat::StorageError; -use crate::app::storage_compat::get_global_notification_sys; -use crate::app::storage_compat::object_api_utils::to_s3s_etag; -use crate::app::storage_compat::{AppObjectLockConfigExt as _, AppVersioningConfigExt as _}; -use crate::app::storage_compat::{ +use crate::app::usecase_storage_compat::ECStore; +use crate::app::usecase_storage_compat::StorageError; +use crate::app::usecase_storage_compat::get_global_notification_sys; +use crate::app::usecase_storage_compat::object_api_utils::to_s3s_etag; +use crate::app::usecase_storage_compat::{AppObjectLockConfigExt as _, AppVersioningConfigExt as _}; +use crate::app::usecase_storage_compat::{ bucket_target_sys::BucketTargetSys, lifecycle::bucket_lifecycle_ops::{ enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_lifecycle_config, @@ -2285,7 +2285,7 @@ mod tests { BucketTargets { targets: arns .iter() - .map(|arn| crate::app::storage_compat::target::BucketTarget { + .map(|arn| crate::app::usecase_storage_compat::target::BucketTarget { arn: (*arn).to_string(), target_type: BucketTargetType::ReplicationService, ..Default::default() diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index 9d0fa85bb..36275c496 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::app::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys}; +use crate::app::usecase_storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys}; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; @@ -77,7 +77,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec, Arc) { }; let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); - crate::app::storage_compat::init_local_disks(endpoint_pools.clone()) + crate::app::usecase_storage_compat::init_local_disks(endpoint_pools.clone()) .await .unwrap(); diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index f74872c75..664a53f52 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -21,6 +21,7 @@ mod global; mod handles; mod interfaces; mod startup; +pub(crate) mod storage_compat; pub use compat::*; pub use global::*; diff --git a/rustfs/src/app/context/compat.rs b/rustfs/src/app/context/compat.rs index 95133ddd4..110bdb0b2 100644 --- a/rustfs/src/app/context/compat.rs +++ b/rustfs/src/app/context/compat.rs @@ -17,11 +17,11 @@ use super::handles::{ default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface, default_server_config_interface, default_tier_config_interface, }; -use crate::app::storage_compat::ECStore; -use crate::app::storage_compat::EndpointServerPools; -use crate::app::storage_compat::TierConfigMgr; -use crate::app::storage_compat::metadata_sys::BucketMetadataSys; -use crate::app::storage_compat::new_object_layer_fn; +use crate::app::context::storage_compat::ECStore; +use crate::app::context::storage_compat::EndpointServerPools; +use crate::app::context::storage_compat::TierConfigMgr; +use crate::app::context::storage_compat::metadata_sys::BucketMetadataSys; +use crate::app::context::storage_compat::new_object_layer_fn; #[cfg(test)] use crate::config::RustFSBufferConfig; use rustfs_config::server_config::Config; @@ -126,10 +126,10 @@ mod tests { BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, ServerConfigInterface, TierConfigInterface, }; - use crate::app::storage_compat::Endpoint; - use crate::app::storage_compat::init_local_disks; - use crate::app::storage_compat::new_object_layer_fn; - use crate::app::storage_compat::{Endpoints, PoolEndpoints}; + use crate::app::context::storage_compat::Endpoint; + use crate::app::context::storage_compat::init_local_disks; + use crate::app::context::storage_compat::new_object_layer_fn; + use crate::app::context::storage_compat::{Endpoints, PoolEndpoints}; use crate::config::{RustFSBufferConfig, WorkloadProfile}; use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use std::path::PathBuf; diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index e105d0183..8fd9033b8 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -21,7 +21,7 @@ use super::interfaces::{ BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface, }; -use crate::app::storage_compat::{ECStore, set_object_store_resolver}; +use crate::app::context::storage_compat::{ECStore, set_object_store_resolver}; use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use rustfs_kms::KmsServiceManager; use std::sync::{Arc, OnceLock}; diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 0de00f7c4..0df7ae530 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -16,10 +16,10 @@ use super::interfaces::{ BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface, }; -use crate::app::storage_compat::EndpointServerPools; -use crate::app::storage_compat::TierConfigMgr; -use crate::app::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; -use crate::app::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr}; +use crate::app::context::storage_compat::EndpointServerPools; +use crate::app::context::storage_compat::TierConfigMgr; +use crate::app::context::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; +use crate::app::context::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr}; use crate::config::{RustFSBufferConfig, get_global_buffer_config}; use async_trait::async_trait; use rustfs_config::server_config::Config; diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index cfd65e61d..1930ccd1e 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::app::storage_compat::EndpointServerPools; -use crate::app::storage_compat::TierConfigMgr; -use crate::app::storage_compat::metadata_sys::BucketMetadataSys; +use crate::app::context::storage_compat::EndpointServerPools; +use crate::app::context::storage_compat::TierConfigMgr; +use crate::app::context::storage_compat::metadata_sys::BucketMetadataSys; use crate::config::RustFSBufferConfig; use async_trait::async_trait; use rustfs_config::server_config::Config; diff --git a/rustfs/src/app/context/startup.rs b/rustfs/src/app/context/startup.rs index 442c56eb3..d61b69e37 100644 --- a/rustfs/src/app/context/startup.rs +++ b/rustfs/src/app/context/startup.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::global::{AppContext, get_global_app_context, init_global_app_context}; -use crate::app::storage_compat::ECStore; +use crate::app::context::storage_compat::ECStore; use rustfs_kms::KmsServiceManager; use std::io::{Error, Result}; use std::sync::Arc; diff --git a/rustfs/src/app/context/storage_compat.rs b/rustfs/src/app/context/storage_compat.rs new file mode 100644 index 000000000..21ce22e76 --- /dev/null +++ b/rustfs/src/app/context/storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::app::storage_compat::*; diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 3c5ed5cf5..5b9c2f388 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -14,7 +14,7 @@ use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase}; use crate::app::bucket_usecase::DefaultBucketUsecase; -use crate::app::storage_compat::{ +use crate::app::usecase_storage_compat::{ AppWarmBackend, ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackendGetOpts, metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG}, @@ -107,7 +107,7 @@ async fn setup_test_env() -> (Vec, Arc) { let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); - crate::app::storage_compat::init_local_disks(endpoint_pools.clone()) + crate::app::usecase_storage_compat::init_local_disks(endpoint_pools.clone()) .await .unwrap(); @@ -126,7 +126,7 @@ async fn setup_test_env() -> (Vec, Arc) { let buckets = buckets_list.into_iter().map(|v| v.name).collect(); metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; - crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + crate::app::usecase_storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone())); @@ -926,7 +926,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() { .expect("Failed to set lifecycle configuration"); let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; - crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( + crate::app::usecase_storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( ecstore.clone(), bucket.as_str(), ) @@ -986,7 +986,7 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() { .expect("Failed to set lifecycle configuration"); let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; - crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( + crate::app::usecase_storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( ecstore.clone(), bucket.as_str(), ) diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index 3a066cf39..e3c8f68fe 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -22,6 +22,7 @@ pub mod multipart_usecase; pub mod object_usecase; mod select_object; pub(crate) mod storage_compat; +pub(crate) mod usecase_storage_compat; #[cfg(test)] mod capacity_dirty_scope_test; diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index d25a231d2..f261d346a 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -16,16 +16,16 @@ use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context}; use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write}; -use crate::app::storage_compat::ECStore; -use crate::app::storage_compat::is_disk_compressible; -use crate::app::storage_compat::is_valid_storage_class; -use crate::app::storage_compat::object_api_utils::to_s3s_etag; -use crate::app::storage_compat::quota::checker::QuotaChecker; +use crate::app::usecase_storage_compat::ECStore; +use crate::app::usecase_storage_compat::is_disk_compressible; +use crate::app::usecase_storage_compat::is_valid_storage_class; +use crate::app::usecase_storage_compat::object_api_utils::to_s3s_etag; +use crate::app::usecase_storage_compat::quota::checker::QuotaChecker; #[cfg(test)] -use crate::app::storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; -use crate::app::storage_compat::{HashReader, WritePlan}; -use crate::app::storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found}; -use crate::app::storage_compat::{ +use crate::app::usecase_storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; +use crate::app::usecase_storage_compat::{HashReader, WritePlan}; +use crate::app::usecase_storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found}; +use crate::app::usecase_storage_compat::{ lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate}, metadata_sys, quota::QuotaOperation, @@ -482,7 +482,7 @@ impl DefaultMultipartUsecase { )); } // Update quota tracking after successful multipart upload - crate::app::storage_compat::record_bucket_object_write_memory( + crate::app::usecase_storage_compat::record_bucket_object_write_memory( &bucket, previous_current_size, obj_info.size.max(0) as u64, @@ -675,7 +675,7 @@ impl DefaultMultipartUsecase { rustfs_utils::http::insert_str( &mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, - crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()), + crate::app::usecase_storage_compat::compression_metadata_value(CompressionAlgorithm::default()), ); } @@ -894,9 +894,12 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; let ssec_write = match ssec_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) + crate::app::usecase_storage_compat::WriteEncryption::multipart_object_key( + ssec_material.key_bytes, + part_id as u32, + ) } - crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart( + crate::storage::sse::EncryptionKeyKind::Direct => crate::app::usecase_storage_compat::WriteEncryption::multipart( ssec_material.key_bytes, ssec_material.base_nonce, part_id, @@ -916,9 +919,12 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; let managed_write = match managed_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + crate::app::usecase_storage_compat::WriteEncryption::multipart_object_key( + managed_material.key_bytes, + part_id as u32, + ) } - crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart( + crate::storage::sse::EncryptionKeyKind::Direct => crate::app::usecase_storage_compat::WriteEncryption::multipart( managed_material.key_bytes, managed_material.base_nonce, part_id, @@ -1237,9 +1243,12 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; let ssec_write = match ssec_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) + crate::app::usecase_storage_compat::WriteEncryption::multipart_object_key( + ssec_material.key_bytes, + part_id as u32, + ) } - crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart( + crate::storage::sse::EncryptionKeyKind::Direct => crate::app::usecase_storage_compat::WriteEncryption::multipart( ssec_material.key_bytes, ssec_material.base_nonce, part_id, @@ -1263,9 +1272,12 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; let managed_write = match managed_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + crate::app::usecase_storage_compat::WriteEncryption::multipart_object_key( + managed_material.key_bytes, + part_id as u32, + ) } - crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart( + crate::storage::sse::EncryptionKeyKind::Direct => crate::app::usecase_storage_compat::WriteEncryption::multipart( managed_material.key_bytes, managed_material.base_nonce, part_id, diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 74b0ef36e..22fbcc7f1 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -48,21 +48,21 @@ use metrics::{counter, histogram}; use pin_project_lite::pin_project; use rustfs_object_capacity::capacity_manager::get_capacity_manager; // Performance metrics recording (with zero-copy-metrics integration) -use crate::app::storage_compat::ECStore; -use crate::app::storage_compat::object_api_utils::to_s3s_etag; -use crate::app::storage_compat::quota::checker::QuotaChecker; -use crate::app::storage_compat::storageclass; -use crate::app::storage_compat::{ +use crate::app::usecase_storage_compat::ECStore; +use crate::app::usecase_storage_compat::object_api_utils::to_s3s_etag; +use crate::app::usecase_storage_compat::quota::checker::QuotaChecker; +use crate::app::usecase_storage_compat::storageclass; +use crate::app::usecase_storage_compat::{ AppReplicationConfigExt as _, AppVersioningConfigExt as _, predict_lifecycle_expiration, validate_restore_request, }; -use crate::app::storage_compat::{DiskError, is_all_buckets_not_found}; -use crate::app::storage_compat::{DynReader, HashReader, WritePlan, wrap_reader}; -use crate::app::storage_compat::{ +use crate::app::usecase_storage_compat::{DiskError, is_all_buckets_not_found}; +use crate::app::usecase_storage_compat::{DynReader, HashReader, WritePlan, wrap_reader}; +use crate::app::usecase_storage_compat::{ Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; -use crate::app::storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; -use crate::app::storage_compat::{get_lock_acquire_timeout, is_valid_storage_class}; -use crate::app::storage_compat::{ +use crate::app::usecase_storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; +use crate::app::usecase_storage_compat::{get_lock_acquire_timeout, is_valid_storage_class}; +use crate::app::usecase_storage_compat::{ lifecycle::{ bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{enqueue_transition_immediate, post_restore_opts}, @@ -279,12 +279,12 @@ async fn enqueue_transitioned_delete_cleanup( let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup); let je = if opts.delete_prefix { - crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry( + crate::app::usecase_storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry( &existing.transitioned_object, ) } else { let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()); - crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry( + crate::app::usecase_storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry( version_id, opts.versioned, opts.version_suspended, @@ -295,9 +295,9 @@ async fn enqueue_transitioned_delete_cleanup( return Ok(()); }; - crate::app::storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?; + crate::app::usecase_storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?; - let mut expiry_state = crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState + let mut expiry_state = crate::app::usecase_storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState .write() .await; if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await { @@ -1542,7 +1542,7 @@ impl DefaultObjectUsecase { #[allow(clippy::too_many_arguments)] async fn prepare_get_object_read( req: &S3Request, - store: &crate::app::storage_compat::ECStore, + store: &crate::app::usecase_storage_compat::ECStore, manager: &ConcurrencyManager, bucket: &str, key: &str, @@ -2136,7 +2136,7 @@ impl DefaultObjectUsecase { insert_str( &mut metadata, SUFFIX_COMPRESSION, - crate::app::storage_compat::compression_metadata_value(algorithm), + crate::app::usecase_storage_compat::compression_metadata_value(algorithm), ); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); @@ -2151,7 +2151,7 @@ impl DefaultObjectUsecase { insert_str( &mut opts.user_defined, SUFFIX_COMPRESSION, - crate::app::storage_compat::compression_metadata_value(algorithm), + crate::app::usecase_storage_compat::compression_metadata_value(algorithm), ); insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); @@ -2343,7 +2343,7 @@ impl DefaultObjectUsecase { maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; // Fast in-memory update for immediate quota and admin usage consistency - crate::app::storage_compat::record_bucket_object_write_memory( + crate::app::usecase_storage_compat::record_bucket_object_write_memory( &bucket, previous_current_size, obj_info.size.max(0) as u64, @@ -3213,7 +3213,7 @@ impl DefaultObjectUsecase { insert_str( &mut compress_metadata, SUFFIX_COMPRESSION, - crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()), + crate::app::usecase_storage_compat::compression_metadata_value(CompressionAlgorithm::default()), ); insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); } else { @@ -3306,8 +3306,12 @@ impl DefaultObjectUsecase { // Update quota tracking after successful copy if has_bucket_metadata { - crate::app::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64) - .await; + crate::app::usecase_storage_compat::record_bucket_object_write_memory( + &bucket, + previous_current_size, + oi.size.max(0) as u64, + ) + .await; } let raw_dest_version = oi.version_id.map(|v| v.to_string()); @@ -3584,7 +3588,7 @@ impl DefaultObjectUsecase { ); } let size = object_sizes[i].max(0) as u64; - crate::app::storage_compat::record_bucket_object_delete_memory( + crate::app::usecase_storage_compat::record_bucket_object_delete_memory( &bucket, size, existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(), @@ -3822,7 +3826,7 @@ impl DefaultObjectUsecase { } // Fast in-memory update for immediate quota and admin usage consistency - crate::app::storage_compat::record_bucket_object_delete_memory( + crate::app::usecase_storage_compat::record_bucket_object_delete_memory( &bucket, obj_info.size.max(0) as u64, opts.version_id.is_none(), @@ -4757,7 +4761,7 @@ impl DefaultObjectUsecase { insert_str( &mut metadata, SUFFIX_COMPRESSION, - crate::app::storage_compat::compression_metadata_value(algorithm), + crate::app::usecase_storage_compat::compression_metadata_value(algorithm), ); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); diff --git a/rustfs/src/app/usecase_storage_compat.rs b/rustfs/src/app/usecase_storage_compat.rs new file mode 100644 index 000000000..21ce22e76 --- /dev/null +++ b/rustfs/src/app/usecase_storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::app::storage_compat::*; diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 3ae40d20c..43636def8 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -14,11 +14,11 @@ use crate::server::RPC_PREFIX; use crate::storage::request_context::spawn_traced; -use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE; -use crate::storage::storage_compat::StorageDiskRpcExt as _; -use crate::storage::storage_compat::WalkDirOptions; -use crate::storage::storage_compat::find_local_disk_by_ref; -use crate::storage::storage_compat::verify_rpc_signature; +use crate::storage::rpc::storage_compat::DEFAULT_READ_BUFFER_SIZE; +use crate::storage::rpc::storage_compat::StorageDiskRpcExt as _; +use crate::storage::rpc::storage_compat::WalkDirOptions; +use crate::storage::rpc::storage_compat::find_local_disk_by_ref; +use crate::storage::rpc::storage_compat::verify_rpc_signature; use bytes::{Bytes, BytesMut}; use futures_util::TryStreamExt; use http::{HeaderMap, Method, Request, Response, StatusCode, Uri}; diff --git a/rustfs/src/storage/rpc/mod.rs b/rustfs/src/storage/rpc/mod.rs index 86c5f8f90..cb808e89e 100644 --- a/rustfs/src/storage/rpc/mod.rs +++ b/rustfs/src/storage/rpc/mod.rs @@ -14,6 +14,7 @@ pub mod http_service; pub mod node_service; +pub(crate) mod storage_compat; pub use http_service::InternodeRpcService; pub use node_service::{NodeService, make_server}; diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 7fe5b9095..42b499c25 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -16,7 +16,7 @@ use crate::admin::service::{ config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot}, site_replication::reload_site_replication_runtime_state, }; -use crate::storage::storage_compat::{ +use crate::storage::rpc::storage_compat::{ CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, UpdateMetadataOpts, all_local_disk_path, @@ -122,7 +122,7 @@ fn unimplemented_rpc(method: &str) -> Status { Status::unimplemented(format!("{method} is not implemented")) } -fn background_rebalance_start_error_message(result: crate::storage::storage_compat::Result<()>) -> Option { +fn background_rebalance_start_error_message(result: crate::storage::rpc::storage_compat::Result<()>) -> Option { result.err().map(|err| format!("start_rebalance failed: {err}")) } @@ -2366,7 +2366,7 @@ mod tests { #[test] fn test_background_rebalance_start_error_message_formats_error() { - let message = background_rebalance_start_error_message(Err(crate::storage::storage_compat::Error::other("boom"))) + let message = background_rebalance_start_error_message(Err(crate::storage::rpc::storage_compat::Error::other("boom"))) .expect("background rebalance start failure should be formatted"); assert!(message.contains("start_rebalance failed")); @@ -2697,7 +2697,7 @@ mod tests { vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string()); vars.insert( PEER_RESTSUB_SYS.to_string(), - crate::storage::storage_compat::STORAGE_CLASS_SUB_SYS.to_string(), + crate::storage::rpc::storage_compat::STORAGE_CLASS_SUB_SYS.to_string(), ); let request = Request::new(SignalServiceRequest { diff --git a/rustfs/src/storage/rpc/storage_compat.rs b/rustfs/src/storage/rpc/storage_compat.rs new file mode 100644 index 000000000..f3d75945f --- /dev/null +++ b/rustfs/src/storage/rpc/storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::storage::storage_compat::*; diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index 509028636..69e3cfedc 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::storage::s3_api::common::rustfs_owner; -use crate::storage::storage_compat::to_s3s_etag; +use crate::storage::s3_api::storage_compat::to_s3s_etag; use percent_encoding::percent_decode_str; use rustfs_storage_api::{ BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, diff --git a/rustfs/src/storage/s3_api/mod.rs b/rustfs/src/storage/s3_api/mod.rs index 7b7b11966..e63bacf12 100644 --- a/rustfs/src/storage/s3_api/mod.rs +++ b/rustfs/src/storage/s3_api/mod.rs @@ -23,4 +23,5 @@ pub(crate) mod acl; pub(crate) mod bucket; pub(crate) mod common; pub(crate) mod multipart; +pub(crate) mod storage_compat; pub(crate) mod tagging; diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index ab4523dfe..dae7401ac 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; -use crate::storage::storage_compat::to_s3s_etag; +use crate::storage::s3_api::storage_compat::to_s3s_etag; use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo}; use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp}; use s3s::{S3Error, S3ErrorCode}; @@ -191,7 +191,7 @@ mod tests { parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, }; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; - use crate::storage::storage_compat::to_s3s_etag; + use crate::storage::s3_api::storage_compat::to_s3s_etag; use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo}; use s3s::S3ErrorCode; use s3s::dto::Timestamp; diff --git a/rustfs/src/storage/s3_api/storage_compat.rs b/rustfs/src/storage/s3_api/storage_compat.rs new file mode 100644 index 000000000..30f4a506b --- /dev/null +++ b/rustfs/src/storage/s3_api/storage_compat.rs @@ -0,0 +1,17 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { + rustfs_ecstore::api::client::object_api_utils::to_s3s_etag(etag) +} diff --git a/rustfs/src/storage/storage_compat.rs b/rustfs/src/storage/storage_compat.rs index 0c7bdcabd..d92b6f347 100644 --- a/rustfs/src/storage/storage_compat.rs +++ b/rustfs/src/storage/storage_compat.rs @@ -17,9 +17,8 @@ use std::sync::Arc; #[cfg(test)] use rustfs_ecstore::api::config as ecstore_config; use rustfs_ecstore::api::{ - admin as ecstore_admin, bucket as ecstore_bucket, client as ecstore_client, disk as ecstore_disk, error as ecstore_error, - global as ecstore_global, metrics as ecstore_metrics, rio as ecstore_rio, rpc as ecstore_rpc, set_disk as ecstore_set_disk, - storage as ecstore_storage, + admin as ecstore_admin, bucket as ecstore_bucket, disk as ecstore_disk, error as ecstore_error, global as ecstore_global, + metrics as ecstore_metrics, rio as ecstore_rio, rpc as ecstore_rpc, set_disk as ecstore_set_disk, storage as ecstore_storage, }; pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG; @@ -485,10 +484,6 @@ pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult< ecstore_bucket::utils::serialize(val) } -pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { - ecstore_client::object_api_utils::to_s3s_etag(etag) -} - pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { ecstore_error::is_err_bucket_not_found(err) } diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index ba6499892..5cf237865 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -82,6 +82,7 @@ RUSTFS_ROOT_STORAGE_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/rustfs_root_storage_co RUSTFS_ROOT_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_root_bucket_storage_compat_module_hits.txt" RUSTFS_ROOT_RUNTIME_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_root_runtime_storage_compat_module_hits.txt" RUSTFS_ROOT_CONSUMER_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_root_consumer_compat_hits.txt" +RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_owner_compat_consumer_hits.txt" RUSTFS_ADMIN_CONFIG_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_admin_config_storage_compat_module_hits.txt" RUSTFS_STORAGE_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_storage_bucket_storage_compat_module_hits.txt" RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_compat_reexport_hits.txt" @@ -827,6 +828,25 @@ if [[ -s "$RUSTFS_ROOT_CONSUMER_COMPAT_HITS_FILE" ]]; then report_failure "RustFS root storage compatibility must not own capacity/server/startup consumer wrappers: $(paste -sd '; ' "$RUSTFS_ROOT_CONSUMER_COMPAT_HITS_FILE")" fi +( + cd "$ROOT_DIR" + { + rg -n --no-heading 'crate::admin::storage_compat' \ + rustfs/src/admin/handlers rustfs/src/admin/service rustfs/src/admin/router.rs \ + --glob '!**/*storage_compat.rs' || true + rg -n --no-heading 'crate::app::storage_compat' \ + rustfs/src/app \ + --glob '!**/*storage_compat.rs' || true + rg -n --no-heading 'crate::storage::storage_compat' \ + rustfs/src/storage/rpc rustfs/src/storage/s3_api \ + --glob '!**/*storage_compat.rs' || true + } +) >"$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS owner compatibility consumers must route through their local compatibility boundary: $(paste -sd '; ' "$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::config::\{[^}]*\bcom\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::\{[^}]*\binit\s*(?:,|})[^}]*\}\s*;' \