From e57962d5e8239efa7aaf0de147544983910af3d7 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Mon, 22 Jun 2026 18:55:59 +0800 Subject: [PATCH] refactor: remove RustFS owner compat bridges (#3743) --- docs/architecture/migration-progress.md | 50 +- rustfs/src/admin/handlers/account_info.rs | 2 +- .../admin/handlers/audit_runtime_config.rs | 2 +- rustfs/src/admin/handlers/bucket_meta.rs | 4 +- rustfs/src/admin/handlers/config_admin.rs | 24 +- rustfs/src/admin/handlers/heal.rs | 10 +- rustfs/src/admin/handlers/kms_dynamic.rs | 2 +- rustfs/src/admin/handlers/metrics.rs | 2 +- .../src/admin/handlers/object_zip_download.rs | 4 +- rustfs/src/admin/handlers/oidc.rs | 2 +- rustfs/src/admin/handlers/pools.rs | 2 +- rustfs/src/admin/handlers/quota.rs | 8 +- rustfs/src/admin/handlers/rebalance.rs | 8 +- rustfs/src/admin/handlers/replication.rs | 18 +- rustfs/src/admin/handlers/site_replication.rs | 33 +- rustfs/src/admin/handlers/sts.rs | 2 +- rustfs/src/admin/handlers/table_catalog.rs | 2 +- rustfs/src/admin/handlers/tier.rs | 6 +- rustfs/src/admin/handlers/trace.rs | 2 +- rustfs/src/admin/mod.rs | 443 ++++++++++++- rustfs/src/admin/router.rs | 52 +- rustfs/src/admin/service/config.rs | 18 +- rustfs/src/admin/service/site_replication.rs | 4 +- rustfs/src/admin/storage_compat.rs | 433 ------------ rustfs/src/app/admin_usecase.rs | 12 +- rustfs/src/app/bucket_usecase.rs | 16 +- rustfs/src/app/capacity_dirty_scope_test.rs | 4 +- rustfs/src/app/context/compat.rs | 18 +- rustfs/src/app/context/global.rs | 2 +- rustfs/src/app/context/handles.rs | 8 +- rustfs/src/app/context/interfaces.rs | 6 +- rustfs/src/app/context/startup.rs | 2 +- .../src/app/lifecycle_transition_api_test.rs | 24 +- rustfs/src/app/mod.rs | 626 +++++++++++++++++- rustfs/src/app/multipart_usecase.rs | 55 +- rustfs/src/app/object_usecase.rs | 95 ++- rustfs/src/app/storage_compat.rs | 624 ----------------- rustfs/src/storage/access.rs | 8 +- rustfs/src/storage/ecfs.rs | 4 +- rustfs/src/storage/ecfs_extend.rs | 6 +- rustfs/src/storage/ecfs_test.rs | 10 +- rustfs/src/storage/head_prefix.rs | 2 +- rustfs/src/storage/mod.rs | 575 +++++++++++++++- rustfs/src/storage/options.rs | 6 +- rustfs/src/storage/rpc/http_service.rs | 10 +- rustfs/src/storage/rpc/node_service.rs | 15 +- rustfs/src/storage/sse.rs | 18 +- rustfs/src/storage/storage_compat.rs | 565 ---------------- scripts/check_architecture_migration_rules.sh | 38 +- 49 files changed, 1932 insertions(+), 1950 deletions(-) delete mode 100644 rustfs/src/admin/storage_compat.rs delete mode 100644 rustfs/src/app/storage_compat.rs delete mode 100644 rustfs/src/storage/storage_compat.rs diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 68a6d71ee..f6d7b1add 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,17 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-external-owner-compat-cleanup` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126`. -- Based on: API-126 slice. +- Branch: `overtrue/arch-rustfs-owner-compat-cleanup` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127`. +- Based on: API-127 slice. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: remove external owner storage compatibility bridges from - IAM root, heal, and scanner, then route their consumers directly through - owner API definitions. -- CI/script changes: allow migrated owner roots to import ECStore facade APIs - directly and reject reintroduced IAM/heal/scanner bridge modules. -- Docs changes: record the API-127 external owner bridge cleanup. +- Rust code changes: remove RustFS runtime owner storage compatibility bridges + from app, admin, and storage, then route consumers directly through owner API + definitions. +- CI/script changes: allow migrated RustFS owner roots to import ECStore facade + APIs directly and reject reintroduced app/admin/storage bridge modules. +- Docs changes: record the API-128 RustFS owner bridge cleanup. ## Phase 0 Tasks @@ -791,6 +791,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block - Verification: focused IAM/heal/scanner compile coverage, external owner bridge residual scan, migration and layer guards, formatting, diff hygiene, Rust risk scan, and three-expert review. +- [x] `API-128` Remove RustFS owner compatibility bridges. + - Completed slice: move app, admin, and storage bridge contracts into their + owner modules, delete their local `storage_compat.rs` bridge modules, and + update consumers to import owner APIs directly. + - Acceptance: RustFS app, admin, and storage no longer route through local + storage compatibility bridges; migration rules reject deleted files, module + declarations, or bridge consumers. + - Must preserve: app object/multipart/bucket behavior, admin route and + config contracts, storage access/SSE/RPC behavior, object DTO aliases, + bucket metadata helpers, and ECStore facade ownership. + - Verification: focused RustFS compile coverage, RustFS owner bridge + residual scan, migration and layer guards, formatting, diff hygiene, Rust + risk scan, and three-expert review. - [x] `G-012` Inventory placement and repair invariants. - Acceptance: [`placement-repair-invariants.md`](placement-repair-invariants.md) records @@ -3824,14 +3837,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | pass | API-127 removes only external owner bridge modules and keeps call sites on explicit owner APIs. | -| Migration preservation | pass | The migration guard now rejects deleted IAM-root, heal, and scanner bridge files, module declarations, and bridge consumers. | -| Testing/verification | pass | Focused IAM/heal/scanner compile, residual scan, migration guard, layer guard, formatting, diff hygiene, and diff-only Rust risk scan passed. | +| Quality/architecture | pass | API-128 removes only RustFS owner bridge modules and keeps call sites on explicit owner APIs. | +| Migration preservation | pass | The migration guard now rejects deleted app, admin, and storage bridge files, module declarations, and bridge consumers. | +| Testing/verification | pass | Focused RustFS compile, residual scan, migration guard, layer guard, formatting, diff hygiene, and diff-only Rust risk scan passed. | ## Verification Notes Passed before push: +- Issue #660 API-128 current slice: + - `cargo check --tests -p rustfs`: passed. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - RustFS owner compatibility bridge residual scan: passed. + - Rust risk scan: diff-only scan found path-rewritten existing test unwraps, + test expects, and part-number casts only; no new risky behavior added. + - `make pre-commit`: passed, including 6509 nextest tests passed and 111 + skipped. + - Issue #660 API-127 current slice: - `cargo check --tests -p rustfs-iam -p rustfs-heal -p rustfs-scanner`: passed. - `cargo fmt --all`: passed. diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index 24c941045..d1284e2e6 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::versioning_sys::BucketVersioningSys; +use super::super::versioning_sys::BucketVersioningSys; use crate::admin::auth::authenticate_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::app::context::resolve_object_store_handle; diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index 6fffc2ffa..f7994d8d4 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::{read_admin_config_without_migrate, save_admin_server_config}; +use super::super::{read_admin_config_without_migrate, save_admin_server_config}; use crate::admin::handlers::target_descriptor::AdminTargetSpec; use crate::app::context::resolve_object_store_handle; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 66304be6a..88b206073 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::utils::{deserialize, serialize}; -use super::super::storage_compat::{ +use super::super::ecstore_utils::{deserialize, serialize}; +use super::super::{ StorageError, metadata::{ BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index 6fdabd969..b944bded8 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; -use super::super::storage_compat::{ +use super::super::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; +use super::super::{ RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, save_admin_config, save_admin_server_config, }; @@ -710,7 +710,7 @@ fn success_response(config_applied: bool) -> S3Result S3Result> { +fn object_store() -> S3Result> { resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized")) } @@ -755,7 +755,7 @@ fn config_update_sub_system(directives: &[ConfigDirective]) -> S3Result S3Result<()> { if DEFAULT_KVS.get().is_none() { - super::super::storage_compat::init_admin_config_defaults(); + super::super::init_admin_config_defaults(); } let Some(defaults) = DEFAULT_KVS.get() else { return Err(s3_error!(InternalError, "config defaults are not initialized")); @@ -1409,7 +1409,7 @@ fn env_help_key(sub_system: &str, key: &str) -> String { fn default_help_postfix(sub_system: &str, key: &str) -> String { if DEFAULT_KVS.get().is_none() { - super::super::storage_compat::init_admin_config_defaults(); + super::super::init_admin_config_defaults(); } DEFAULT_KVS @@ -1896,7 +1896,7 @@ mod tests { #[test] fn full_config_export_can_be_reapplied() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let mut original = ServerConfig::new(); apply_set_directives( &mut original, @@ -1943,7 +1943,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn build_help_response_appends_default_value_postfix() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response"); assert_eq!(response.keys_help.len(), 2); @@ -2056,7 +2056,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_includes_env_override_lines() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); temp_env::with_vars( [ ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")), @@ -2092,7 +2092,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_lists_env_only_targets() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || { let config = ServerConfig::new(); let rendered = String::from_utf8( @@ -2115,7 +2115,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_supports_specific_env_only_target_queries() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || { let config = ServerConfig::new(); let rendered = String::from_utf8( @@ -2138,7 +2138,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#, #[test] fn render_selected_config_orders_default_before_named_targets() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || { let mut config = ServerConfig::new(); apply_set_directives( @@ -2319,7 +2319,7 @@ identity_openid client_id="existing-client""#, #[test] fn storage_class_get_target_none_matches_full_export() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let mut config = ServerConfig::new(); apply_set_directives( &mut config, diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 500940579..c8e50c094 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::is_reserved_or_invalid_bucket; -use super::super::storage_compat::utils::is_valid_object_prefix; +use super::super::ecstore_utils::is_valid_object_prefix; +use super::super::is_reserved_or_invalid_bucket; use crate::admin::auth::{authenticate_request, validate_admin_request}; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::app::context::resolve_object_store_handle; @@ -378,10 +378,10 @@ fn should_handle_root_heal_directly(hip: &HealInitParams) -> bool { && hip.hs.set.is_none() } -fn map_root_heal_status(heal_err: Option) -> S3Result<()> { +fn map_root_heal_status(heal_err: Option) -> S3Result<()> { match heal_err { None => Ok(()), - Some(super::super::storage_compat::StorageError::NoHealRequired) => { + Some(super::super::StorageError::NoHealRequired) => { info!( event = EVENT_ADMIN_RESPONSE_EMITTED, component = LOG_COMPONENT_ADMIN_API, @@ -730,7 +730,7 @@ impl Operation for BackgroundHealStatusHandler { #[cfg(test)] mod tests { - use super::super::super::storage_compat::StorageError; + use super::super::super::StorageError; use super::extract_heal_init_params; use super::{ HealInitParams, HealResp, build_heal_channel_request, encode_background_heal_status, encode_heal_start_success, diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index 07ce35af3..0eee8f537 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -14,7 +14,7 @@ //! KMS dynamic configuration admin API handlers -use super::super::storage_compat::{read_admin_config, save_admin_config}; +use super::super::{read_admin_config, save_admin_config}; use crate::admin::auth::validate_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle}; diff --git a/rustfs/src/admin/handlers/metrics.rs b/rustfs/src/admin/handlers/metrics.rs index ac57702a4..9db087df0 100644 --- a/rustfs/src/admin/handlers/metrics.rs +++ b/rustfs/src/admin/handlers/metrics.rs @@ -18,7 +18,7 @@ //! keeping the response format explicitly NDJSON. It is not a Prometheus text //! exposition endpoint. -use super::super::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics}; +use super::super::{CollectMetricsOpts, MetricType, collect_local_metrics}; use crate::admin::auth::validate_admin_request; use crate::admin::router::Operation; use crate::auth::{check_key_valid, get_session_token}; diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index a1b9d528c..90476d90b 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::get_global_region; +use super::super::get_global_region; use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router}; use crate::app::context::resolve_object_store_handle; use crate::auth::{check_key_valid, get_session_token}; @@ -649,7 +649,7 @@ async fn preflight_zip_items(request: &CreateObjectZipDownloadRequest, items: &[ Ok(()) } -fn storage_error_to_s3(err: super::super::storage_compat::Error) -> s3s::S3Error { +fn storage_error_to_s3(err: super::super::Error) -> s3s::S3Error { ApiError::from(err).into() } diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index 50e582858..5619d3fb4 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::{read_admin_config_without_migrate, save_admin_server_config}; +use super::super::{read_admin_config_without_migrate, save_admin_server_config}; use super::sts::create_oidc_sts_credentials; use crate::admin::auth::validate_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index b12f65174..f2c8b2544 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -194,7 +194,7 @@ macro_rules! log_pool_response_emitted { }; } -fn endpoints_from_context() -> Option { +fn endpoints_from_context() -> Option { resolve_endpoints_handle() } diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index 376126a85..c1c9ae9fd 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -14,9 +14,9 @@ //! Quota admin handlers for HTTP API -use super::super::storage_compat::metadata_sys::BucketMetadataSys; -use super::super::storage_compat::quota::checker::QuotaChecker; -use super::super::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation}; +use super::super::metadata_sys::BucketMetadataSys; +use super::super::quota::checker::QuotaChecker; +use super::super::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket}; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle}; @@ -175,7 +175,7 @@ async fn current_usage_from_context(bucket: &str) -> u64 { return 0; }; - match super::super::storage_compat::load_data_usage_from_backend(store).await { + match super::super::load_data_usage_from_backend(store).await { Ok(data_usage_info) => data_usage_info .buckets_usage .get(bucket) diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 50ae7633d..4a80c8d39 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::{ +use super::super::{ DiskStat, ECStore, NotificationSys, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStopPropagationRecord, StorageError, decode_rebalance_stop_propagation_record, get_global_notification_sys, }; @@ -296,7 +296,7 @@ fn build_rebalance_pool_progress( now: OffsetDateTime, stop_time: Option, percent_free_goal: f64, - ps: &super::super::storage_compat::RebalanceStats, + ps: &super::super::RebalanceStats, ) -> Option { let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64; let terminal_time = ps.info.end_time.or(stop_time); @@ -339,7 +339,7 @@ fn build_rebalance_pool_statuses( now: OffsetDateTime, stop_time: Option, percent_free_goal: f64, - pool_stats: &[super::super::storage_compat::RebalanceStats], + pool_stats: &[super::super::RebalanceStats], disk_stats: &[DiskStat], ) -> Vec { pool_stats @@ -895,7 +895,7 @@ mod offsetdatetime_rfc3339 { #[cfg(test)] mod rebalance_handler_tests { - use super::super::super::storage_compat::{ + use super::super::super::{ DiskStat, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record, }; diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 89c5cbfdf..36a922d82 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::StorageError; -use super::super::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys}; -use super::super::storage_compat::global_rustfs_port; -use super::super::storage_compat::metadata::BUCKET_TARGETS_FILE; -use super::super::storage_compat::metadata_sys; -use super::super::storage_compat::metadata_sys::get_replication_config; -use super::super::storage_compat::replication::BucketStats; -use super::super::storage_compat::replication::GLOBAL_REPLICATION_STATS; -use super::super::storage_compat::target::BucketTarget; +use super::super::StorageError; +use super::super::bucket_target_sys::{BucketTargetError, BucketTargetSys}; +use super::super::global_rustfs_port; +use super::super::metadata::BUCKET_TARGETS_FILE; +use super::super::metadata_sys; +use super::super::metadata_sys::get_replication_config; +use super::super::replication::BucketStats; +use super::super::replication::GLOBAL_REPLICATION_STATS; +use super::super::target::BucketTarget; use crate::admin::auth::validate_admin_request; use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint; use crate::admin::router::{AdminOperation, Operation, S3Router}; diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 8188afd7f..a8fcd6efb 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -12,20 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::Error as StorageError; -use super::super::storage_compat::bucket_target_sys::BucketTargetSys; -use super::super::storage_compat::metadata::{ +use super::super::Error as StorageError; +use super::super::bucket_target_sys::BucketTargetSys; +use super::super::ecstore_utils::{deserialize, serialize}; +use super::super::metadata::{ BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, }; -use super::super::storage_compat::metadata_sys; -use super::super::storage_compat::replication::GLOBAL_REPLICATION_STATS; -use super::super::storage_compat::replication::{ResyncOpts, get_global_replication_pool}; -use super::super::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; -use super::super::storage_compat::utils::{deserialize, serialize}; -use super::super::storage_compat::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; -use super::super::storage_compat::{delete_admin_config, read_admin_config, save_admin_config}; -use super::super::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port}; +use super::super::metadata_sys; +use super::super::replication::GLOBAL_REPLICATION_STATS; +use super::super::replication::{ResyncOpts, get_global_replication_pool}; +use super::super::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; +use super::super::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; +use super::super::{delete_admin_config, read_admin_config, save_admin_config}; +use super::super::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port}; use crate::admin::auth::validate_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::site_replication_identity::{ @@ -652,7 +652,7 @@ async fn site_replication_peer_client() -> S3Result { built } -fn runtime_tls_enabled_with(endpoints: Option<&super::super::storage_compat::EndpointServerPools>) -> bool { +fn runtime_tls_enabled_with(endpoints: Option<&super::super::EndpointServerPools>) -> bool { if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() { return true; } @@ -3352,10 +3352,7 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option incoming_updated_at.is_some_and(|incoming_updated_at| incoming_updated_at < local_updated_at) } -fn bucket_meta_local_updated_at( - bucket_meta: &super::super::storage_compat::metadata::BucketMetadata, - config_file: &str, -) -> OffsetDateTime { +fn bucket_meta_local_updated_at(bucket_meta: &super::super::metadata::BucketMetadata, config_file: &str) -> OffsetDateTime { match config_file { BUCKET_POLICY_CONFIG => bucket_meta.policy_config_updated_at, BUCKET_TAGGING_CONFIG => bucket_meta.tagging_config_updated_at, @@ -4573,8 +4570,8 @@ impl Operation for SRRotateServiceAccountHandler { #[cfg(test)] mod tests { - use super::super::super::storage_compat::Endpoint; - use super::super::super::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints}; + use super::super::super::Endpoint; + use super::super::super::{EndpointServerPools, Endpoints, PoolEndpoints}; use super::*; use http::{HeaderMap, HeaderValue, Uri}; use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation}; diff --git a/rustfs/src/admin/handlers/sts.rs b/rustfs/src/admin/handlers/sts.rs index d9d1cebbc..322504025 100644 --- a/rustfs/src/admin/handlers/sts.rs +++ b/rustfs/src/admin/handlers/sts.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::utils::serialize; +use super::super::ecstore_utils::serialize; use super::is_admin::IsAdminHandler; use crate::{ admin::{ diff --git a/rustfs/src/admin/handlers/table_catalog.rs b/rustfs/src/admin/handlers/table_catalog.rs index ed90e9059..37b0741e9 100644 --- a/rustfs/src/admin/handlers/table_catalog.rs +++ b/rustfs/src/admin/handlers/table_catalog.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys}; +use super::super::{ECStore, metadata::table_catalog_path_hash, metadata_sys}; use crate::admin::{ auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object}, router::{AdminOperation, Operation, S3Router}, diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 0de46ebb8..a0ce70206 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -13,8 +13,8 @@ // limitations under the License. #![allow(unused_variables, unused_mut, unused_must_use)] -use super::super::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState; -use super::super::storage_compat::{ +use super::super::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState; +use super::super::{ AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND, TierConfig, TierCreds, TierType, get_global_notification_sys, storageclass, @@ -930,7 +930,7 @@ impl Operation for ClearTier { #[cfg(test)] mod tests { - use super::super::super::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats; + use super::super::super::lifecycle::tier_last_day_stats::LastDayTierStats; use super::*; use http::Uri; use matchit::Router; diff --git a/rustfs/src/admin/handlers/trace.rs b/rustfs/src/admin/handlers/trace.rs index 22895a270..fa6484911 100644 --- a/rustfs/src/admin/handlers/trace.rs +++ b/rustfs/src/admin/handlers/trace.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::PeerRestClient; +use super::super::PeerRestClient; use crate::admin::router::Operation; use crate::app::context::resolve_endpoints_handle; use http::StatusCode; diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index c678b2155..983965f24 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -23,7 +23,6 @@ pub(crate) mod route_policy; pub mod router; pub mod service; pub mod site_replication_identity; -pub(crate) mod storage_compat; pub mod utils; #[cfg(test)] @@ -33,8 +32,8 @@ mod route_registration_test; use handlers::{ audit, bucket_meta, config_admin, extensions, heal, health, kms, module_switch, object_zip_download, oidc, plugins_catalog, - plugins_instances, pools, profile_admin, quota, rebalance, replication, scanner, site_replication, sts, system, - table_catalog, tier, tls_debug, user, + plugins_instances, pools, profile_admin, quota as quota_handler, rebalance, replication as replication_handler, scanner, + site_replication, sts, system, table_catalog, tier, tls_debug, user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -65,7 +64,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() tier::register_tier_route(r)?; - quota::register_quota_route(r)?; + quota_handler::register_quota_route(r)?; bucket_meta::register_bucket_meta_route(r)?; config_admin::register_config_route(r)?; scanner::register_scanner_route(r)?; @@ -76,7 +75,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() plugins_catalog::register_plugin_catalog_route(r)?; plugins_instances::register_plugin_instance_route(r)?; - replication::register_replication_route(r)?; + replication_handler::register_replication_route(r)?; site_replication::register_site_replication_route(r)?; profile_admin::register_profiling_route(r)?; tls_debug::register_tls_debug_route(r)?; @@ -86,3 +85,437 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() Ok(()) } + +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::Deref; +use std::sync::Arc; + +use rustfs_ecstore::api::bucket as ecstore_bucket; +use rustfs_ecstore::api::capacity as ecstore_capacity; +use rustfs_ecstore::api::client as ecstore_client; +use rustfs_ecstore::api::config as ecstore_config; +use rustfs_ecstore::api::data_usage as ecstore_data_usage; +use rustfs_ecstore::api::disk as ecstore_disk; +use rustfs_ecstore::api::error as ecstore_error; +use rustfs_ecstore::api::global as ecstore_global; +use rustfs_ecstore::api::layout as ecstore_layout; +use rustfs_ecstore::api::metrics as ecstore_metrics; +use rustfs_ecstore::api::notification as ecstore_notification; +use rustfs_ecstore::api::rebalance as ecstore_rebalance; +use rustfs_ecstore::api::rpc as ecstore_rpc; +use rustfs_ecstore::api::storage as ecstore_storage; +use rustfs_ecstore::api::tier as ecstore_tier; + +pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET; +pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; + +pub(crate) type AdminError = ecstore_client::admin_handler_utils::AdminError; +pub(crate) type CollectMetricsOpts = ecstore_metrics::CollectMetricsOpts; +pub(crate) type DiskStat = ecstore_rebalance::DiskStat; +pub(crate) type ECStore = ecstore_storage::ECStore; +pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; +pub(crate) type MetricType = ecstore_metrics::MetricType; +pub(crate) type NotificationSys = ecstore_notification::NotificationSys; +pub(crate) type PeerRestClient = ecstore_rpc::PeerRestClient; +pub(crate) type RebalSaveOpt = ecstore_rebalance::RebalSaveOpt; +pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWarnings; +pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta; +pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats; +pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord; +pub(crate) type StorageError = ecstore_error::StorageError; +pub(crate) type Error = StorageError; +pub(crate) type Result = core::result::Result; +pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig; +pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds; +pub(crate) type TierType = ecstore_tier::tier_config::TierType; + +#[cfg(test)] +pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; +#[cfg(test)] +pub(crate) type Endpoints = ecstore_layout::Endpoints; +#[cfg(test)] +pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; +#[cfg(test)] +pub(crate) type RebalStatus = ecstore_rebalance::RebalStatus; +#[cfg(test)] +pub(crate) type RebalanceCleanupWarningEntry = ecstore_rebalance::RebalanceCleanupWarningEntry; +#[cfg(test)] +pub(crate) type RebalanceInfo = ecstore_rebalance::RebalanceInfo; + +pub(crate) fn decode_rebalance_stop_propagation_record(message: &str) -> Option { + ecstore_rebalance::decode_rebalance_stop_propagation_record(message) +} + +#[cfg(test)] +pub(crate) fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String { + ecstore_rebalance::encode_rebalance_stop_propagation_record(record) +} + +pub(crate) trait AdminReplicationConfigExt { + fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec; + fn has_existing_object_replication(&self, arn: &str) -> (bool, bool); +} + +impl AdminReplicationConfigExt for s3s::dto::ReplicationConfiguration { + fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec { + ::filter_target_arns( + self, obj, + ) + } + + fn has_existing_object_replication(&self, arn: &str) -> (bool, bool) { + ::has_existing_object_replication( + self, arn, + ) + } +} + +pub(crate) trait AdminVersioningConfigExt { + fn enabled(&self) -> bool; +} + +impl AdminVersioningConfigExt for s3s::dto::VersioningConfiguration { + fn enabled(&self) -> bool { + ::enabled(self) + } +} + +pub(crate) mod bandwidth { + pub(crate) mod monitor { + pub(crate) type BandwidthDetails = super::super::ecstore_bucket::bandwidth::monitor::BandwidthDetails; + pub(crate) type Monitor = super::super::ecstore_bucket::bandwidth::monitor::Monitor; + } +} + +pub(crate) mod bucket_target_sys { + pub(crate) type AdvancedPutOptions = super::ecstore_bucket::bucket_target_sys::AdvancedPutOptions; + pub(crate) type BucketTargetError = super::ecstore_bucket::bucket_target_sys::BucketTargetError; + pub(crate) type BucketTargetSys = super::ecstore_bucket::bucket_target_sys::BucketTargetSys; + pub(crate) type PutObjectOptions = super::ecstore_bucket::bucket_target_sys::PutObjectOptions; + pub(crate) type RemoveObjectOptions = super::ecstore_bucket::bucket_target_sys::RemoveObjectOptions; + pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError; + pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient; +} + +pub(crate) mod lifecycle { + pub(crate) mod bucket_lifecycle_ops { + use super::super::DailyAllTierStats; + + pub(crate) struct GlobalTransitionStateCompat; + + #[allow(non_upper_case_globals)] + pub(crate) static GLOBAL_TransitionState: GlobalTransitionStateCompat = GlobalTransitionStateCompat; + + impl GlobalTransitionStateCompat { + pub(crate) fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState.get_daily_all_tier_stats() + } + } + } + + pub(crate) mod tier_last_day_stats { + #[cfg(test)] + pub(crate) type LastDayTierStats = super::super::ecstore_bucket::lifecycle::tier_last_day_stats::LastDayTierStats; + } +} + +pub(crate) mod metadata { + pub(crate) const BUCKET_CORS_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_CORS_CONFIG; + pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; + pub(crate) const BUCKET_NOTIFICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_NOTIFICATION_CONFIG; + pub(crate) const BUCKET_POLICY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG; + pub(crate) const BUCKET_QUOTA_CONFIG_FILE: &str = super::ecstore_bucket::metadata::BUCKET_QUOTA_CONFIG_FILE; + pub(crate) const BUCKET_REPLICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG; + pub(crate) const BUCKET_SSECONFIG: &str = super::ecstore_bucket::metadata::BUCKET_SSECONFIG; + pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; + pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; + pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; + pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; + + pub(crate) type BucketMetadata = super::ecstore_bucket::metadata::BucketMetadata; + + pub(crate) fn table_catalog_path_hash(value: &str) -> String { + super::ecstore_bucket::metadata::table_catalog_path_hash(value) + } +} + +pub(crate) mod metadata_sys { + use std::sync::Arc; + + use rustfs_policy::policy::BucketPolicy; + use s3s::dto::{ + BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ServerSideEncryptionConfiguration, + Tagging, VersioningConfiguration, + }; + use time::OffsetDateTime; + + use super::Result; + use super::metadata::BucketMetadata; + use super::quota::BucketQuota; + use super::target::BucketTargets; + + pub(crate) type BucketMetadataSys = super::ecstore_bucket::metadata_sys::BucketMetadataSys; + + pub(crate) async fn get(bucket: &str) -> Result> { + super::ecstore_bucket::metadata_sys::get(bucket).await + } + + pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { + super::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await + } + + pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { + super::ecstore_bucket::metadata_sys::delete(bucket, config_file).await + } + + pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await + } + + pub(crate) async fn get_bucket_targets_config(bucket: &str) -> Result { + super::ecstore_bucket::metadata_sys::get_bucket_targets_config(bucket).await + } + + pub(crate) async fn get_config_from_disk(bucket: &str) -> Result { + super::ecstore_bucket::metadata_sys::get_config_from_disk(bucket).await + } + + pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await + } + + pub(crate) async fn get_notification_config(bucket: &str) -> Result> { + super::ecstore_bucket::metadata_sys::get_notification_config(bucket).await + } + + pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await + } + + pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await + } + + pub(crate) async fn get_replication_config(bucket: &str) -> Result<(s3s::dto::ReplicationConfiguration, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_replication_config(bucket).await + } + + pub(crate) async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_sse_config(bucket).await + } + + pub(crate) async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await + } + + pub(crate) async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> { + super::ecstore_bucket::metadata_sys::get_versioning_config(bucket).await + } + + pub(crate) async fn list_bucket_targets(bucket: &str) -> Result { + super::ecstore_bucket::metadata_sys::list_bucket_targets(bucket).await + } +} + +pub(crate) mod quota { + pub(crate) mod checker { + pub(crate) type QuotaChecker = super::super::ecstore_bucket::quota::checker::QuotaChecker; + } + + pub(crate) type BucketQuota = super::ecstore_bucket::quota::BucketQuota; + pub(crate) type QuotaError = super::ecstore_bucket::quota::QuotaError; + pub(crate) type QuotaOperation = super::ecstore_bucket::quota::QuotaOperation; +} + +pub(crate) mod replication { + use std::sync::Arc; + + pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus; + pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats; + pub(crate) type DynReplicationPool = super::ecstore_bucket::replication::DynReplicationPool; + pub(crate) type ObjectOpts = super::ecstore_bucket::replication::ObjectOpts; + pub(crate) type ReplicationStats = super::ecstore_bucket::replication::ReplicationStats; + pub(crate) type ResyncOpts = super::ecstore_bucket::replication::ResyncOpts; + #[cfg(test)] + pub(crate) type ResyncStatusType = super::ecstore_bucket::replication::ResyncStatusType; + #[cfg(test)] + pub(crate) type TargetReplicationResyncStatus = super::ecstore_bucket::replication::TargetReplicationResyncStatus; + + pub(crate) struct GlobalReplicationStatsCompat; + + pub(crate) static GLOBAL_REPLICATION_STATS: GlobalReplicationStatsCompat = GlobalReplicationStatsCompat; + + impl GlobalReplicationStatsCompat { + pub(crate) fn get(&self) -> Option<&'static Arc> { + super::ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() + } + } + + pub(crate) fn get_global_replication_pool() -> Option> { + super::ecstore_bucket::replication::get_global_replication_pool() + } +} + +pub(crate) mod target { + #[allow(clippy::upper_case_acronyms)] + pub(crate) type ARN = super::ecstore_bucket::target::ARN; + pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget; + pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType; + pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets; + pub(crate) type Credentials = super::ecstore_bucket::target::Credentials; +} + +pub(crate) mod ecstore_utils { + pub(crate) fn deserialize(input: &[u8]) -> s3s::xml::DeResult + where + T: for<'xml> s3s::xml::Deserialize<'xml>, + { + super::ecstore_bucket::utils::deserialize(input) + } + + pub(crate) fn is_valid_object_prefix(object: &str) -> bool { + super::ecstore_bucket::utils::is_valid_object_prefix(object) + } + + pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { + super::ecstore_bucket::utils::serialize(val) + } +} + +pub(crate) mod versioning {} + +pub(crate) mod versioning_sys { + pub(crate) type BucketVersioningSys = super::ecstore_bucket::versioning_sys::BucketVersioningSys; +} + +pub(crate) mod storageclass { + pub(crate) const INLINE_BLOCK_ENV: &str = super::ecstore_config::storageclass::INLINE_BLOCK_ENV; + pub(crate) const OPTIMIZE_ENV: &str = super::ecstore_config::storageclass::OPTIMIZE_ENV; + pub(crate) const RRS: &str = super::ecstore_config::storageclass::RRS; + pub(crate) const RRS_ENV: &str = super::ecstore_config::storageclass::RRS_ENV; + pub(crate) const STANDARD: &str = super::ecstore_config::storageclass::STANDARD; + pub(crate) const STANDARD_ENV: &str = super::ecstore_config::storageclass::STANDARD_ENV; + + pub(crate) type Config = super::ecstore_config::storageclass::Config; + + pub(crate) fn lookup_config(kvs: &rustfs_config::server_config::KVS, set_drive_count: usize) -> super::Result { + super::ecstore_config::storageclass::lookup_config(kvs, set_drive_count) + } +} + +pub(crate) type DailyAllTierStats = ecstore_bucket::lifecycle::tier_last_day_stats::DailyAllTierStats; + +pub(crate) fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool { + ecstore_capacity::is_reserved_or_invalid_bucket(bucket_entry, strict) +} + +pub(crate) async fn read_admin_config(api: Arc, file: &str) -> Result> { + ecstore_config::com::read_config(api, file).await +} + +pub(crate) async fn read_admin_config_without_migrate(api: Arc) -> Result { + ecstore_config::com::read_config_without_migrate(api).await +} + +pub(crate) async fn save_admin_config(api: Arc, file: &str, data: Vec) -> Result<()> { + ecstore_config::com::save_config(api, file, data).await +} + +pub(crate) async fn delete_admin_config(api: Arc, file: &str) -> Result<()> { + ecstore_config::com::delete_config(api, file).await +} + +pub(crate) async fn save_admin_server_config(api: Arc, cfg: &rustfs_config::server_config::Config) -> Result<()> { + ecstore_config::com::save_server_config(api, cfg).await +} + +pub(crate) fn init_admin_config_defaults() { + ecstore_config::init(); +} + +pub(crate) fn set_global_storage_class(cfg: storageclass::Config) { + ecstore_config::set_global_storage_class(cfg); +} + +pub(crate) async fn load_data_usage_from_backend( + store: Arc, +) -> std::result::Result { + ecstore_data_usage::load_data_usage_from_backend(store).await +} + +pub(crate) fn get_global_bucket_monitor() -> Option> { + ecstore_global::get_global_bucket_monitor() +} + +pub(crate) fn get_global_deployment_id() -> Option { + ecstore_global::get_global_deployment_id() +} + +pub(crate) fn get_global_endpoints_opt() -> Option { + ecstore_global::get_global_endpoints_opt() +} + +pub(crate) fn get_global_region() -> Option { + ecstore_global::get_global_region() +} + +pub(crate) fn global_rustfs_port() -> u16 { + ecstore_global::global_rustfs_port() +} + +pub(crate) async fn collect_local_metrics( + types: MetricType, + opts: &CollectMetricsOpts, +) -> rustfs_madmin::metrics::RealtimeMetrics { + ecstore_metrics::collect_local_metrics(types, opts).await +} + +pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { + ecstore_notification::get_global_notification_sys() +} + +pub(crate) struct BootTimeCompat; + +pub(crate) static GLOBAL_BOOT_TIME: BootTimeCompat = BootTimeCompat; + +impl BootTimeCompat { + pub(crate) fn get(&self) -> Option<&'static std::time::SystemTime> { + ecstore_global::GLOBAL_BOOT_TIME.get() + } +} + +pub(crate) struct AdminErrorRef(fn() -> &'static AdminError); + +impl Deref for AdminErrorRef { + type Target = AdminError; + + fn deref(&self) -> &Self::Target { + (self.0)() + } +} + +pub(crate) static ERR_TIER_BACKEND_IN_USE: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier::ERR_TIER_BACKEND_IN_USE); +pub(crate) static ERR_TIER_BACKEND_NOT_EMPTY: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier::ERR_TIER_BACKEND_NOT_EMPTY); +pub(crate) static ERR_TIER_MISSING_CREDENTIALS: AdminErrorRef = + AdminErrorRef(|| &ecstore_tier::tier::ERR_TIER_MISSING_CREDENTIALS); +pub(crate) static ERR_TIER_ALREADY_EXISTS: AdminErrorRef = + AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_ALREADY_EXISTS); +pub(crate) static ERR_TIER_CONNECT_ERR: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_CONNECT_ERR); +pub(crate) static ERR_TIER_INVALID_CREDENTIALS: AdminErrorRef = + AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_INVALID_CREDENTIALS); +pub(crate) static ERR_TIER_NAME_NOT_UPPERCASE: AdminErrorRef = + AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_NAME_NOT_UPPERCASE); +pub(crate) static ERR_TIER_NOT_FOUND: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_NOT_FOUND); diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 3ed1a4f80..02852deaf 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -12,23 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::GLOBAL_BOOT_TIME; -use super::storage_compat::PeerRestClient; -use super::storage_compat::bandwidth::monitor::BandwidthDetails; -use super::storage_compat::bucket_target_sys::{ - BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient, -}; -use super::storage_compat::get_global_notification_sys; -use super::storage_compat::metadata::BUCKET_TARGETS_FILE; -use super::storage_compat::metadata_sys; -use super::storage_compat::read_admin_config_without_migrate; -use super::storage_compat::replication::{ +use super::GLOBAL_BOOT_TIME; +use super::PeerRestClient; +use super::bandwidth::monitor::BandwidthDetails; +use super::bucket_target_sys::{BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient}; +use super::get_global_notification_sys; +use super::metadata::BUCKET_TARGETS_FILE; +use super::metadata_sys; +use super::read_admin_config_without_migrate; +use super::replication::{ BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ResyncOpts, get_global_replication_pool, }; -use super::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets}; -use super::storage_compat::versioning_sys::BucketVersioningSys; -use super::storage_compat::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; -use super::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region}; +use super::target::{BucketTarget, BucketTargetType, BucketTargets}; +use super::versioning_sys::BucketVersioningSys; +use super::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; +use super::{get_global_bucket_monitor, get_global_deployment_id, get_global_region}; use crate::admin::console::{is_console_path, make_console_server}; use crate::admin::handlers::oidc::is_oidc_path; use crate::app::context::resolve_object_store_handle; @@ -1419,7 +1417,7 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> { async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> { match metadata_sys::get_replication_config(bucket).await { Ok(_) => Ok(()), - Err(super::storage_compat::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)), + Err(super::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)), Err(err) => Err(ApiError::from(err).into()), } } @@ -1944,7 +1942,7 @@ async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget) fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions { PutObjectOptions { - internal: super::storage_compat::bucket_target_sys::AdvancedPutOptions { + internal: super::bucket_target_sys::AdvancedPutOptions { source_version_id: Uuid::new_v4().to_string(), replication_status: ReplicationStatusType::Replica, source_mtime: now, @@ -2059,7 +2057,7 @@ async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result { .object_lock_enabled .as_ref() .is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)), - Err(super::storage_compat::StorageError::ConfigNotFound) => Ok(false), + Err(super::StorageError::ConfigNotFound) => Ok(false), Err(err) => Err(ApiError::from(err).into()), } } @@ -2773,7 +2771,7 @@ mod tests { #[test] fn apply_replication_reset_to_targets_updates_matching_target() { let mut targets = BucketTargets { - targets: vec![super::super::storage_compat::target::BucketTarget { + targets: vec![super::super::target::BucketTarget { arn: "arn:target".to_string(), ..Default::default() }], @@ -2795,10 +2793,10 @@ mod tests { let mut status = BucketReplicationResyncStatus::new(); status.targets_map.insert( "arn:z".to_string(), - super::super::storage_compat::replication::TargetReplicationResyncStatus { + super::super::replication::TargetReplicationResyncStatus { resync_id: "rid-z".to_string(), last_update: Some(datetime!(2025-01-03 00:00 UTC)), - resync_status: super::super::storage_compat::replication::ResyncStatusType::ResyncFailed, + resync_status: super::super::replication::ResyncStatusType::ResyncFailed, failed_count: 2, failed_size: 4, bucket: "bucket-z".to_string(), @@ -2808,10 +2806,10 @@ mod tests { ); status.targets_map.insert( "arn:a".to_string(), - super::super::storage_compat::replication::TargetReplicationResyncStatus { + super::super::replication::TargetReplicationResyncStatus { resync_id: "rid-a".to_string(), last_update: Some(datetime!(2025-01-02 00:00 UTC)), - resync_status: super::super::storage_compat::replication::ResyncStatusType::ResyncCompleted, + resync_status: super::super::replication::ResyncStatusType::ResyncCompleted, replicated_count: 3, replicated_size: 9, bucket: "bucket-a".to_string(), @@ -2841,10 +2839,10 @@ mod tests { let mut status = BucketReplicationResyncStatus::new(); status.targets_map.insert( "arn:z".to_string(), - super::super::storage_compat::replication::TargetReplicationResyncStatus { + super::super::replication::TargetReplicationResyncStatus { resync_id: "rid-z".to_string(), last_update: Some(datetime!(2025-02-03 00:00 UTC)), - resync_status: super::super::storage_compat::replication::ResyncStatusType::ResyncFailed, + resync_status: super::super::replication::ResyncStatusType::ResyncFailed, failed_count: 2, failed_size: 4, bucket: "bucket-z".to_string(), @@ -2854,10 +2852,10 @@ mod tests { ); status.targets_map.insert( "arn:a".to_string(), - super::super::storage_compat::replication::TargetReplicationResyncStatus { + super::super::replication::TargetReplicationResyncStatus { resync_id: "rid-a".to_string(), last_update: Some(datetime!(2025-02-02 00:00 UTC)), - resync_status: super::super::storage_compat::replication::ResyncStatusType::ResyncCompleted, + resync_status: super::super::replication::ResyncStatusType::ResyncCompleted, replicated_count: 3, replicated_size: 9, bucket: "bucket-a".to_string(), diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index 8486cdb91..5c847972d 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::get_global_notification_sys; -use super::super::storage_compat::set_global_storage_class; -use super::super::storage_compat::storageclass; -use super::super::storage_compat::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate}; +use super::super::get_global_notification_sys; +use super::super::set_global_storage_class; +use super::super::storageclass; +use super::super::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate}; use crate::app::context::resolve_object_store_handle; use rustfs_audit::reload_audit_config; use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS}; @@ -370,7 +370,7 @@ pub async fn signal_config_snapshot_reload() { #[cfg(test)] mod tests { - use super::super::super::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; + use super::super::super::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; use super::*; use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES}; @@ -427,7 +427,7 @@ mod tests { #[test] fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); @@ -441,7 +441,7 @@ mod tests { #[test] fn validate_audit_subsystem_config_rejects_relative_queue_dir() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); @@ -456,7 +456,7 @@ mod tests { #[test] fn validate_identity_openid_config_rejects_missing_openid_scope() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); @@ -473,7 +473,7 @@ mod tests { #[test] fn validate_identity_openid_config_rejects_invalid_named_provider_id() { - super::super::super::storage_compat::init_admin_config_defaults(); + super::super::super::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults"); let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target"); diff --git a/rustfs/src/admin/service/site_replication.rs b/rustfs/src/admin/service/site_replication.rs index b77c12eda..b9145d629 100644 --- a/rustfs/src/admin/service/site_replication.rs +++ b/rustfs/src/admin/service/site_replication.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::Error as StorageError; -use super::super::storage_compat::{read_admin_config, save_admin_config}; +use super::super::Error as StorageError; +use super::super::{read_admin_config, save_admin_config}; use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with}; use crate::app::context::resolve_object_store_handle; use rustfs_madmin::PeerInfo; diff --git a/rustfs/src/admin/storage_compat.rs b/rustfs/src/admin/storage_compat.rs deleted file mode 100644 index 251b0c975..000000000 --- a/rustfs/src/admin/storage_compat.rs +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::ops::Deref; -use std::sync::Arc; - -use rustfs_ecstore::api::bucket as ecstore_bucket; -use rustfs_ecstore::api::capacity as ecstore_capacity; -use rustfs_ecstore::api::client as ecstore_client; -use rustfs_ecstore::api::config as ecstore_config; -use rustfs_ecstore::api::data_usage as ecstore_data_usage; -use rustfs_ecstore::api::disk as ecstore_disk; -use rustfs_ecstore::api::error as ecstore_error; -use rustfs_ecstore::api::global as ecstore_global; -use rustfs_ecstore::api::layout as ecstore_layout; -use rustfs_ecstore::api::metrics as ecstore_metrics; -use rustfs_ecstore::api::notification as ecstore_notification; -use rustfs_ecstore::api::rebalance as ecstore_rebalance; -use rustfs_ecstore::api::rpc as ecstore_rpc; -use rustfs_ecstore::api::storage as ecstore_storage; -use rustfs_ecstore::api::tier as ecstore_tier; - -pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET; -pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; - -pub(crate) type AdminError = ecstore_client::admin_handler_utils::AdminError; -pub(crate) type CollectMetricsOpts = ecstore_metrics::CollectMetricsOpts; -pub(crate) type DiskStat = ecstore_rebalance::DiskStat; -pub(crate) type ECStore = ecstore_storage::ECStore; -pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; -pub(crate) type MetricType = ecstore_metrics::MetricType; -pub(crate) type NotificationSys = ecstore_notification::NotificationSys; -pub(crate) type PeerRestClient = ecstore_rpc::PeerRestClient; -pub(crate) type RebalSaveOpt = ecstore_rebalance::RebalSaveOpt; -pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWarnings; -pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta; -pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats; -pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord; -pub(crate) type StorageError = ecstore_error::StorageError; -pub(crate) type Error = StorageError; -pub(crate) type Result = core::result::Result; -pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig; -pub(crate) type TierCreds = ecstore_tier::tier_admin::TierCreds; -pub(crate) type TierType = ecstore_tier::tier_config::TierType; - -#[cfg(test)] -pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; -#[cfg(test)] -pub(crate) type Endpoints = ecstore_layout::Endpoints; -#[cfg(test)] -pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; -#[cfg(test)] -pub(crate) type RebalStatus = ecstore_rebalance::RebalStatus; -#[cfg(test)] -pub(crate) type RebalanceCleanupWarningEntry = ecstore_rebalance::RebalanceCleanupWarningEntry; -#[cfg(test)] -pub(crate) type RebalanceInfo = ecstore_rebalance::RebalanceInfo; - -pub(crate) fn decode_rebalance_stop_propagation_record(message: &str) -> Option { - ecstore_rebalance::decode_rebalance_stop_propagation_record(message) -} - -#[cfg(test)] -pub(crate) fn encode_rebalance_stop_propagation_record(record: &RebalanceStopPropagationRecord) -> String { - ecstore_rebalance::encode_rebalance_stop_propagation_record(record) -} - -pub(crate) trait AdminReplicationConfigExt { - fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec; - fn has_existing_object_replication(&self, arn: &str) -> (bool, bool); -} - -impl AdminReplicationConfigExt for s3s::dto::ReplicationConfiguration { - fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec { - ::filter_target_arns( - self, obj, - ) - } - - fn has_existing_object_replication(&self, arn: &str) -> (bool, bool) { - ::has_existing_object_replication( - self, arn, - ) - } -} - -pub(crate) trait AdminVersioningConfigExt { - fn enabled(&self) -> bool; -} - -impl AdminVersioningConfigExt for s3s::dto::VersioningConfiguration { - fn enabled(&self) -> bool { - ::enabled(self) - } -} - -pub(crate) mod bandwidth { - pub(crate) mod monitor { - pub(crate) type BandwidthDetails = super::super::ecstore_bucket::bandwidth::monitor::BandwidthDetails; - pub(crate) type Monitor = super::super::ecstore_bucket::bandwidth::monitor::Monitor; - } -} - -pub(crate) mod bucket_target_sys { - pub(crate) type AdvancedPutOptions = super::ecstore_bucket::bucket_target_sys::AdvancedPutOptions; - pub(crate) type BucketTargetError = super::ecstore_bucket::bucket_target_sys::BucketTargetError; - pub(crate) type BucketTargetSys = super::ecstore_bucket::bucket_target_sys::BucketTargetSys; - pub(crate) type PutObjectOptions = super::ecstore_bucket::bucket_target_sys::PutObjectOptions; - pub(crate) type RemoveObjectOptions = super::ecstore_bucket::bucket_target_sys::RemoveObjectOptions; - pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError; - pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient; -} - -pub(crate) mod lifecycle { - pub(crate) mod bucket_lifecycle_ops { - use super::super::DailyAllTierStats; - - pub(crate) struct GlobalTransitionStateCompat; - - #[allow(non_upper_case_globals)] - pub(crate) static GLOBAL_TransitionState: GlobalTransitionStateCompat = GlobalTransitionStateCompat; - - impl GlobalTransitionStateCompat { - pub(crate) fn get_daily_all_tier_stats(&self) -> DailyAllTierStats { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState.get_daily_all_tier_stats() - } - } - } - - pub(crate) mod tier_last_day_stats { - #[cfg(test)] - pub(crate) type LastDayTierStats = super::super::ecstore_bucket::lifecycle::tier_last_day_stats::LastDayTierStats; - } -} - -pub(crate) mod metadata { - pub(crate) const BUCKET_CORS_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_CORS_CONFIG; - pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; - pub(crate) const BUCKET_NOTIFICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_NOTIFICATION_CONFIG; - pub(crate) const BUCKET_POLICY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG; - pub(crate) const BUCKET_QUOTA_CONFIG_FILE: &str = super::ecstore_bucket::metadata::BUCKET_QUOTA_CONFIG_FILE; - pub(crate) const BUCKET_REPLICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG; - pub(crate) const BUCKET_SSECONFIG: &str = super::ecstore_bucket::metadata::BUCKET_SSECONFIG; - pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; - pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; - pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; - pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; - - pub(crate) type BucketMetadata = super::ecstore_bucket::metadata::BucketMetadata; - - pub(crate) fn table_catalog_path_hash(value: &str) -> String { - super::ecstore_bucket::metadata::table_catalog_path_hash(value) - } -} - -pub(crate) mod metadata_sys { - use std::sync::Arc; - - use rustfs_policy::policy::BucketPolicy; - use s3s::dto::{ - BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ServerSideEncryptionConfiguration, - Tagging, VersioningConfiguration, - }; - use time::OffsetDateTime; - - use super::Result; - use super::metadata::BucketMetadata; - use super::quota::BucketQuota; - use super::target::BucketTargets; - - pub(crate) type BucketMetadataSys = super::ecstore_bucket::metadata_sys::BucketMetadataSys; - - pub(crate) async fn get(bucket: &str) -> Result> { - super::ecstore_bucket::metadata_sys::get(bucket).await - } - - pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { - super::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await - } - - pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { - super::ecstore_bucket::metadata_sys::delete(bucket, config_file).await - } - - pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await - } - - pub(crate) async fn get_bucket_targets_config(bucket: &str) -> Result { - super::ecstore_bucket::metadata_sys::get_bucket_targets_config(bucket).await - } - - pub(crate) async fn get_config_from_disk(bucket: &str) -> Result { - super::ecstore_bucket::metadata_sys::get_config_from_disk(bucket).await - } - - pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await - } - - pub(crate) async fn get_notification_config(bucket: &str) -> Result> { - super::ecstore_bucket::metadata_sys::get_notification_config(bucket).await - } - - pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await - } - - pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await - } - - pub(crate) async fn get_replication_config(bucket: &str) -> Result<(s3s::dto::ReplicationConfiguration, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_replication_config(bucket).await - } - - pub(crate) async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_sse_config(bucket).await - } - - pub(crate) async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await - } - - pub(crate) async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> { - super::ecstore_bucket::metadata_sys::get_versioning_config(bucket).await - } - - pub(crate) async fn list_bucket_targets(bucket: &str) -> Result { - super::ecstore_bucket::metadata_sys::list_bucket_targets(bucket).await - } -} - -pub(crate) mod quota { - pub(crate) mod checker { - pub(crate) type QuotaChecker = super::super::ecstore_bucket::quota::checker::QuotaChecker; - } - - pub(crate) type BucketQuota = super::ecstore_bucket::quota::BucketQuota; - pub(crate) type QuotaError = super::ecstore_bucket::quota::QuotaError; - pub(crate) type QuotaOperation = super::ecstore_bucket::quota::QuotaOperation; -} - -pub(crate) mod replication { - use std::sync::Arc; - - pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus; - pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats; - pub(crate) type DynReplicationPool = super::ecstore_bucket::replication::DynReplicationPool; - pub(crate) type ObjectOpts = super::ecstore_bucket::replication::ObjectOpts; - pub(crate) type ReplicationStats = super::ecstore_bucket::replication::ReplicationStats; - pub(crate) type ResyncOpts = super::ecstore_bucket::replication::ResyncOpts; - #[cfg(test)] - pub(crate) type ResyncStatusType = super::ecstore_bucket::replication::ResyncStatusType; - #[cfg(test)] - pub(crate) type TargetReplicationResyncStatus = super::ecstore_bucket::replication::TargetReplicationResyncStatus; - - pub(crate) struct GlobalReplicationStatsCompat; - - pub(crate) static GLOBAL_REPLICATION_STATS: GlobalReplicationStatsCompat = GlobalReplicationStatsCompat; - - impl GlobalReplicationStatsCompat { - pub(crate) fn get(&self) -> Option<&'static Arc> { - super::ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() - } - } - - pub(crate) fn get_global_replication_pool() -> Option> { - super::ecstore_bucket::replication::get_global_replication_pool() - } -} - -pub(crate) mod target { - #[allow(clippy::upper_case_acronyms)] - pub(crate) type ARN = super::ecstore_bucket::target::ARN; - pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget; - pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType; - pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets; - pub(crate) type Credentials = super::ecstore_bucket::target::Credentials; -} - -pub(crate) mod utils { - pub(crate) fn deserialize(input: &[u8]) -> s3s::xml::DeResult - where - T: for<'xml> s3s::xml::Deserialize<'xml>, - { - super::ecstore_bucket::utils::deserialize(input) - } - - pub(crate) fn is_valid_object_prefix(object: &str) -> bool { - super::ecstore_bucket::utils::is_valid_object_prefix(object) - } - - pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { - super::ecstore_bucket::utils::serialize(val) - } -} - -pub(crate) mod versioning {} - -pub(crate) mod versioning_sys { - pub(crate) type BucketVersioningSys = super::ecstore_bucket::versioning_sys::BucketVersioningSys; -} - -pub(crate) mod storageclass { - pub(crate) const INLINE_BLOCK_ENV: &str = super::ecstore_config::storageclass::INLINE_BLOCK_ENV; - pub(crate) const OPTIMIZE_ENV: &str = super::ecstore_config::storageclass::OPTIMIZE_ENV; - pub(crate) const RRS: &str = super::ecstore_config::storageclass::RRS; - pub(crate) const RRS_ENV: &str = super::ecstore_config::storageclass::RRS_ENV; - pub(crate) const STANDARD: &str = super::ecstore_config::storageclass::STANDARD; - pub(crate) const STANDARD_ENV: &str = super::ecstore_config::storageclass::STANDARD_ENV; - - pub(crate) type Config = super::ecstore_config::storageclass::Config; - - pub(crate) fn lookup_config(kvs: &rustfs_config::server_config::KVS, set_drive_count: usize) -> super::Result { - super::ecstore_config::storageclass::lookup_config(kvs, set_drive_count) - } -} - -pub(crate) type DailyAllTierStats = ecstore_bucket::lifecycle::tier_last_day_stats::DailyAllTierStats; - -pub(crate) fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool { - ecstore_capacity::is_reserved_or_invalid_bucket(bucket_entry, strict) -} - -pub(crate) async fn read_admin_config(api: Arc, file: &str) -> Result> { - ecstore_config::com::read_config(api, file).await -} - -pub(crate) async fn read_admin_config_without_migrate(api: Arc) -> Result { - ecstore_config::com::read_config_without_migrate(api).await -} - -pub(crate) async fn save_admin_config(api: Arc, file: &str, data: Vec) -> Result<()> { - ecstore_config::com::save_config(api, file, data).await -} - -pub(crate) async fn delete_admin_config(api: Arc, file: &str) -> Result<()> { - ecstore_config::com::delete_config(api, file).await -} - -pub(crate) async fn save_admin_server_config(api: Arc, cfg: &rustfs_config::server_config::Config) -> Result<()> { - ecstore_config::com::save_server_config(api, cfg).await -} - -pub(crate) fn init_admin_config_defaults() { - ecstore_config::init(); -} - -pub(crate) fn set_global_storage_class(cfg: storageclass::Config) { - ecstore_config::set_global_storage_class(cfg); -} - -pub(crate) async fn load_data_usage_from_backend( - store: Arc, -) -> std::result::Result { - ecstore_data_usage::load_data_usage_from_backend(store).await -} - -pub(crate) fn get_global_bucket_monitor() -> Option> { - ecstore_global::get_global_bucket_monitor() -} - -pub(crate) fn get_global_deployment_id() -> Option { - ecstore_global::get_global_deployment_id() -} - -pub(crate) fn get_global_endpoints_opt() -> Option { - ecstore_global::get_global_endpoints_opt() -} - -pub(crate) fn get_global_region() -> Option { - ecstore_global::get_global_region() -} - -pub(crate) fn global_rustfs_port() -> u16 { - ecstore_global::global_rustfs_port() -} - -pub(crate) async fn collect_local_metrics( - types: MetricType, - opts: &CollectMetricsOpts, -) -> rustfs_madmin::metrics::RealtimeMetrics { - ecstore_metrics::collect_local_metrics(types, opts).await -} - -pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { - ecstore_notification::get_global_notification_sys() -} - -pub(crate) struct BootTimeCompat; - -pub(crate) static GLOBAL_BOOT_TIME: BootTimeCompat = BootTimeCompat; - -impl BootTimeCompat { - pub(crate) fn get(&self) -> Option<&'static std::time::SystemTime> { - ecstore_global::GLOBAL_BOOT_TIME.get() - } -} - -pub(crate) struct AdminErrorRef(fn() -> &'static AdminError); - -impl Deref for AdminErrorRef { - type Target = AdminError; - - fn deref(&self) -> &Self::Target { - (self.0)() - } -} - -pub(crate) static ERR_TIER_BACKEND_IN_USE: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier::ERR_TIER_BACKEND_IN_USE); -pub(crate) static ERR_TIER_BACKEND_NOT_EMPTY: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier::ERR_TIER_BACKEND_NOT_EMPTY); -pub(crate) static ERR_TIER_MISSING_CREDENTIALS: AdminErrorRef = - AdminErrorRef(|| &ecstore_tier::tier::ERR_TIER_MISSING_CREDENTIALS); -pub(crate) static ERR_TIER_ALREADY_EXISTS: AdminErrorRef = - AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_ALREADY_EXISTS); -pub(crate) static ERR_TIER_CONNECT_ERR: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_CONNECT_ERR); -pub(crate) static ERR_TIER_INVALID_CREDENTIALS: AdminErrorRef = - AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_INVALID_CREDENTIALS); -pub(crate) static ERR_TIER_NAME_NOT_UPPERCASE: AdminErrorRef = - AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_NAME_NOT_UPPERCASE); -pub(crate) static ERR_TIER_NOT_FOUND: AdminErrorRef = AdminErrorRef(|| &ecstore_tier::tier_handlers::ERR_TIER_NOT_FOUND); diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index 127a86224..0d85c9264 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -14,11 +14,11 @@ //! Admin application use-case contracts. -use super::storage_compat::ECStore; -use super::storage_compat::EndpointServerPools; -use super::storage_compat::get_server_info; -use super::storage_compat::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free}; -use super::storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; +use super::ECStore; +use super::EndpointServerPools; +use super::get_server_info; +use super::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free}; +use super::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context}; use crate::capacity::resolve_admin_used_capacity; use crate::error::ApiError; @@ -401,7 +401,7 @@ impl DefaultAdminUsecase { #[cfg(test)] mod tests { - use super::super::storage_compat::{PoolDecommissionInfo, PoolStatus}; + use super::super::{PoolDecommissionInfo, PoolStatus}; use super::*; use time::OffsetDateTime; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 24d240002..7861d1b09 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -14,12 +14,12 @@ //! Bucket application use-case contracts. -use super::storage_compat::ECStore; -use super::storage_compat::StorageError; -use super::storage_compat::get_global_notification_sys; -use super::storage_compat::object_api_utils::to_s3s_etag; -use super::storage_compat::{AppObjectLockConfigExt as _, AppVersioningConfigExt as _}; -use super::storage_compat::{ +use super::ECStore; +use super::StorageError; +use super::get_global_notification_sys; +use super::object_api_utils::to_s3s_etag; +use super::{AppObjectLockConfigExt as _, AppVersioningConfigExt as _}; +use super::{ bucket_target_sys::BucketTargetSys, lifecycle::bucket_lifecycle_ops::{ enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_lifecycle_config, @@ -685,7 +685,7 @@ fn assign_lifecycle_rule_ids(rules: &mut [LifecycleRule]) { } } -fn validate_lifecycle_rule_status(rules: &[LifecycleRule]) -> Result<(), &'static str> { +fn validate_lifecycle_rule_status(rules: &[LifecycleRule]) -> std::result::Result<(), &'static str> { for rule in rules { if rule.status != ExpirationStatus::from_static(ExpirationStatus::ENABLED) && rule.status != ExpirationStatus::from_static(ExpirationStatus::DISABLED) @@ -2285,7 +2285,7 @@ mod tests { BucketTargets { targets: arns .iter() - .map(|arn| super::super::storage_compat::target::BucketTarget { + .map(|arn| super::super::target::BucketTarget { arn: (*arn).to_string(), target_type: BucketTargetType::ReplicationService, ..Default::default() diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index eddd14c3f..997417b6b 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys}; +use super::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys}; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; @@ -77,7 +77,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec, Arc) { }; let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); - super::storage_compat::init_local_disks(endpoint_pools.clone()).await.unwrap(); + super::init_local_disks(endpoint_pools.clone()).await.unwrap(); let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) diff --git a/rustfs/src/app/context/compat.rs b/rustfs/src/app/context/compat.rs index 5cfb79f7b..bd22f8b1c 100644 --- a/rustfs/src/app/context/compat.rs +++ b/rustfs/src/app/context/compat.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::ECStore; -use super::super::storage_compat::EndpointServerPools; -use super::super::storage_compat::TierConfigMgr; -use super::super::storage_compat::metadata_sys::BucketMetadataSys; -use super::super::storage_compat::new_object_layer_fn; +use super::super::ECStore; +use super::super::EndpointServerPools; +use super::super::TierConfigMgr; +use super::super::metadata_sys::BucketMetadataSys; +use super::super::new_object_layer_fn; use super::global::{AppContext, get_global_app_context}; use super::handles::{ default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface, @@ -119,10 +119,10 @@ fn resolve_buffer_config_with( #[cfg(test)] mod tests { - use super::super::super::storage_compat::Endpoint; - use super::super::super::storage_compat::init_local_disks; - use super::super::super::storage_compat::new_object_layer_fn; - use super::super::super::storage_compat::{Endpoints, PoolEndpoints}; + use super::super::super::Endpoint; + use super::super::super::init_local_disks; + use super::super::super::new_object_layer_fn; + use super::super::super::{Endpoints, PoolEndpoints}; use super::*; use crate::app::context::global::AppContextTestInterfaces; use crate::app::context::handles::{default_notify_interface, default_region_interface}; diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index 937949512..2ef96bcea 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::{ECStore, set_object_store_resolver}; +use super::super::{ECStore, set_object_store_resolver}; use super::handles::{ IamHandle, KmsHandle, default_bucket_metadata_interface, default_buffer_config_interface, default_endpoints_interface, default_kms_runtime_interface, default_notify_interface, default_region_interface, default_server_config_interface, diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index ac8dcabce..bb0e2ecbb 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::EndpointServerPools; -use super::super::storage_compat::TierConfigMgr; -use super::super::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; -use super::super::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr}; +use super::super::EndpointServerPools; +use super::super::TierConfigMgr; +use super::super::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; +use super::super::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr}; use super::interfaces::{ BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface, diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index 07e3a1746..28066f7ec 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::EndpointServerPools; -use super::super::storage_compat::TierConfigMgr; -use super::super::storage_compat::metadata_sys::BucketMetadataSys; +use super::super::EndpointServerPools; +use super::super::TierConfigMgr; +use super::super::metadata_sys::BucketMetadataSys; use crate::config::RustFSBufferConfig; use async_trait::async_trait; use rustfs_config::server_config::Config; diff --git a/rustfs/src/app/context/startup.rs b/rustfs/src/app/context/startup.rs index da589d6d7..8f6981d30 100644 --- a/rustfs/src/app/context/startup.rs +++ b/rustfs/src/app/context/startup.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::ECStore; +use super::super::ECStore; use super::global::{AppContext, get_global_app_context, init_global_app_context}; use rustfs_kms::KmsServiceManager; use std::io::{Error, Result}; diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index c03b621e7..921487832 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::{ +use super::{ AppWarmBackend, ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackendGetOpts, metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG}, @@ -107,7 +107,7 @@ async fn setup_test_env() -> (Vec, Arc) { let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); - super::storage_compat::init_local_disks(endpoint_pools.clone()).await.unwrap(); + super::init_local_disks(endpoint_pools.clone()).await.unwrap(); let server_addr: std::net::SocketAddr = "127.0.0.1:9003".parse().unwrap(); let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) @@ -124,7 +124,7 @@ async fn setup_test_env() -> (Vec, Arc) { let buckets = buckets_list.into_iter().map(|v| v.name).collect(); metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; - super::storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; + super::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone())); @@ -924,12 +924,9 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() { .expect("Failed to set lifecycle configuration"); let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; - super::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - bucket.as_str(), - ) - .await - .expect("Failed to enqueue transitioned object"); + super::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str()) + .await + .expect("Failed to enqueue transitioned object"); let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT) .await @@ -984,12 +981,9 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() { .expect("Failed to set lifecycle configuration"); let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; - super::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( - ecstore.clone(), - bucket.as_str(), - ) - .await - .expect("Failed to enqueue transitioned object"); + super::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str()) + .await + .expect("Failed to enqueue transitioned object"); let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT) .await diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index 3a066cf39..c4ef21268 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -21,9 +21,633 @@ pub mod context; pub mod multipart_usecase; pub mod object_usecase; mod select_object; -pub(crate) mod storage_compat; #[cfg(test)] mod capacity_dirty_scope_test; #[cfg(test)] mod lifecycle_transition_api_test; + +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use rustfs_ecstore::api::admin as ecstore_admin; +use rustfs_ecstore::api::bucket as ecstore_bucket; +use rustfs_ecstore::api::capacity as ecstore_capacity; +use rustfs_ecstore::api::client as ecstore_client; +use rustfs_ecstore::api::compression as ecstore_compression; +use rustfs_ecstore::api::config as ecstore_config; +use rustfs_ecstore::api::data_usage as ecstore_data_usage; +use rustfs_ecstore::api::disk as ecstore_disk; +use rustfs_ecstore::api::error as ecstore_error; +use rustfs_ecstore::api::global as ecstore_global; +use rustfs_ecstore::api::layout as ecstore_layout; +use rustfs_ecstore::api::notification as ecstore_notification; +use rustfs_ecstore::api::rio as ecstore_rio; +use rustfs_ecstore::api::set_disk as ecstore_set_disk; +use rustfs_ecstore::api::storage as ecstore_storage; +use rustfs_ecstore::api::tier as ecstore_tier; + +pub(crate) const MIN_DISK_COMPRESSIBLE_SIZE: usize = ecstore_compression::MIN_DISK_COMPRESSIBLE_SIZE; + +pub(crate) type DiskError = ecstore_disk::error::DiskError; +pub(crate) type DynReader = ecstore_rio::DynReader; +pub(crate) type ECStore = ecstore_storage::ECStore; +pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; +pub(crate) type HashReader = ecstore_rio::HashReader; +pub(crate) type NotificationSys = ecstore_notification::NotificationSys; +pub(crate) type ObjectStoreResolver = dyn Fn() -> Option> + Send + Sync + 'static; +pub(crate) type ObjectInfo = ::ObjectInfo; +pub(crate) type ObjectOptions = ::ObjectOptions; +pub(crate) type PoolDecommissionInfo = ecstore_capacity::PoolDecommissionInfo; +pub(crate) type PoolStatus = ecstore_capacity::PoolStatus; +pub(crate) type StorageError = ecstore_error::StorageError; +pub(crate) type Error = StorageError; +pub(crate) type TierConfigMgr = ecstore_tier::tier::TierConfigMgr; +pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption; +pub(crate) type WritePlan = ecstore_rio::WritePlan; + +#[cfg(test)] +pub(crate) type DecryptReader = ecstore_rio::DecryptReader; +#[cfg(test)] +pub(crate) type EncryptReader = ecstore_rio::EncryptReader; +#[cfg(test)] +pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; +#[cfg(test)] +pub(crate) type Endpoints = ecstore_layout::Endpoints; +#[cfg(test)] +pub(crate) type HardLimitReader = ecstore_rio::HardLimitReader; +#[cfg(test)] +pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; +#[cfg(test)] +pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig; +#[cfg(test)] +pub(crate) type TierType = ecstore_tier::tier_config::TierType; +#[cfg(test)] +pub(crate) use ecstore_tier::warm_backend::WarmBackend as AppWarmBackend; +#[cfg(test)] +pub(crate) type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts; + +#[cfg(test)] +#[allow(non_snake_case)] +pub(crate) fn EndpointServerPools(pools: Vec) -> EndpointServerPools { + ecstore_layout::EndpointServerPools::from(pools) +} + +pub(crate) trait AppObjectLockConfigExt { + fn enabled(&self) -> bool; +} + +impl AppObjectLockConfigExt for s3s::dto::ObjectLockConfiguration { + fn enabled(&self) -> bool { + ::enabled(self) + } +} + +pub(crate) trait AppReplicationConfigExt { + fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec; + fn replicate(&self, opts: &replication::ObjectOpts) -> bool; +} + +impl AppReplicationConfigExt for s3s::dto::ReplicationConfiguration { + fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec { + ::filter_target_arns( + self, obj, + ) + } + + fn replicate(&self, opts: &replication::ObjectOpts) -> bool { + ::replicate(self, opts) + } +} + +pub(crate) trait AppVersioningConfigExt { + fn prefix_enabled(&self, prefix: &str) -> bool; + fn suspended(&self) -> bool; +} + +impl AppVersioningConfigExt for s3s::dto::VersioningConfiguration { + fn prefix_enabled(&self, prefix: &str) -> bool { + ::prefix_enabled(self, prefix) + } + + fn suspended(&self) -> bool { + ::suspended(self) + } +} + +pub(crate) async fn predict_lifecycle_expiration( + lifecycle: &s3s::dto::BucketLifecycleConfiguration, + obj: &lifecycle::lifecycle::ObjectOpts, +) -> lifecycle::lifecycle::Event { + ecstore_bucket::lifecycle::lifecycle::Lifecycle::predict_expiration(lifecycle, obj).await +} + +pub(crate) fn validate_restore_request(request: &s3s::dto::RestoreRequest, api: Arc) -> std::io::Result<()> { + ::validate(request, api) +} + +pub(crate) async fn get_server_info(get_pools: bool) -> rustfs_madmin::InfoMessage { + ecstore_admin::get_server_info(get_pools).await +} + +pub(crate) mod bucket_target_sys { + pub(crate) type BucketTargetSys = super::ecstore_bucket::bucket_target_sys::BucketTargetSys; +} + +pub(crate) mod lifecycle { + use super::ECStore; + + pub(crate) mod bucket_lifecycle_audit { + pub(crate) type LcEventSrc = super::super::ecstore_bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; + } + + pub(crate) mod bucket_lifecycle_ops { + use std::ops::Deref; + use std::sync::Arc; + + use super::ECStore; + use super::bucket_lifecycle_audit::LcEventSrc; + + pub(crate) type ExpiryState = super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; + + pub(crate) struct GlobalExpiryStateCompat; + + #[allow(non_upper_case_globals)] + pub(crate) static GLOBAL_ExpiryState: GlobalExpiryStateCompat = GlobalExpiryStateCompat; + + impl Deref for GlobalExpiryStateCompat { + type Target = Arc>; + + fn deref(&self) -> &Self::Target { + &super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState + } + } + + #[cfg(test)] + pub(crate) async fn init_background_expiry(api: Arc) { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await; + } + + pub(crate) async fn enqueue_expiry_for_existing_objects( + api: Arc, + bucket: &str, + ) -> Result<(), super::super::Error> { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_expiry_for_existing_objects(api, bucket).await + } + + pub(crate) async fn enqueue_transition_for_existing_objects( + api: Arc, + bucket: &str, + ) -> Result<(), super::super::Error> { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(api, bucket) + .await + } + + pub(crate) async fn enqueue_transition_immediate(oi: &super::super::ObjectInfo, src: LcEventSrc) { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_immediate(oi, src).await; + } + + pub(crate) async fn post_restore_opts( + version_id: &str, + bucket: &str, + object: &str, + ) -> Result { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(version_id, bucket, object).await + } + + pub(crate) async fn validate_transition_tier(lc: &s3s::dto::BucketLifecycleConfiguration) -> Result<(), std::io::Error> { + super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier(lc).await + } + + pub(crate) async fn validate_lifecycle_config( + lc: &s3s::dto::BucketLifecycleConfiguration, + lock_config: &s3s::dto::ObjectLockConfiguration, + ) -> Result<(), std::io::Error> { + use super::super::ecstore_bucket::lifecycle::lifecycle::Lifecycle as _; + + lc.validate(lock_config).await + } + } + + pub(crate) mod lifecycle_contract { + #[cfg(test)] + pub(crate) type IlmAction = super::super::ecstore_bucket::lifecycle::lifecycle::IlmAction; + pub(crate) type Event = super::super::ecstore_bucket::lifecycle::lifecycle::Event; + pub(crate) type ObjectOpts = super::super::ecstore_bucket::lifecycle::lifecycle::ObjectOpts; + pub(crate) type TransitionOptions = super::super::ecstore_bucket::lifecycle::lifecycle::TransitionOptions; + + pub(crate) const TRANSITION_COMPLETE: &str = super::super::ecstore_bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; + + pub(crate) fn expected_expiry_time(mod_time: time::OffsetDateTime, days: i32) -> time::OffsetDateTime { + super::super::ecstore_bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, days) + } + } + pub(crate) use lifecycle_contract as lifecycle; + + pub(crate) mod tier_delete_journal { + use std::sync::Arc; + + use super::ECStore; + + pub(crate) async fn persist_tier_delete_journal_entry( + api: Arc, + je: &super::tier_sweeper::Jentry, + ) -> std::io::Result<()> { + super::super::ecstore_bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(api, je).await + } + } + + pub(crate) mod tier_sweeper { + pub(crate) type Jentry = super::super::ecstore_bucket::lifecycle::tier_sweeper::Jentry; + + pub(crate) fn transitioned_delete_journal_entry( + version_id: Option, + versioned: bool, + suspended: bool, + transitioned: &rustfs_storage_api::TransitionedObject, + ) -> Option { + super::super::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( + version_id, + versioned, + suspended, + transitioned, + ) + } + + pub(crate) fn transitioned_force_delete_journal_entry( + transitioned: &rustfs_storage_api::TransitionedObject, + ) -> Option { + super::super::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(transitioned) + } + } +} + +pub(crate) mod metadata { + pub(crate) const BUCKET_CORS_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_CORS_CONFIG; + pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; + pub(crate) const BUCKET_NOTIFICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_NOTIFICATION_CONFIG; + pub(crate) const BUCKET_POLICY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG; + pub(crate) const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG; + pub(crate) const BUCKET_REPLICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG; + pub(crate) const BUCKET_SSECONFIG: &str = super::ecstore_bucket::metadata::BUCKET_SSECONFIG; + pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; + pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; + pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; + #[cfg(test)] + pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; +} + +pub(crate) mod metadata_sys { + use std::sync::Arc; + + use rustfs_policy::policy::BucketPolicy; + use s3s::dto::{ + BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, + PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, + }; + use time::OffsetDateTime; + use tokio::sync::RwLock; + + #[cfg(test)] + use super::ECStore; + use super::Error; + use super::target::BucketTargets; + + pub(crate) type BucketMetadataSys = super::ecstore_bucket::metadata_sys::BucketMetadataSys; + + #[cfg(test)] + pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { + super::ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; + } + + pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { + super::ecstore_bucket::metadata_sys::delete(bucket, config_file).await + } + + pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await + } + + pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await + } + + pub(crate) async fn get_bucket_targets_config(bucket: &str) -> Result { + super::ecstore_bucket::metadata_sys::get_bucket_targets_config(bucket).await + } + + pub(crate) async fn get_cors_config(bucket: &str) -> Result<(CORSConfiguration, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_cors_config(bucket).await + } + + pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { + super::ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() + } + + pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await + } + + pub(crate) async fn get_notification_config(bucket: &str) -> Result, Error> { + super::ecstore_bucket::metadata_sys::get_notification_config(bucket).await + } + + pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await + } + + pub(crate) async fn get_public_access_block_config( + bucket: &str, + ) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await + } + + pub(crate) async fn get_replication_config( + bucket: &str, + ) -> Result<(s3s::dto::ReplicationConfiguration, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_replication_config(bucket).await + } + + pub(crate) async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_sse_config(bucket).await + } + + pub(crate) async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime), Error> { + super::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await + } + + pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { + super::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await + } +} + +pub(crate) mod object_api_utils { + pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { + super::ecstore_client::object_api_utils::to_s3s_etag(etag) + } +} + +pub(crate) mod object_lock { + pub(crate) mod objectlock { + pub(crate) fn get_object_legalhold_meta( + meta: &std::collections::HashMap, + ) -> s3s::dto::ObjectLockLegalHold { + super::super::ecstore_bucket::object_lock::objectlock::get_object_legalhold_meta(meta) + } + + pub(crate) fn get_object_retention_meta( + meta: &std::collections::HashMap, + ) -> s3s::dto::ObjectLockRetention { + super::super::ecstore_bucket::object_lock::objectlock::get_object_retention_meta(meta) + } + } + + pub(crate) mod objectlock_sys { + pub(crate) type BucketObjectLockSys = super::super::ecstore_bucket::object_lock::objectlock_sys::BucketObjectLockSys; + pub(crate) type ObjectLockBlockReason = super::super::ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; + + pub(crate) async fn check_object_lock_for_deletion( + bucket: &str, + obj_info: &super::super::ObjectInfo, + bypass_governance: bool, + ) -> Option { + super::super::ecstore_bucket::object_lock::objectlock_sys::check_object_lock_for_deletion( + bucket, + obj_info, + bypass_governance, + ) + .await + } + + pub(crate) fn is_retention_active(mode: &str, retain_until_date: Option<&s3s::dto::Date>) -> bool { + super::super::ecstore_bucket::object_lock::objectlock_sys::is_retention_active(mode, retain_until_date) + } + } +} + +pub(crate) mod policy_sys { + pub(crate) type PolicySys = super::ecstore_bucket::policy_sys::PolicySys; +} + +pub(crate) mod quota { + pub(crate) mod checker { + pub(crate) type QuotaChecker = super::super::ecstore_bucket::quota::checker::QuotaChecker; + } + + pub(crate) type QuotaOperation = super::ecstore_bucket::quota::QuotaOperation; +} + +pub(crate) mod replication { + use std::collections::HashMap; + use std::sync::Arc; + + pub(crate) type DeletedObjectReplicationInfo = super::ecstore_bucket::replication::DeletedObjectReplicationInfo; + pub(crate) type MustReplicateOptions = super::ecstore_bucket::replication::MustReplicateOptions; + pub(crate) type ObjectOpts = super::ecstore_bucket::replication::ObjectOpts; + pub(crate) type ReplicateDecision = rustfs_filemeta::ReplicateDecision; + + pub(crate) async fn check_replicate_delete( + bucket: &str, + dobj: &rustfs_storage_api::ObjectToDelete, + oi: &super::ObjectInfo, + del_opts: &super::ObjectOptions, + gerr: Option, + ) -> ReplicateDecision { + super::ecstore_bucket::replication::check_replicate_delete(bucket, dobj, oi, del_opts, gerr).await + } + + pub(crate) fn get_must_replicate_options( + user_defined: &HashMap, + user_tags: String, + status: rustfs_filemeta::ReplicationStatusType, + op_type: rustfs_filemeta::ReplicationType, + opts: super::ObjectOptions, + ) -> MustReplicateOptions { + super::ecstore_bucket::replication::get_must_replicate_options(user_defined, user_tags, status, op_type, opts) + } + + pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision { + super::ecstore_bucket::replication::must_replicate(bucket, object, mopts).await + } + + pub(crate) async fn schedule_replication( + oi: super::ObjectInfo, + store: Arc, + dsc: ReplicateDecision, + op_type: rustfs_filemeta::ReplicationType, + ) { + super::ecstore_bucket::replication::schedule_replication(oi, store, dsc, op_type).await; + } + + pub(crate) async fn schedule_replication_delete(dv: DeletedObjectReplicationInfo) { + super::ecstore_bucket::replication::schedule_replication_delete(dv).await; + } +} + +pub(crate) mod tagging { + pub(crate) fn decode_tags(tags: &str) -> Vec { + super::ecstore_bucket::tagging::decode_tags(tags) + } +} + +pub(crate) mod target { + #[cfg(test)] + pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget; + pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType; + pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets; +} + +pub(crate) mod utils { + pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { + super::ecstore_bucket::utils::serialize(val) + } +} + +pub(crate) mod versioning {} + +pub(crate) mod versioning_sys { + pub(crate) type BucketVersioningSys = super::ecstore_bucket::versioning_sys::BucketVersioningSys; +} + +#[cfg(test)] +pub(crate) mod transition_api { + pub(crate) type ReadCloser = super::ecstore_client::transition_api::ReadCloser; + pub(crate) type ReaderImpl = super::ecstore_client::transition_api::ReaderImpl; +} + +pub(crate) mod storageclass { + pub(crate) const STANDARD: &str = super::ecstore_config::storageclass::STANDARD; + #[cfg(test)] + pub(crate) const STANDARD_IA: &str = super::ecstore_config::storageclass::STANDARD_IA; +} + +pub(crate) fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { + ecstore_capacity::get_total_usable_capacity(disks, info) +} + +pub(crate) fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { + ecstore_capacity::get_total_usable_capacity_free(disks, info) +} + +pub(crate) fn is_disk_compressible(headers: &http::HeaderMap, object_name: &str) -> bool { + ecstore_compression::is_disk_compressible(headers, object_name) +} + +pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { + ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; +} + +pub(crate) async fn load_data_usage_from_backend( + store: Arc, +) -> std::result::Result { + ecstore_data_usage::load_data_usage_from_backend(store).await +} + +pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { + ecstore_data_usage::record_bucket_object_delete_memory(bucket, deleted_size, removed_current_object).await; +} + +pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option, new_size: u64) { + ecstore_data_usage::record_bucket_object_write_memory(bucket, previous_current_size, new_size).await; +} + +pub(crate) fn is_all_buckets_not_found(errs: &[Option]) -> bool { + ecstore_disk::error_reduce::is_all_buckets_not_found(errs) +} + +pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { + ecstore_error::is_err_bucket_not_found(err) +} + +pub(crate) fn is_err_object_not_found(err: &Error) -> bool { + ecstore_error::is_err_object_not_found(err) +} + +pub(crate) fn is_err_version_not_found(err: &Error) -> bool { + ecstore_error::is_err_version_not_found(err) +} + +#[cfg(test)] +pub(crate) struct GlobalTierConfigMgrCompat; + +#[cfg(test)] +#[allow(non_upper_case_globals)] +pub(crate) static GLOBAL_TierConfigMgr: GlobalTierConfigMgrCompat = GlobalTierConfigMgrCompat; + +#[cfg(test)] +impl std::ops::Deref for GlobalTierConfigMgrCompat { + type Target = Arc>; + + fn deref(&self) -> &Self::Target { + &ecstore_global::GLOBAL_TierConfigMgr + } +} + +pub(crate) fn get_global_endpoints_opt() -> Option { + ecstore_global::get_global_endpoints_opt() +} + +pub(crate) fn get_global_region() -> Option { + ecstore_global::get_global_region() +} + +pub(crate) fn get_global_tier_config_mgr() -> Arc> { + ecstore_global::get_global_tier_config_mgr() +} + +pub(crate) fn new_object_layer_fn() -> Option> { + ecstore_global::new_object_layer_fn() +} + +pub(crate) fn set_object_store_resolver(resolver: Arc) -> bool { + ecstore_global::set_object_store_resolver(resolver) +} + +pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { + ecstore_notification::get_global_notification_sys() +} + +#[cfg(test)] +pub(crate) fn boxed_reader(reader: R) -> DynReader +where + R: ecstore_rio::Reader + 'static, +{ + ecstore_rio::boxed_reader(reader) +} + +pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String { + ecstore_rio::compression_metadata_value(algorithm) +} + +pub(crate) fn wrap_reader(reader: R) -> DynReader +where + R: ecstore_rio::ReadStream + 'static, +{ + ecstore_rio::wrap_reader(reader) +} + +pub(crate) fn get_lock_acquire_timeout() -> tokio::time::Duration { + ecstore_set_disk::get_lock_acquire_timeout() +} + +pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool { + ecstore_set_disk::is_valid_storage_class(storage_class) +} + +#[cfg(test)] +pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<(), Error> { + ecstore_storage::init_local_disks(endpoint_pools).await +} diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 2da8906bb..efb822ade 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -14,16 +14,16 @@ //! Multipart application use-case contracts. -use super::storage_compat::ECStore; -use super::storage_compat::is_disk_compressible; -use super::storage_compat::is_valid_storage_class; -use super::storage_compat::object_api_utils::to_s3s_etag; -use super::storage_compat::quota::checker::QuotaChecker; +use super::ECStore; +use super::is_disk_compressible; +use super::is_valid_storage_class; +use super::object_api_utils::to_s3s_etag; +use super::quota::checker::QuotaChecker; #[cfg(test)] -use super::storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; -use super::storage_compat::{HashReader, WritePlan}; -use super::storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found}; -use super::storage_compat::{ +use super::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; +use super::{HashReader, WritePlan}; +use super::{StorageError, is_err_object_not_found, is_err_version_not_found}; +use super::{ lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate}, metadata_sys, quota::QuotaOperation, @@ -482,12 +482,7 @@ impl DefaultMultipartUsecase { )); } // Update quota tracking after successful multipart upload - super::storage_compat::record_bucket_object_write_memory( - &bucket, - previous_current_size, - obj_info.size.max(0) as u64, - ) - .await; + super::record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; } Err(e) => { warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e); @@ -675,7 +670,7 @@ impl DefaultMultipartUsecase { rustfs_utils::http::insert_str( &mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, - super::storage_compat::compression_metadata_value(CompressionAlgorithm::default()), + super::compression_metadata_value(CompressionAlgorithm::default()), ); } @@ -894,10 +889,10 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; let ssec_write = match ssec_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - super::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) + super::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) } crate::storage::sse::EncryptionKeyKind::Direct => { - super::storage_compat::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id) + super::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id) } }; write_plan = write_plan.with_encryption(ssec_write); @@ -914,13 +909,11 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; let managed_write = match managed_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - super::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + super::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + } + crate::storage::sse::EncryptionKeyKind::Direct => { + super::WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id) } - crate::storage::sse::EncryptionKeyKind::Direct => super::storage_compat::WriteEncryption::multipart( - managed_material.key_bytes, - managed_material.base_nonce, - part_id, - ), }; write_plan = write_plan.with_encryption(managed_write); (Some(server_side_encryption), ssekms_key_id) @@ -1235,10 +1228,10 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; let ssec_write = match ssec_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - super::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) + super::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) } crate::storage::sse::EncryptionKeyKind::Direct => { - super::storage_compat::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id) + super::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id) } }; write_plan = write_plan.with_encryption(ssec_write); @@ -1259,13 +1252,11 @@ impl DefaultMultipartUsecase { .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; let managed_write = match managed_material.key_kind { crate::storage::sse::EncryptionKeyKind::Object => { - super::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + super::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + } + crate::storage::sse::EncryptionKeyKind::Direct => { + super::WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id) } - crate::storage::sse::EncryptionKeyKind::Direct => super::storage_compat::WriteEncryption::multipart( - managed_material.key_bytes, - managed_material.base_nonce, - part_id, - ), }; write_plan = write_plan.with_encryption(managed_write); (Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone()) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index f574c1f9f..9a5da2b4c 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -48,21 +48,17 @@ use metrics::{counter, histogram}; use pin_project_lite::pin_project; use rustfs_object_capacity::capacity_manager::get_capacity_manager; // Performance metrics recording (with zero-copy-metrics integration) -use super::storage_compat::ECStore; -use super::storage_compat::object_api_utils::to_s3s_etag; -use super::storage_compat::quota::checker::QuotaChecker; -use super::storage_compat::storageclass; -use super::storage_compat::{ - AppReplicationConfigExt as _, AppVersioningConfigExt as _, predict_lifecycle_expiration, validate_restore_request, -}; -use super::storage_compat::{DiskError, is_all_buckets_not_found}; -use super::storage_compat::{DynReader, HashReader, WritePlan, wrap_reader}; -use super::storage_compat::{ - Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, -}; -use super::storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; -use super::storage_compat::{get_lock_acquire_timeout, is_valid_storage_class}; -use super::storage_compat::{ +use super::ECStore; +use super::object_api_utils::to_s3s_etag; +use super::quota::checker::QuotaChecker; +use super::storageclass; +use super::{AppReplicationConfigExt as _, AppVersioningConfigExt as _, predict_lifecycle_expiration, validate_restore_request}; +use super::{DiskError, is_all_buckets_not_found}; +use super::{DynReader, HashReader, WritePlan, wrap_reader}; +use super::{Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}; +use super::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; +use super::{get_lock_acquire_timeout, is_valid_storage_class}; +use super::{ lifecycle::{ bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{enqueue_transition_immediate, post_restore_opts}, @@ -279,10 +275,10 @@ async fn enqueue_transitioned_delete_cleanup( let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup); let je = if opts.delete_prefix { - super::storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object) + super::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object) } else { let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()); - super::storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry( + super::lifecycle::tier_sweeper::transitioned_delete_journal_entry( version_id, opts.versioned, opts.version_suspended, @@ -293,11 +289,9 @@ async fn enqueue_transitioned_delete_cleanup( return Ok(()); }; - super::storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?; + super::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?; - let mut expiry_state = super::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState - .write() - .await; + let mut expiry_state = super::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState.write().await; if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await { warn!( bucket, @@ -708,7 +702,9 @@ struct PutObjectChecksums { crc64nvme: Option, } -fn normalize_delete_objects_version_id(version_id: Option) -> Result<(Option, Option), String> { +fn normalize_delete_objects_version_id( + version_id: Option, +) -> std::result::Result<(Option, Option), String> { let version_id = version_id.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()); match version_id { Some(id) => { @@ -1601,7 +1597,7 @@ impl DefaultObjectUsecase { #[allow(clippy::too_many_arguments)] async fn prepare_get_object_read( req: &S3Request, - store: &super::storage_compat::ECStore, + store: &super::ECStore, manager: &ConcurrencyManager, bucket: &str, key: &str, @@ -2192,11 +2188,7 @@ impl DefaultObjectUsecase { StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), ); let algorithm = CompressionAlgorithm::default(); - insert_str( - &mut metadata, - SUFFIX_COMPRESSION, - super::storage_compat::compression_metadata_value(algorithm), - ); + insert_str(&mut metadata, SUFFIX_COMPRESSION, super::compression_metadata_value(algorithm)); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let mut hrd = @@ -2207,11 +2199,7 @@ impl DefaultObjectUsecase { } opts.want_checksum = hrd.checksum(); - insert_str( - &mut opts.user_defined, - SUFFIX_COMPRESSION, - super::storage_compat::compression_metadata_value(algorithm), - ); + insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, super::compression_metadata_value(algorithm)); insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); size = HashReader::SIZE_PRESERVE_LAYER; @@ -2402,8 +2390,7 @@ impl DefaultObjectUsecase { maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; // Fast in-memory update for immediate quota and admin usage consistency - super::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64) - .await; + super::record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await; let raw_version = obj_info.version_id.map(|v| v.to_string()); @@ -3263,7 +3250,7 @@ impl DefaultObjectUsecase { insert_str( &mut compress_metadata, SUFFIX_COMPRESSION, - super::storage_compat::compression_metadata_value(CompressionAlgorithm::default()), + super::compression_metadata_value(CompressionAlgorithm::default()), ); insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); } else { @@ -3356,7 +3343,7 @@ impl DefaultObjectUsecase { // Update quota tracking after successful copy if has_bucket_metadata { - super::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; + super::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await; } let raw_dest_version = oi.version_id.map(|v| v.to_string()); @@ -3439,7 +3426,7 @@ impl DefaultObjectUsecase { #[derive(Default, Clone)] struct DeleteResult { delete_object: Option, - error: Option, + error: Option, } let mut delete_results = vec![DeleteResult::default(); delete.objects.len()]; @@ -3453,7 +3440,7 @@ impl DefaultObjectUsecase { let (version_id, version_uuid) = match normalize_delete_objects_version_id(raw_version_id.clone()) { Ok(parsed) => parsed, Err(err) => { - delete_results[idx].error = Some(Error { + delete_results[idx].error = Some(s3s::dto::Error { code: Some("NoSuchVersion".to_string()), key: Some(obj_id.key.clone()), message: Some(err), @@ -3472,7 +3459,7 @@ impl DefaultObjectUsecase { let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::DeleteObjectAction)).await; if let Err(e) = auth_res { - delete_results[idx].error = Some(Error { + delete_results[idx].error = Some(s3s::dto::Error { code: Some("AccessDenied".to_string()), key: Some(obj_id.key.clone()), message: Some(e.to_string()), @@ -3484,7 +3471,7 @@ impl DefaultObjectUsecase { if bypass_governance { let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await; if let Err(e) = auth_res { - delete_results[idx].error = Some(Error { + delete_results[idx].error = Some(s3s::dto::Error { code: Some("AccessDenied".to_string()), key: Some(obj_id.key.clone()), message: Some(e.to_string()), @@ -3495,7 +3482,7 @@ impl DefaultObjectUsecase { } if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await { - delete_results[idx].error = Some(Error { + delete_results[idx].error = Some(s3s::dto::Error { code: Some("InvalidRequest".to_string()), key: Some(obj_id.key.clone()), message: Some(err.to_string()), @@ -3530,7 +3517,7 @@ impl DefaultObjectUsecase { && !delete_creates_delete_marker(&opts) && let Some(block_reason) = check_object_lock_for_deletion(&bucket, &goi, bypass_governance).await { - delete_results[idx].error = Some(Error { + delete_results[idx].error = Some(s3s::dto::Error { code: Some("AccessDenied".to_string()), key: Some(obj_id.key.clone()), message: Some(block_reason.error_message()), @@ -3633,7 +3620,7 @@ impl DefaultObjectUsecase { ); } let size = object_sizes[i].max(0) as u64; - super::storage_compat::record_bucket_object_delete_memory( + super::record_bucket_object_delete_memory( &bucket, size, existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(), @@ -3643,7 +3630,7 @@ impl DefaultObjectUsecase { } if let Some(err) = err.clone() { - delete_results[didx].error = Some(Error { + delete_results[didx].error = Some(s3s::dto::Error { code: Some(err.to_string()), key: Some(object_to_delete[i].object_name.clone()), message: Some(err.to_string()), @@ -3669,7 +3656,10 @@ impl DefaultObjectUsecase { }) .collect(); - let errors = delete_results.iter().filter_map(|v| v.error.clone()).collect::>(); + let errors = delete_results + .iter() + .filter_map(|v| v.error.clone()) + .collect::>(); let output = DeleteObjectsOutput { deleted: Some(deleted), errors: Some(errors), @@ -3871,12 +3861,7 @@ impl DefaultObjectUsecase { } // Fast in-memory update for immediate quota and admin usage consistency - super::storage_compat::record_bucket_object_delete_memory( - &bucket, - obj_info.size.max(0) as u64, - opts.version_id.is_none(), - ) - .await; + super::record_bucket_object_delete_memory(&bucket, obj_info.size.max(0) as u64, opts.version_id.is_none()).await; if obj_info.name.is_empty() { if replicate_force_delete { @@ -4845,11 +4830,7 @@ impl DefaultObjectUsecase { .map_err(ApiError::from)? } else if should_compress { let algorithm = CompressionAlgorithm::default(); - insert_str( - &mut metadata, - SUFFIX_COMPRESSION, - super::storage_compat::compression_metadata_value(algorithm), - ); + insert_str(&mut metadata, SUFFIX_COMPRESSION, super::compression_metadata_value(algorithm)); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?; diff --git a/rustfs/src/app/storage_compat.rs b/rustfs/src/app/storage_compat.rs deleted file mode 100644 index 25d9b483f..000000000 --- a/rustfs/src/app/storage_compat.rs +++ /dev/null @@ -1,624 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use rustfs_ecstore::api::admin as ecstore_admin; -use rustfs_ecstore::api::bucket as ecstore_bucket; -use rustfs_ecstore::api::capacity as ecstore_capacity; -use rustfs_ecstore::api::client as ecstore_client; -use rustfs_ecstore::api::compression as ecstore_compression; -use rustfs_ecstore::api::config as ecstore_config; -use rustfs_ecstore::api::data_usage as ecstore_data_usage; -use rustfs_ecstore::api::disk as ecstore_disk; -use rustfs_ecstore::api::error as ecstore_error; -use rustfs_ecstore::api::global as ecstore_global; -use rustfs_ecstore::api::layout as ecstore_layout; -use rustfs_ecstore::api::notification as ecstore_notification; -use rustfs_ecstore::api::rio as ecstore_rio; -use rustfs_ecstore::api::set_disk as ecstore_set_disk; -use rustfs_ecstore::api::storage as ecstore_storage; -use rustfs_ecstore::api::tier as ecstore_tier; - -pub(crate) const MIN_DISK_COMPRESSIBLE_SIZE: usize = ecstore_compression::MIN_DISK_COMPRESSIBLE_SIZE; - -pub(crate) type DiskError = ecstore_disk::error::DiskError; -pub(crate) type DynReader = ecstore_rio::DynReader; -pub(crate) type ECStore = ecstore_storage::ECStore; -pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; -pub(crate) type HashReader = ecstore_rio::HashReader; -pub(crate) type NotificationSys = ecstore_notification::NotificationSys; -pub(crate) type ObjectStoreResolver = dyn Fn() -> Option> + Send + Sync + 'static; -pub(crate) type ObjectInfo = ::ObjectInfo; -pub(crate) type ObjectOptions = ::ObjectOptions; -pub(crate) type PoolDecommissionInfo = ecstore_capacity::PoolDecommissionInfo; -pub(crate) type PoolStatus = ecstore_capacity::PoolStatus; -pub(crate) type StorageError = ecstore_error::StorageError; -pub(crate) type Error = StorageError; -pub(crate) type TierConfigMgr = ecstore_tier::tier::TierConfigMgr; -pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption; -pub(crate) type WritePlan = ecstore_rio::WritePlan; - -#[cfg(test)] -pub(crate) type DecryptReader = ecstore_rio::DecryptReader; -#[cfg(test)] -pub(crate) type EncryptReader = ecstore_rio::EncryptReader; -#[cfg(test)] -pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; -#[cfg(test)] -pub(crate) type Endpoints = ecstore_layout::Endpoints; -#[cfg(test)] -pub(crate) type HardLimitReader = ecstore_rio::HardLimitReader; -#[cfg(test)] -pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; -#[cfg(test)] -pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig; -#[cfg(test)] -pub(crate) type TierType = ecstore_tier::tier_config::TierType; -#[cfg(test)] -pub(crate) use ecstore_tier::warm_backend::WarmBackend as AppWarmBackend; -#[cfg(test)] -pub(crate) type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts; - -#[cfg(test)] -#[allow(non_snake_case)] -pub(crate) fn EndpointServerPools(pools: Vec) -> EndpointServerPools { - ecstore_layout::EndpointServerPools::from(pools) -} - -pub(crate) trait AppObjectLockConfigExt { - fn enabled(&self) -> bool; -} - -impl AppObjectLockConfigExt for s3s::dto::ObjectLockConfiguration { - fn enabled(&self) -> bool { - ::enabled(self) - } -} - -pub(crate) trait AppReplicationConfigExt { - fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec; - fn replicate(&self, opts: &replication::ObjectOpts) -> bool; -} - -impl AppReplicationConfigExt for s3s::dto::ReplicationConfiguration { - fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec { - ::filter_target_arns( - self, obj, - ) - } - - fn replicate(&self, opts: &replication::ObjectOpts) -> bool { - ::replicate(self, opts) - } -} - -pub(crate) trait AppVersioningConfigExt { - fn prefix_enabled(&self, prefix: &str) -> bool; - fn suspended(&self) -> bool; -} - -impl AppVersioningConfigExt for s3s::dto::VersioningConfiguration { - fn prefix_enabled(&self, prefix: &str) -> bool { - ::prefix_enabled(self, prefix) - } - - fn suspended(&self) -> bool { - ::suspended(self) - } -} - -pub(crate) async fn predict_lifecycle_expiration( - lifecycle: &s3s::dto::BucketLifecycleConfiguration, - obj: &lifecycle::lifecycle::ObjectOpts, -) -> lifecycle::lifecycle::Event { - ecstore_bucket::lifecycle::lifecycle::Lifecycle::predict_expiration(lifecycle, obj).await -} - -pub(crate) fn validate_restore_request(request: &s3s::dto::RestoreRequest, api: Arc) -> std::io::Result<()> { - ::validate(request, api) -} - -pub(crate) async fn get_server_info(get_pools: bool) -> rustfs_madmin::InfoMessage { - ecstore_admin::get_server_info(get_pools).await -} - -pub(crate) mod bucket_target_sys { - pub(crate) type BucketTargetSys = super::ecstore_bucket::bucket_target_sys::BucketTargetSys; -} - -pub(crate) mod lifecycle { - use super::ECStore; - - pub(crate) mod bucket_lifecycle_audit { - pub(crate) type LcEventSrc = super::super::ecstore_bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; - } - - pub(crate) mod bucket_lifecycle_ops { - use std::ops::Deref; - use std::sync::Arc; - - use super::ECStore; - use super::bucket_lifecycle_audit::LcEventSrc; - - pub(crate) type ExpiryState = super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; - - pub(crate) struct GlobalExpiryStateCompat; - - #[allow(non_upper_case_globals)] - pub(crate) static GLOBAL_ExpiryState: GlobalExpiryStateCompat = GlobalExpiryStateCompat; - - impl Deref for GlobalExpiryStateCompat { - type Target = Arc>; - - fn deref(&self) -> &Self::Target { - &super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState - } - } - - #[cfg(test)] - pub(crate) async fn init_background_expiry(api: Arc) { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await; - } - - pub(crate) async fn enqueue_expiry_for_existing_objects( - api: Arc, - bucket: &str, - ) -> Result<(), super::super::Error> { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_expiry_for_existing_objects(api, bucket).await - } - - pub(crate) async fn enqueue_transition_for_existing_objects( - api: Arc, - bucket: &str, - ) -> Result<(), super::super::Error> { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(api, bucket) - .await - } - - pub(crate) async fn enqueue_transition_immediate(oi: &super::super::ObjectInfo, src: LcEventSrc) { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_immediate(oi, src).await; - } - - pub(crate) async fn post_restore_opts( - version_id: &str, - bucket: &str, - object: &str, - ) -> Result { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(version_id, bucket, object).await - } - - pub(crate) async fn validate_transition_tier(lc: &s3s::dto::BucketLifecycleConfiguration) -> Result<(), std::io::Error> { - super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier(lc).await - } - - pub(crate) async fn validate_lifecycle_config( - lc: &s3s::dto::BucketLifecycleConfiguration, - lock_config: &s3s::dto::ObjectLockConfiguration, - ) -> Result<(), std::io::Error> { - use super::super::ecstore_bucket::lifecycle::lifecycle::Lifecycle as _; - - lc.validate(lock_config).await - } - } - - pub(crate) mod lifecycle_contract { - #[cfg(test)] - pub(crate) type IlmAction = super::super::ecstore_bucket::lifecycle::lifecycle::IlmAction; - pub(crate) type Event = super::super::ecstore_bucket::lifecycle::lifecycle::Event; - pub(crate) type ObjectOpts = super::super::ecstore_bucket::lifecycle::lifecycle::ObjectOpts; - pub(crate) type TransitionOptions = super::super::ecstore_bucket::lifecycle::lifecycle::TransitionOptions; - - pub(crate) const TRANSITION_COMPLETE: &str = super::super::ecstore_bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; - - pub(crate) fn expected_expiry_time(mod_time: time::OffsetDateTime, days: i32) -> time::OffsetDateTime { - super::super::ecstore_bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, days) - } - } - pub(crate) use lifecycle_contract as lifecycle; - - pub(crate) mod tier_delete_journal { - use std::sync::Arc; - - use super::ECStore; - - pub(crate) async fn persist_tier_delete_journal_entry( - api: Arc, - je: &super::tier_sweeper::Jentry, - ) -> std::io::Result<()> { - super::super::ecstore_bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(api, je).await - } - } - - pub(crate) mod tier_sweeper { - pub(crate) type Jentry = super::super::ecstore_bucket::lifecycle::tier_sweeper::Jentry; - - pub(crate) fn transitioned_delete_journal_entry( - version_id: Option, - versioned: bool, - suspended: bool, - transitioned: &rustfs_storage_api::TransitionedObject, - ) -> Option { - super::super::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( - version_id, - versioned, - suspended, - transitioned, - ) - } - - pub(crate) fn transitioned_force_delete_journal_entry( - transitioned: &rustfs_storage_api::TransitionedObject, - ) -> Option { - super::super::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(transitioned) - } - } -} - -pub(crate) mod metadata { - pub(crate) const BUCKET_CORS_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_CORS_CONFIG; - pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; - pub(crate) const BUCKET_NOTIFICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_NOTIFICATION_CONFIG; - pub(crate) const BUCKET_POLICY_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG; - pub(crate) const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG; - pub(crate) const BUCKET_REPLICATION_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG; - pub(crate) const BUCKET_SSECONFIG: &str = super::ecstore_bucket::metadata::BUCKET_SSECONFIG; - pub(crate) const BUCKET_TAGGING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; - pub(crate) const BUCKET_TARGETS_FILE: &str = super::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; - pub(crate) const BUCKET_VERSIONING_CONFIG: &str = super::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; - #[cfg(test)] - pub(crate) const OBJECT_LOCK_CONFIG: &str = super::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; -} - -pub(crate) mod metadata_sys { - use std::sync::Arc; - - use rustfs_policy::policy::BucketPolicy; - use s3s::dto::{ - BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, - PublicAccessBlockConfiguration, ServerSideEncryptionConfiguration, Tagging, - }; - use time::OffsetDateTime; - use tokio::sync::RwLock; - - #[cfg(test)] - use super::ECStore; - use super::Error; - use super::target::BucketTargets; - - pub(crate) type BucketMetadataSys = super::ecstore_bucket::metadata_sys::BucketMetadataSys; - - #[cfg(test)] - pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { - super::ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; - } - - pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { - super::ecstore_bucket::metadata_sys::delete(bucket, config_file).await - } - - pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await - } - - pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await - } - - pub(crate) async fn get_bucket_targets_config(bucket: &str) -> Result { - super::ecstore_bucket::metadata_sys::get_bucket_targets_config(bucket).await - } - - pub(crate) async fn get_cors_config(bucket: &str) -> Result<(CORSConfiguration, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_cors_config(bucket).await - } - - pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { - super::ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() - } - - pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await - } - - pub(crate) async fn get_notification_config(bucket: &str) -> Result, Error> { - super::ecstore_bucket::metadata_sys::get_notification_config(bucket).await - } - - pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await - } - - pub(crate) async fn get_public_access_block_config( - bucket: &str, - ) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await - } - - pub(crate) async fn get_replication_config( - bucket: &str, - ) -> Result<(s3s::dto::ReplicationConfiguration, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_replication_config(bucket).await - } - - pub(crate) async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_sse_config(bucket).await - } - - pub(crate) async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime), Error> { - super::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await - } - - pub(crate) async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { - super::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await - } -} - -pub(crate) mod object_api_utils { - pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { - super::ecstore_client::object_api_utils::to_s3s_etag(etag) - } -} - -pub(crate) mod object_lock { - pub(crate) mod objectlock { - pub(crate) fn get_object_legalhold_meta( - meta: &std::collections::HashMap, - ) -> s3s::dto::ObjectLockLegalHold { - super::super::ecstore_bucket::object_lock::objectlock::get_object_legalhold_meta(meta) - } - - pub(crate) fn get_object_retention_meta( - meta: &std::collections::HashMap, - ) -> s3s::dto::ObjectLockRetention { - super::super::ecstore_bucket::object_lock::objectlock::get_object_retention_meta(meta) - } - } - - pub(crate) mod objectlock_sys { - pub(crate) type BucketObjectLockSys = super::super::ecstore_bucket::object_lock::objectlock_sys::BucketObjectLockSys; - pub(crate) type ObjectLockBlockReason = super::super::ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; - - pub(crate) async fn check_object_lock_for_deletion( - bucket: &str, - obj_info: &super::super::ObjectInfo, - bypass_governance: bool, - ) -> Option { - super::super::ecstore_bucket::object_lock::objectlock_sys::check_object_lock_for_deletion( - bucket, - obj_info, - bypass_governance, - ) - .await - } - - pub(crate) fn is_retention_active(mode: &str, retain_until_date: Option<&s3s::dto::Date>) -> bool { - super::super::ecstore_bucket::object_lock::objectlock_sys::is_retention_active(mode, retain_until_date) - } - } -} - -pub(crate) mod policy_sys { - pub(crate) type PolicySys = super::ecstore_bucket::policy_sys::PolicySys; -} - -pub(crate) mod quota { - pub(crate) mod checker { - pub(crate) type QuotaChecker = super::super::ecstore_bucket::quota::checker::QuotaChecker; - } - - pub(crate) type QuotaOperation = super::ecstore_bucket::quota::QuotaOperation; -} - -pub(crate) mod replication { - use std::collections::HashMap; - use std::sync::Arc; - - pub(crate) type DeletedObjectReplicationInfo = super::ecstore_bucket::replication::DeletedObjectReplicationInfo; - pub(crate) type MustReplicateOptions = super::ecstore_bucket::replication::MustReplicateOptions; - pub(crate) type ObjectOpts = super::ecstore_bucket::replication::ObjectOpts; - pub(crate) type ReplicateDecision = rustfs_filemeta::ReplicateDecision; - - pub(crate) async fn check_replicate_delete( - bucket: &str, - dobj: &rustfs_storage_api::ObjectToDelete, - oi: &super::ObjectInfo, - del_opts: &super::ObjectOptions, - gerr: Option, - ) -> ReplicateDecision { - super::ecstore_bucket::replication::check_replicate_delete(bucket, dobj, oi, del_opts, gerr).await - } - - pub(crate) fn get_must_replicate_options( - user_defined: &HashMap, - user_tags: String, - status: rustfs_filemeta::ReplicationStatusType, - op_type: rustfs_filemeta::ReplicationType, - opts: super::ObjectOptions, - ) -> MustReplicateOptions { - super::ecstore_bucket::replication::get_must_replicate_options(user_defined, user_tags, status, op_type, opts) - } - - pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision { - super::ecstore_bucket::replication::must_replicate(bucket, object, mopts).await - } - - pub(crate) async fn schedule_replication( - oi: super::ObjectInfo, - store: Arc, - dsc: ReplicateDecision, - op_type: rustfs_filemeta::ReplicationType, - ) { - super::ecstore_bucket::replication::schedule_replication(oi, store, dsc, op_type).await; - } - - pub(crate) async fn schedule_replication_delete(dv: DeletedObjectReplicationInfo) { - super::ecstore_bucket::replication::schedule_replication_delete(dv).await; - } -} - -pub(crate) mod tagging { - pub(crate) fn decode_tags(tags: &str) -> Vec { - super::ecstore_bucket::tagging::decode_tags(tags) - } -} - -pub(crate) mod target { - #[cfg(test)] - pub(crate) type BucketTarget = super::ecstore_bucket::target::BucketTarget; - pub(crate) type BucketTargetType = super::ecstore_bucket::target::BucketTargetType; - pub(crate) type BucketTargets = super::ecstore_bucket::target::BucketTargets; -} - -pub(crate) mod utils { - pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { - super::ecstore_bucket::utils::serialize(val) - } -} - -pub(crate) mod versioning {} - -pub(crate) mod versioning_sys { - pub(crate) type BucketVersioningSys = super::ecstore_bucket::versioning_sys::BucketVersioningSys; -} - -#[cfg(test)] -pub(crate) mod transition_api { - pub(crate) type ReadCloser = super::ecstore_client::transition_api::ReadCloser; - pub(crate) type ReaderImpl = super::ecstore_client::transition_api::ReaderImpl; -} - -pub(crate) mod storageclass { - pub(crate) const STANDARD: &str = super::ecstore_config::storageclass::STANDARD; - #[cfg(test)] - pub(crate) const STANDARD_IA: &str = super::ecstore_config::storageclass::STANDARD_IA; -} - -pub(crate) fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { - ecstore_capacity::get_total_usable_capacity(disks, info) -} - -pub(crate) fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { - ecstore_capacity::get_total_usable_capacity_free(disks, info) -} - -pub(crate) fn is_disk_compressible(headers: &http::HeaderMap, object_name: &str) -> bool { - ecstore_compression::is_disk_compressible(headers, object_name) -} - -pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { - ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; -} - -pub(crate) async fn load_data_usage_from_backend( - store: Arc, -) -> std::result::Result { - ecstore_data_usage::load_data_usage_from_backend(store).await -} - -pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { - ecstore_data_usage::record_bucket_object_delete_memory(bucket, deleted_size, removed_current_object).await; -} - -pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option, new_size: u64) { - ecstore_data_usage::record_bucket_object_write_memory(bucket, previous_current_size, new_size).await; -} - -pub(crate) fn is_all_buckets_not_found(errs: &[Option]) -> bool { - ecstore_disk::error_reduce::is_all_buckets_not_found(errs) -} - -pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { - ecstore_error::is_err_bucket_not_found(err) -} - -pub(crate) fn is_err_object_not_found(err: &Error) -> bool { - ecstore_error::is_err_object_not_found(err) -} - -pub(crate) fn is_err_version_not_found(err: &Error) -> bool { - ecstore_error::is_err_version_not_found(err) -} - -#[cfg(test)] -pub(crate) struct GlobalTierConfigMgrCompat; - -#[cfg(test)] -#[allow(non_upper_case_globals)] -pub(crate) static GLOBAL_TierConfigMgr: GlobalTierConfigMgrCompat = GlobalTierConfigMgrCompat; - -#[cfg(test)] -impl std::ops::Deref for GlobalTierConfigMgrCompat { - type Target = Arc>; - - fn deref(&self) -> &Self::Target { - &ecstore_global::GLOBAL_TierConfigMgr - } -} - -pub(crate) fn get_global_endpoints_opt() -> Option { - ecstore_global::get_global_endpoints_opt() -} - -pub(crate) fn get_global_region() -> Option { - ecstore_global::get_global_region() -} - -pub(crate) fn get_global_tier_config_mgr() -> Arc> { - ecstore_global::get_global_tier_config_mgr() -} - -pub(crate) fn new_object_layer_fn() -> Option> { - ecstore_global::new_object_layer_fn() -} - -pub(crate) fn set_object_store_resolver(resolver: Arc) -> bool { - ecstore_global::set_object_store_resolver(resolver) -} - -pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { - ecstore_notification::get_global_notification_sys() -} - -#[cfg(test)] -pub(crate) fn boxed_reader(reader: R) -> DynReader -where - R: ecstore_rio::Reader + 'static, -{ - ecstore_rio::boxed_reader(reader) -} - -pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String { - ecstore_rio::compression_metadata_value(algorithm) -} - -pub(crate) fn wrap_reader(reader: R) -> DynReader -where - R: ecstore_rio::ReadStream + 'static, -{ - ecstore_rio::wrap_reader(reader) -} - -pub(crate) fn get_lock_acquire_timeout() -> tokio::time::Duration { - ecstore_set_disk::get_lock_acquire_timeout() -} - -pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool { - ecstore_set_disk::is_valid_storage_class(storage_class) -} - -#[cfg(test)] -pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<(), Error> { - ecstore_storage::init_local_disks(endpoint_pools).await -} diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index faf5fceac..aee9f6211 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::ECStore; use super::ecfs::FS; -use super::storage_compat::ECStore; -use super::storage_compat::resolve_object_store_handle; -use super::storage_compat::{ +use super::resolve_object_store_handle; +use super::{ PolicySys, StorageError, get_bucket_metadata, get_bucket_policy_raw, get_public_access_block_config, is_err_bucket_not_found, }; use crate::auth::{check_key_valid, get_condition_values_with_query_and_client_info, get_session_token}; @@ -930,7 +930,7 @@ impl S3Access for FS { let req_info = ReqInfo { cred, is_owner, - region: super::storage_compat::get_global_region(), + region: super::get_global_region(), request_context, ..Default::default() }; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index a7e7eceaa..c1418bd27 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::{ +use super::{ BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, check_retention_for_modification, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, encode_tags, get_bucket_accelerate_config, get_bucket_logging_config, @@ -20,7 +20,7 @@ use super::storage_compat::{ is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, record_replication_proxy, serialize, update_bucket_metadata_config, }; -use super::storage_compat::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; +use super::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::app::context::resolve_object_store_handle; use crate::app::multipart_usecase::DefaultMultipartUsecase; diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 88fdda900..9fa720be7 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::StorageReplicationConfigExt as _; -use super::storage_compat::{ +use super::StorageReplicationConfigExt as _; +use super::{ StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, get_bucket_replication_config, resolve_object_store_handle, }; @@ -738,7 +738,7 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet } /// Helper function to get store and validate bucket exists -pub(crate) async fn get_validated_store(bucket: &str) -> S3Result> { +pub(crate) async fn get_validated_store(bucket: &str) -> S3Result> { let Some(store) = resolve_object_store_handle() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 67388daa1..46dc89967 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -14,9 +14,7 @@ #[cfg(test)] mod tests { - use super::super::storage_compat::{ - BucketMetadata, DEFAULT_READ_BUFFER_SIZE, get_global_bucket_metadata_sys, set_bucket_metadata, - }; + use super::super::{BucketMetadata, DEFAULT_READ_BUFFER_SIZE, get_global_bucket_metadata_sys, set_bucket_metadata}; use crate::config::WorkloadProfile; use crate::server::cors; use crate::storage::ecfs::{FS, validate_object_lock_configuration_input}; @@ -941,8 +939,8 @@ mod tests { #[tokio::test] async fn test_validate_bucket_object_lock_enabled() { - use super::super::storage_compat::bucket_metadata_sys_initialized; - use super::super::storage_compat::set_bucket_metadata; + use super::super::bucket_metadata_sys_initialized; + use super::super::set_bucket_metadata; use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; use time::OffsetDateTime; @@ -1781,7 +1779,7 @@ mod tests { /// with a single-element vec value, matching the format expected by policy evaluation. #[test] fn test_object_tag_condition_key_format() { - use super::super::storage_compat::decode_tags_to_map; + use super::super::decode_tags_to_map; use std::collections::HashMap; let tags_str = "security=public&project=webapp&env=prod"; diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 7b446ce9f..9e34a95aa 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::ECStore; +use super::ECStore; use rustfs_storage_api::ListOperations as _; use std::sync::Arc; diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 7ceca8c71..58d5d8730 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -24,16 +24,15 @@ pub mod request_context; pub mod rpc; pub(crate) mod s3_api; pub(crate) mod sse; -pub(crate) mod storage_compat; pub mod timeout_wrapper; pub mod tonic_service; pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject; -pub(crate) type StorageGetObjectReader = self::storage_compat::GetObjectReader; -pub(crate) type StorageObjectInfo = self::storage_compat::ObjectInfo; -pub(crate) type StorageObjectOptions = self::storage_compat::ObjectOptions; +pub(crate) type StorageGetObjectReader = self::GetObjectReader; +pub(crate) type StorageObjectInfo = self::ObjectInfo; +pub(crate) type StorageObjectOptions = self::ObjectOptions; pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete; -pub(crate) type StoragePutObjReader = self::storage_compat::PutObjReader; +pub(crate) type StoragePutObjReader = self::PutObjReader; #[cfg(test)] mod concurrent_fix_test; @@ -54,3 +53,569 @@ pub(crate) use sse::{ extract_ssec_params_from_headers, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, }; + +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use rustfs_ecstore::api::admin as ecstore_admin; +use rustfs_ecstore::api::bucket as ecstore_bucket; +#[cfg(test)] +use rustfs_ecstore::api::config as ecstore_config; +use rustfs_ecstore::api::disk as ecstore_disk; +use rustfs_ecstore::api::error as ecstore_error; +use rustfs_ecstore::api::global as ecstore_global; +use rustfs_ecstore::api::metrics as ecstore_metrics; +use rustfs_ecstore::api::rio as ecstore_rio; +use rustfs_ecstore::api::rpc as ecstore_rpc; +use rustfs_ecstore::api::set_disk as ecstore_set_disk; +use rustfs_ecstore::api::storage as ecstore_storage; + +pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG; +pub(crate) const BUCKET_LOGGING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LOGGING_CONFIG; +pub(crate) const BUCKET_REQUEST_PAYMENT_CONFIG: &str = ecstore_bucket::metadata::BUCKET_REQUEST_PAYMENT_CONFIG; +pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; +pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG; +pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE; +pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; +pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL; +pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS; +pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG; +pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC; +#[cfg(test)] +pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; + +pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata; +#[cfg(test)] +pub(crate) type BucketMetadataSys = ecstore_bucket::metadata_sys::BucketMetadataSys; +pub(crate) type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys; +pub(crate) type CheckPartsResp = ecstore_disk::CheckPartsResp; +pub(crate) type CollectMetricsOpts = ecstore_metrics::CollectMetricsOpts; +pub(crate) type DeleteOptions = ecstore_disk::DeleteOptions; +pub(crate) type DiskError = ecstore_disk::error::DiskError; +pub(crate) type DiskInfo = ecstore_disk::DiskInfo; +pub(crate) type DiskInfoOptions = ecstore_disk::DiskInfoOptions; +pub(crate) type DiskResult = ecstore_disk::error::Result; +pub(crate) type DiskStore = ecstore_disk::DiskStore; +pub(crate) type ECStore = ecstore_storage::ECStore; +pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions; +pub(crate) type FileReader = ecstore_disk::FileReader; +pub(crate) type FileWriter = ecstore_disk::FileWriter; +pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client; +pub(crate) type MetricType = ecstore_metrics::MetricType; +pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo; +pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; +pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys; +pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo; +pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; +pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; +pub(crate) type ReadOptions = ecstore_disk::ReadOptions; +pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; +pub(crate) type StorageError = ecstore_error::StorageError; +pub(crate) type Error = StorageError; +pub(crate) type Result = core::result::Result; +pub(crate) type UpdateMetadataOpts = ecstore_disk::UpdateMetadataOpts; +pub(crate) type VolumeInfo = ecstore_disk::VolumeInfo; +pub(crate) type WalkDirOptions = ecstore_disk::WalkDirOptions; +pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption; + +pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperties { + ecstore_admin::get_local_server_property().await +} + +pub(crate) trait StorageDiskRpcExt { + async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult; + async fn delete_volume(&self, volume: &str) -> DiskResult<()>; + async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult>; + async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) + -> Vec>; + async fn delete_version( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()>; + async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult; + async fn read_version( + &self, + org_volume: &str, + volume: &str, + path: &str, + version_id: &str, + opts: &ReadOptions, + ) -> DiskResult; + async fn write_metadata( + &self, + org_volume: &str, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + ) -> DiskResult<()>; + async fn update_metadata( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + opts: &UpdateMetadataOpts, + ) -> DiskResult<()>; + async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; + async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; + async fn stat_volume(&self, volume: &str) -> DiskResult; + async fn list_volumes(&self) -> DiskResult>; + async fn make_volume(&self, volume: &str) -> DiskResult<()>; + async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()>; + async fn rename_data( + &self, + src_volume: &str, + src_path: &str, + file_info: rustfs_filemeta::FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> DiskResult; + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult>; + async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult; + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>; + async fn rename_part( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: bytes::Bytes, + ) -> DiskResult<()>; + async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; + async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; + async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; + async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult>; + async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()>; + async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()>; + async fn read_all(&self, volume: &str, path: &str) -> DiskResult; + async fn append_file(&self, volume: &str, path: &str) -> DiskResult; + async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult; +} + +impl StorageDiskRpcExt for T +where + T: ecstore_disk::DiskAPI, +{ + async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult { + ecstore_disk::DiskAPI::disk_info(self, opts).await + } + + async fn delete_volume(&self, volume: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_volume(self, volume).await + } + + async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult> { + ecstore_disk::DiskAPI::read_multiple(self, req).await + } + + async fn delete_versions( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + ecstore_disk::DiskAPI::delete_versions(self, volume, versions, opts).await + } + + async fn delete_version( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_version(self, volume, path, file_info, force_del_marker, opts).await + } + + async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult { + ecstore_disk::DiskAPI::read_xl(self, volume, path, read_data).await + } + + async fn read_version( + &self, + org_volume: &str, + volume: &str, + path: &str, + version_id: &str, + opts: &ReadOptions, + ) -> DiskResult { + ecstore_disk::DiskAPI::read_version(self, org_volume, volume, path, version_id, opts).await + } + + async fn write_metadata( + &self, + org_volume: &str, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::write_metadata(self, org_volume, volume, path, file_info).await + } + + async fn update_metadata( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + opts: &UpdateMetadataOpts, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::update_metadata(self, volume, path, file_info, opts).await + } + + async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::read_metadata(self, volume, path).await + } + + async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await + } + + async fn stat_volume(&self, volume: &str) -> DiskResult { + ecstore_disk::DiskAPI::stat_volume(self, volume).await + } + + async fn list_volumes(&self) -> DiskResult> { + ecstore_disk::DiskAPI::list_volumes(self).await + } + + async fn make_volume(&self, volume: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::make_volume(self, volume).await + } + + async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()> { + ecstore_disk::DiskAPI::make_volumes(self, volume).await + } + + async fn rename_data( + &self, + src_volume: &str, + src_path: &str, + file_info: rustfs_filemeta::FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> DiskResult { + ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await + } + + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult> { + ecstore_disk::DiskAPI::list_dir(self, origvolume, volume, dir_path, count).await + } + + async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult { + ecstore_disk::DiskAPI::read_file_stream(self, volume, path, offset, length).await + } + + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await + } + + async fn rename_part( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: bytes::Bytes, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await + } + + async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete(self, volume, path, options).await + } + + async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { + ecstore_disk::DiskAPI::verify_file(self, volume, path, file_info).await + } + + async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { + ecstore_disk::DiskAPI::check_parts(self, volume, path, file_info).await + } + + async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult> { + ecstore_disk::DiskAPI::read_parts(self, bucket, paths).await + } + + async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()> { + ecstore_disk::DiskAPI::walk_dir(self, opts, wr).await + } + + async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> { + ecstore_disk::DiskAPI::write_all(self, volume, path, data).await + } + + async fn read_all(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::read_all(self, volume, path).await + } + + async fn append_file(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::append_file(self, volume, path).await + } + + async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult { + ecstore_disk::DiskAPI::create_file(self, origvolume, volume, path, file_size).await + } +} + +pub(crate) trait StoragePeerS3ClientExt { + async fn heal_bucket( + &self, + bucket: &str, + opts: &rustfs_common::heal_channel::HealOpts, + ) -> DiskResult; + async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()>; + async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult>; + async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()>; + async fn get_bucket_info( + &self, + bucket: &str, + opts: &rustfs_storage_api::BucketOptions, + ) -> DiskResult; +} + +impl StoragePeerS3ClientExt for LocalPeerS3Client { + async fn heal_bucket( + &self, + bucket: &str, + opts: &rustfs_common::heal_channel::HealOpts, + ) -> DiskResult { + ecstore_rpc::PeerS3Client::heal_bucket(self, bucket, opts).await + } + + async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()> { + ecstore_rpc::PeerS3Client::make_bucket(self, bucket, opts).await + } + + async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult> { + ecstore_rpc::PeerS3Client::list_bucket(self, opts).await + } + + async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()> { + ecstore_rpc::PeerS3Client::delete_bucket(self, bucket, opts).await + } + + async fn get_bucket_info( + &self, + bucket: &str, + opts: &rustfs_storage_api::BucketOptions, + ) -> DiskResult { + ecstore_rpc::PeerS3Client::get_bucket_info(self, bucket, opts).await + } +} + +pub(crate) async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { + ecstore_bucket::metadata::load_bucket_metadata(api, bucket).await +} + +#[cfg(test)] +pub(crate) fn bucket_metadata_sys_initialized() -> bool { + ecstore_bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_some() +} + +#[cfg(test)] +pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { + ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() +} + +pub(crate) async fn delete_bucket_metadata_config(bucket: &str, config_file: &str) -> Result { + ecstore_bucket::metadata_sys::delete(bucket, config_file).await +} + +pub(crate) async fn get_bucket_metadata(bucket: &str) -> Result> { + ecstore_bucket::metadata_sys::get(bucket).await +} + +pub(crate) async fn get_bucket_accelerate_config( + bucket: &str, +) -> Result<(s3s::dto::AccelerateConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_accelerate_config(bucket).await +} + +pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await +} + +pub(crate) async fn get_bucket_cors_config(bucket: &str) -> Result<(s3s::dto::CORSConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_cors_config(bucket).await +} + +pub(crate) async fn get_bucket_logging_config(bucket: &str) -> Result<(s3s::dto::BucketLoggingStatus, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_logging_config(bucket).await +} + +pub(crate) async fn get_bucket_object_lock_config( + bucket: &str, +) -> Result<(s3s::dto::ObjectLockConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await +} + +pub(crate) async fn get_public_access_block_config( + bucket: &str, +) -> Result<(s3s::dto::PublicAccessBlockConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await +} + +pub(crate) async fn get_bucket_replication_config( + bucket: &str, +) -> Result<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_replication_config(bucket).await +} + +pub(crate) async fn get_bucket_request_payment_config( + bucket: &str, +) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_request_payment_config(bucket).await +} + +pub(crate) async fn get_bucket_sse_config( + bucket: &str, +) -> Result<(s3s::dto::ServerSideEncryptionConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_sse_config(bucket).await +} + +pub(crate) async fn get_bucket_website_config(bucket: &str) -> Result<(s3s::dto::WebsiteConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_website_config(bucket).await +} + +pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> { + ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, bm).await +} + +pub(crate) async fn update_bucket_metadata_config( + bucket: &str, + config_file: &str, + data: Vec, +) -> Result { + ecstore_bucket::metadata_sys::update(bucket, config_file, data).await +} + +pub(crate) fn add_object_lock_years(dt: time::OffsetDateTime, years: i32) -> time::OffsetDateTime { + ecstore_bucket::object_lock::objectlock_sys::add_years(dt, years) +} + +pub(crate) fn check_retention_for_modification( + user_defined: &std::collections::HashMap, + new_mode: Option<&str>, + new_retain_until: Option, + bypass_governance: bool, +) -> Option { + ecstore_bucket::object_lock::objectlock_sys::check_retention_for_modification( + user_defined, + new_mode, + new_retain_until, + bypass_governance, + ) +} + +pub(crate) async fn record_replication_proxy(bucket: &str, api: &str, is_err: bool) { + if let Some(stats) = ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() { + stats.inc_proxy(bucket, api, is_err).await; + } +} + +pub(crate) fn decode_tags(tags: &str) -> Vec { + ecstore_bucket::tagging::decode_tags(tags) +} + +pub(crate) fn decode_tags_to_map(tags: &str) -> std::collections::HashMap { + ecstore_bucket::tagging::decode_tags_to_map(tags) +} + +pub(crate) fn encode_tags(tags: Vec) -> String { + ecstore_bucket::tagging::encode_tags(tags) +} + +pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { + ecstore_bucket::utils::serialize(val) +} + +pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { + ecstore_error::is_err_bucket_not_found(err) +} + +pub(crate) fn is_err_object_not_found(err: &Error) -> bool { + ecstore_error::is_err_object_not_found(err) +} + +pub(crate) fn is_err_version_not_found(err: &Error) -> bool { + ecstore_error::is_err_version_not_found(err) +} + +pub(crate) fn get_global_lock_client() -> Option> { + ecstore_global::get_global_lock_client() +} + +pub(crate) fn get_global_region() -> Option { + ecstore_global::get_global_region() +} + +pub(crate) fn resolve_object_store_handle() -> Option> { + ecstore_global::resolve_object_store_handle() +} + +pub(crate) async fn collect_local_metrics( + types: MetricType, + opts: &CollectMetricsOpts, +) -> rustfs_madmin::metrics::RealtimeMetrics { + ecstore_metrics::collect_local_metrics(types, opts).await +} + +pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &http::HeaderMap) -> std::io::Result<()> { + ecstore_rpc::verify_rpc_signature(url, method, headers) +} + +pub(crate) async fn reload_transition_tier_config(api: Arc) -> std::io::Result<()> { + ecstore_global::GLOBAL_TierConfigMgr.write().await.reload(api).await +} + +pub(crate) async fn all_local_disk_path() -> Vec { + ecstore_storage::all_local_disk_path().await +} + +pub(crate) async fn find_local_disk_by_ref(disk_ref: &str) -> Option { + ecstore_storage::find_local_disk_by_ref(disk_ref).await +} + +pub(crate) trait StorageReplicationConfigExt { + fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool; +} + +impl StorageReplicationConfigExt for s3s::dto::ReplicationConfiguration { + fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool { + ::has_active_rules( + self, prefix, recursive, + ) + } +} + +pub(crate) trait StorageVersioningConfigExt { + fn enabled(&self) -> bool; +} + +impl StorageVersioningConfigExt for s3s::dto::VersioningConfiguration { + fn enabled(&self) -> bool { + ::enabled(self) + } +} + +pub(crate) type GetObjectReader = ::GetObjectReader; +pub(crate) type ObjectInfo = ::ObjectInfo; +pub(crate) type ObjectOptions = ::ObjectOptions; +pub(crate) type PutObjReader = ::PutObjectReader; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 90f2a8681..3ca667386 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::storage_compat::BucketVersioningSys; -use super::storage_compat::Result; -use super::storage_compat::StorageError; +use super::BucketVersioningSys; +use super::Result; +use super::StorageError; use http::header::{IF_MATCH, IF_NONE_MATCH}; use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 4c2dc20bf..14e5204d6 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::DEFAULT_READ_BUFFER_SIZE; -use super::super::storage_compat::StorageDiskRpcExt as _; -use super::super::storage_compat::WalkDirOptions; -use super::super::storage_compat::find_local_disk_by_ref; -use super::super::storage_compat::verify_rpc_signature; +use super::super::DEFAULT_READ_BUFFER_SIZE; +use super::super::StorageDiskRpcExt as _; +use super::super::WalkDirOptions; +use super::super::find_local_disk_by_ref; +use super::super::verify_rpc_signature; use crate::server::RPC_PREFIX; use crate::storage::request_context::spawn_traced; use bytes::{Bytes, BytesMut}; diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 8fdd0f1df..0c02ee134 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::storage_compat::{ +use super::super::{ CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, UpdateMetadataOpts, all_local_disk_path, @@ -122,11 +122,11 @@ fn unimplemented_rpc(method: &str) -> Status { Status::unimplemented(format!("{method} is not implemented")) } -fn background_rebalance_start_error_message(result: super::super::storage_compat::Result<()>) -> Option { +fn background_rebalance_start_error_message(result: super::super::Result<()>) -> Option { result.err().map(|err| format!("start_rebalance failed: {err}")) } -fn stop_rebalance_response(result: super::super::storage_compat::Result<()>) -> StopRebalanceResponse { +fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceResponse { match result { Ok(_) => StopRebalanceResponse { success: true, @@ -2389,7 +2389,7 @@ mod tests { #[test] fn test_background_rebalance_start_error_message_formats_error() { - let message = background_rebalance_start_error_message(Err(super::super::super::storage_compat::Error::other("boom"))) + let message = background_rebalance_start_error_message(Err(super::super::super::Error::other("boom"))) .expect("background rebalance start failure should be formatted"); assert!(message.contains("start_rebalance failed")); @@ -2398,7 +2398,7 @@ mod tests { #[test] fn test_stop_rebalance_response_reports_local_stop_error() { - let response = stop_rebalance_response(Err(super::super::super::storage_compat::Error::other("boom"))); + let response = stop_rebalance_response(Err(super::super::super::Error::other("boom"))); assert!(!response.success); assert!(response.error_info.as_deref().is_some_and(|message| message.contains("boom"))); @@ -2734,10 +2734,7 @@ mod tests { let mut vars = HashMap::new(); vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string()); - vars.insert( - PEER_RESTSUB_SYS.to_string(), - super::super::super::storage_compat::STORAGE_CLASS_SUB_SYS.to_string(), - ); + vars.insert(PEER_RESTSUB_SYS.to_string(), super::super::super::STORAGE_CLASS_SUB_SYS.to_string()); let request = Request::new(SignalServiceRequest { vars: Some(Mss { value: vars }), diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index c7536d2a6..deacdff61 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -69,7 +69,7 @@ //! } //! ``` -use super::storage_compat::StorageError; +use super::StorageError; #[cfg(feature = "rio-v2")] use aes_gcm::aead::Payload; use aes_gcm::{ @@ -133,8 +133,8 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE; #[cfg(feature = "rio-v2")] const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation"; -use super::storage_compat::Error; -use super::storage_compat::get_bucket_sse_config; +use super::Error; +use super::get_bucket_sse_config; use crate::error::ApiError; use rustfs_utils::http::headers::{ AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, @@ -751,18 +751,16 @@ pub struct ManagedSealedKey { } impl EncryptionMaterial { - pub fn write_encryption(&self, multipart_part_number: Option) -> super::storage_compat::WriteEncryption { + pub fn write_encryption(&self, multipart_part_number: Option) -> super::WriteEncryption { match (self.key_kind, multipart_part_number) { (EncryptionKeyKind::Object, Some(part_number)) => { - super::storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32) + super::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32) } - (EncryptionKeyKind::Object, None) => super::storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes), + (EncryptionKeyKind::Object, None) => super::WriteEncryption::singlepart_object_key(self.key_bytes), (EncryptionKeyKind::Direct, Some(part_number)) => { - super::storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number) - } - (EncryptionKeyKind::Direct, None) => { - super::storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce) + super::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number) } + (EncryptionKeyKind::Direct, None) => super::WriteEncryption::singlepart(self.key_bytes, self.base_nonce), } } } diff --git a/rustfs/src/storage/storage_compat.rs b/rustfs/src/storage/storage_compat.rs deleted file mode 100644 index 06a3cfe07..000000000 --- a/rustfs/src/storage/storage_compat.rs +++ /dev/null @@ -1,565 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use rustfs_ecstore::api::admin as ecstore_admin; -use rustfs_ecstore::api::bucket as ecstore_bucket; -#[cfg(test)] -use rustfs_ecstore::api::config as ecstore_config; -use rustfs_ecstore::api::disk as ecstore_disk; -use rustfs_ecstore::api::error as ecstore_error; -use rustfs_ecstore::api::global as ecstore_global; -use rustfs_ecstore::api::metrics as ecstore_metrics; -use rustfs_ecstore::api::rio as ecstore_rio; -use rustfs_ecstore::api::rpc as ecstore_rpc; -use rustfs_ecstore::api::set_disk as ecstore_set_disk; -use rustfs_ecstore::api::storage as ecstore_storage; - -pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG; -pub(crate) const BUCKET_LOGGING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LOGGING_CONFIG; -pub(crate) const BUCKET_REQUEST_PAYMENT_CONFIG: &str = ecstore_bucket::metadata::BUCKET_REQUEST_PAYMENT_CONFIG; -pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; -pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG; -pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE; -pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; -pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL; -pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS; -pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG; -pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC; -#[cfg(test)] -pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; - -pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata; -#[cfg(test)] -pub(crate) type BucketMetadataSys = ecstore_bucket::metadata_sys::BucketMetadataSys; -pub(crate) type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys; -pub(crate) type CheckPartsResp = ecstore_disk::CheckPartsResp; -pub(crate) type CollectMetricsOpts = ecstore_metrics::CollectMetricsOpts; -pub(crate) type DeleteOptions = ecstore_disk::DeleteOptions; -pub(crate) type DiskError = ecstore_disk::error::DiskError; -pub(crate) type DiskInfo = ecstore_disk::DiskInfo; -pub(crate) type DiskInfoOptions = ecstore_disk::DiskInfoOptions; -pub(crate) type DiskResult = ecstore_disk::error::Result; -pub(crate) type DiskStore = ecstore_disk::DiskStore; -pub(crate) type ECStore = ecstore_storage::ECStore; -pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions; -pub(crate) type FileReader = ecstore_disk::FileReader; -pub(crate) type FileWriter = ecstore_disk::FileWriter; -pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client; -pub(crate) type MetricType = ecstore_metrics::MetricType; -pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo; -pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; -pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys; -pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo; -pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; -pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; -pub(crate) type ReadOptions = ecstore_disk::ReadOptions; -pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; -pub(crate) type StorageError = ecstore_error::StorageError; -pub(crate) type Error = StorageError; -pub(crate) type Result = core::result::Result; -pub(crate) type UpdateMetadataOpts = ecstore_disk::UpdateMetadataOpts; -pub(crate) type VolumeInfo = ecstore_disk::VolumeInfo; -pub(crate) type WalkDirOptions = ecstore_disk::WalkDirOptions; -pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption; - -pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperties { - ecstore_admin::get_local_server_property().await -} - -pub(crate) trait StorageDiskRpcExt { - async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult; - async fn delete_volume(&self, volume: &str) -> DiskResult<()>; - async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult>; - async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) - -> Vec>; - async fn delete_version( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> DiskResult<()>; - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult; - async fn read_version( - &self, - org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> DiskResult; - async fn write_metadata( - &self, - org_volume: &str, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - ) -> DiskResult<()>; - async fn update_metadata( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - opts: &UpdateMetadataOpts, - ) -> DiskResult<()>; - async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; - async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; - async fn stat_volume(&self, volume: &str) -> DiskResult; - async fn list_volumes(&self) -> DiskResult>; - async fn make_volume(&self, volume: &str) -> DiskResult<()>; - async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()>; - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - file_info: rustfs_filemeta::FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> DiskResult; - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult>; - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult; - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>; - async fn rename_part( - &self, - src_volume: &str, - src_path: &str, - dst_volume: &str, - dst_path: &str, - meta: bytes::Bytes, - ) -> DiskResult<()>; - async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; - async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; - async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; - async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult>; - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()>; - async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()>; - async fn read_all(&self, volume: &str, path: &str) -> DiskResult; - async fn append_file(&self, volume: &str, path: &str) -> DiskResult; - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult; -} - -impl StorageDiskRpcExt for T -where - T: ecstore_disk::DiskAPI, -{ - async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult { - ecstore_disk::DiskAPI::disk_info(self, opts).await - } - - async fn delete_volume(&self, volume: &str) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete_volume(self, volume).await - } - - async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult> { - ecstore_disk::DiskAPI::read_multiple(self, req).await - } - - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - opts: DeleteOptions, - ) -> Vec> { - ecstore_disk::DiskAPI::delete_versions(self, volume, versions, opts).await - } - - async fn delete_version( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete_version(self, volume, path, file_info, force_del_marker, opts).await - } - - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult { - ecstore_disk::DiskAPI::read_xl(self, volume, path, read_data).await - } - - async fn read_version( - &self, - org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> DiskResult { - ecstore_disk::DiskAPI::read_version(self, org_volume, volume, path, version_id, opts).await - } - - async fn write_metadata( - &self, - org_volume: &str, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::write_metadata(self, org_volume, volume, path, file_info).await - } - - async fn update_metadata( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - opts: &UpdateMetadataOpts, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::update_metadata(self, volume, path, file_info, opts).await - } - - async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult { - ecstore_disk::DiskAPI::read_metadata(self, volume, path).await - } - - async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await - } - - async fn stat_volume(&self, volume: &str) -> DiskResult { - ecstore_disk::DiskAPI::stat_volume(self, volume).await - } - - async fn list_volumes(&self) -> DiskResult> { - ecstore_disk::DiskAPI::list_volumes(self).await - } - - async fn make_volume(&self, volume: &str) -> DiskResult<()> { - ecstore_disk::DiskAPI::make_volume(self, volume).await - } - - async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()> { - ecstore_disk::DiskAPI::make_volumes(self, volume).await - } - - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - file_info: rustfs_filemeta::FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> DiskResult { - ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await - } - - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult> { - ecstore_disk::DiskAPI::list_dir(self, origvolume, volume, dir_path, count).await - } - - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult { - ecstore_disk::DiskAPI::read_file_stream(self, volume, path, offset, length).await - } - - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> { - ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await - } - - async fn rename_part( - &self, - src_volume: &str, - src_path: &str, - dst_volume: &str, - dst_path: &str, - meta: bytes::Bytes, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await - } - - async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete(self, volume, path, options).await - } - - async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { - ecstore_disk::DiskAPI::verify_file(self, volume, path, file_info).await - } - - async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { - ecstore_disk::DiskAPI::check_parts(self, volume, path, file_info).await - } - - async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult> { - ecstore_disk::DiskAPI::read_parts(self, bucket, paths).await - } - - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()> { - ecstore_disk::DiskAPI::walk_dir(self, opts, wr).await - } - - async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> { - ecstore_disk::DiskAPI::write_all(self, volume, path, data).await - } - - async fn read_all(&self, volume: &str, path: &str) -> DiskResult { - ecstore_disk::DiskAPI::read_all(self, volume, path).await - } - - async fn append_file(&self, volume: &str, path: &str) -> DiskResult { - ecstore_disk::DiskAPI::append_file(self, volume, path).await - } - - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult { - ecstore_disk::DiskAPI::create_file(self, origvolume, volume, path, file_size).await - } -} - -pub(crate) trait StoragePeerS3ClientExt { - async fn heal_bucket( - &self, - bucket: &str, - opts: &rustfs_common::heal_channel::HealOpts, - ) -> DiskResult; - async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()>; - async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult>; - async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()>; - async fn get_bucket_info( - &self, - bucket: &str, - opts: &rustfs_storage_api::BucketOptions, - ) -> DiskResult; -} - -impl StoragePeerS3ClientExt for LocalPeerS3Client { - async fn heal_bucket( - &self, - bucket: &str, - opts: &rustfs_common::heal_channel::HealOpts, - ) -> DiskResult { - ecstore_rpc::PeerS3Client::heal_bucket(self, bucket, opts).await - } - - async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()> { - ecstore_rpc::PeerS3Client::make_bucket(self, bucket, opts).await - } - - async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult> { - ecstore_rpc::PeerS3Client::list_bucket(self, opts).await - } - - async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()> { - ecstore_rpc::PeerS3Client::delete_bucket(self, bucket, opts).await - } - - async fn get_bucket_info( - &self, - bucket: &str, - opts: &rustfs_storage_api::BucketOptions, - ) -> DiskResult { - ecstore_rpc::PeerS3Client::get_bucket_info(self, bucket, opts).await - } -} - -pub(crate) async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { - ecstore_bucket::metadata::load_bucket_metadata(api, bucket).await -} - -#[cfg(test)] -pub(crate) fn bucket_metadata_sys_initialized() -> bool { - ecstore_bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_some() -} - -#[cfg(test)] -pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { - ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() -} - -pub(crate) async fn delete_bucket_metadata_config(bucket: &str, config_file: &str) -> Result { - ecstore_bucket::metadata_sys::delete(bucket, config_file).await -} - -pub(crate) async fn get_bucket_metadata(bucket: &str) -> Result> { - ecstore_bucket::metadata_sys::get(bucket).await -} - -pub(crate) async fn get_bucket_accelerate_config( - bucket: &str, -) -> Result<(s3s::dto::AccelerateConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_accelerate_config(bucket).await -} - -pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await -} - -pub(crate) async fn get_bucket_cors_config(bucket: &str) -> Result<(s3s::dto::CORSConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_cors_config(bucket).await -} - -pub(crate) async fn get_bucket_logging_config(bucket: &str) -> Result<(s3s::dto::BucketLoggingStatus, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_logging_config(bucket).await -} - -pub(crate) async fn get_bucket_object_lock_config( - bucket: &str, -) -> Result<(s3s::dto::ObjectLockConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await -} - -pub(crate) async fn get_public_access_block_config( - bucket: &str, -) -> Result<(s3s::dto::PublicAccessBlockConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await -} - -pub(crate) async fn get_bucket_replication_config( - bucket: &str, -) -> Result<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_replication_config(bucket).await -} - -pub(crate) async fn get_bucket_request_payment_config( - bucket: &str, -) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_request_payment_config(bucket).await -} - -pub(crate) async fn get_bucket_sse_config( - bucket: &str, -) -> Result<(s3s::dto::ServerSideEncryptionConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_sse_config(bucket).await -} - -pub(crate) async fn get_bucket_website_config(bucket: &str) -> Result<(s3s::dto::WebsiteConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_website_config(bucket).await -} - -pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> { - ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, bm).await -} - -pub(crate) async fn update_bucket_metadata_config( - bucket: &str, - config_file: &str, - data: Vec, -) -> Result { - ecstore_bucket::metadata_sys::update(bucket, config_file, data).await -} - -pub(crate) fn add_object_lock_years(dt: time::OffsetDateTime, years: i32) -> time::OffsetDateTime { - ecstore_bucket::object_lock::objectlock_sys::add_years(dt, years) -} - -pub(crate) fn check_retention_for_modification( - user_defined: &std::collections::HashMap, - new_mode: Option<&str>, - new_retain_until: Option, - bypass_governance: bool, -) -> Option { - ecstore_bucket::object_lock::objectlock_sys::check_retention_for_modification( - user_defined, - new_mode, - new_retain_until, - bypass_governance, - ) -} - -pub(crate) async fn record_replication_proxy(bucket: &str, api: &str, is_err: bool) { - if let Some(stats) = ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() { - stats.inc_proxy(bucket, api, is_err).await; - } -} - -pub(crate) fn decode_tags(tags: &str) -> Vec { - ecstore_bucket::tagging::decode_tags(tags) -} - -pub(crate) fn decode_tags_to_map(tags: &str) -> std::collections::HashMap { - ecstore_bucket::tagging::decode_tags_to_map(tags) -} - -pub(crate) fn encode_tags(tags: Vec) -> String { - ecstore_bucket::tagging::encode_tags(tags) -} - -pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { - ecstore_bucket::utils::serialize(val) -} - -pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { - ecstore_error::is_err_bucket_not_found(err) -} - -pub(crate) fn is_err_object_not_found(err: &Error) -> bool { - ecstore_error::is_err_object_not_found(err) -} - -pub(crate) fn is_err_version_not_found(err: &Error) -> bool { - ecstore_error::is_err_version_not_found(err) -} - -pub(crate) fn get_global_lock_client() -> Option> { - ecstore_global::get_global_lock_client() -} - -pub(crate) fn get_global_region() -> Option { - ecstore_global::get_global_region() -} - -pub(crate) fn resolve_object_store_handle() -> Option> { - ecstore_global::resolve_object_store_handle() -} - -pub(crate) async fn collect_local_metrics( - types: MetricType, - opts: &CollectMetricsOpts, -) -> rustfs_madmin::metrics::RealtimeMetrics { - ecstore_metrics::collect_local_metrics(types, opts).await -} - -pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &http::HeaderMap) -> std::io::Result<()> { - ecstore_rpc::verify_rpc_signature(url, method, headers) -} - -pub(crate) async fn reload_transition_tier_config(api: Arc) -> std::io::Result<()> { - ecstore_global::GLOBAL_TierConfigMgr.write().await.reload(api).await -} - -pub(crate) async fn all_local_disk_path() -> Vec { - ecstore_storage::all_local_disk_path().await -} - -pub(crate) async fn find_local_disk_by_ref(disk_ref: &str) -> Option { - ecstore_storage::find_local_disk_by_ref(disk_ref).await -} - -pub(crate) trait StorageReplicationConfigExt { - fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool; -} - -impl StorageReplicationConfigExt for s3s::dto::ReplicationConfiguration { - fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool { - ::has_active_rules( - self, prefix, recursive, - ) - } -} - -pub(crate) trait StorageVersioningConfigExt { - fn enabled(&self) -> bool; -} - -impl StorageVersioningConfigExt for s3s::dto::VersioningConfiguration { - fn enabled(&self) -> bool { - ::enabled(self) - } -} - -pub(crate) type GetObjectReader = ::GetObjectReader; -pub(crate) type ObjectInfo = ::ObjectInfo; -pub(crate) type ObjectOptions = ::ObjectOptions; -pub(crate) type PutObjReader = ::PutObjectReader; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index f5a141cc1..42e6c3d30 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -715,7 +715,7 @@ fi --glob '!**/storage_compat.rs' \ --glob '!**/*storage_compat.rs' \ --glob '!target/**' \ - | rg -v '^(rustfs/src/(capacity/service|config/config_test|error|init|runtime_capabilities|startup_(background|bucket_metadata|fs_guard|iam|lifecycle|notification|server|services|shutdown|storage)|table_catalog|workload_admission)\.rs|rustfs/src/server/(event|http|module_switch|readiness)\.rs|rustfs/src/storage/s3_api/(bucket|multipart)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/lib\.rs|crates/notify/src/config_manager\.rs|crates/obs/src/metrics/(scheduler|stats_collector)\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/object_store\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true + | rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod|capacity/service|config/config_test|error|init|runtime_capabilities|startup_(background|bucket_metadata|fs_guard|iam|lifecycle|notification|server|services|shutdown|storage)|table_catalog|workload_admission)\.rs|rustfs/src/server/(event|http|module_switch|readiness)\.rs|rustfs/src/storage/s3_api/(bucket|multipart)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/lib\.rs|crates/notify/src/config_manager\.rs|crates/obs/src/metrics/(scheduler|stats_collector)\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/object_store\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true ) | cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE" @@ -854,20 +854,22 @@ fi ( cd "$ROOT_DIR" { - rg -n --no-heading 'crate::admin::storage_compat' \ - rustfs/src/admin/handlers rustfs/src/admin/service rustfs/src/admin/router.rs \ - --glob '!**/*storage_compat.rs' || true - rg -n --no-heading 'crate::app::storage_compat' \ + for file in \ + rustfs/src/admin/storage_compat.rs \ + rustfs/src/app/storage_compat.rs \ + rustfs/src/storage/storage_compat.rs; do + [[ -e "$file" ]] && printf '%s:1:RustFS owner bridge file exists\n' "$file" + done + rg -n --with-filename 'storage_compat' \ + rustfs/src/admin \ rustfs/src/app \ - --glob '!**/*storage_compat.rs' || true - rg -n --no-heading 'crate::storage::storage_compat' \ rustfs/src/storage \ - --glob '!**/*storage_compat.rs' || true + -g '*.rs' || true } ) >"$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE" if [[ -s "$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE" ]]; then - report_failure "RustFS owner compatibility consumers must route through their local compatibility boundary: $(paste -sd '; ' "$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE")" + report_failure "RustFS app/admin/storage consumers must import owner APIs directly instead of local storage compatibility bridges: $(paste -sd '; ' "$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE")" fi ( @@ -883,7 +885,7 @@ fi ( cd "$ROOT_DIR" rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::config::\{[^}]*\bcom\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::\{[^}]*\binit\s*(?:,|})[^}]*\}\s*;' \ - rustfs/src/admin/storage_compat.rs || true + rustfs/src/admin/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_ADMIN_CONFIG_STORAGE_COMPAT_MODULE_HITS_FILE" if [[ -s "$RUSTFS_ADMIN_CONFIG_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then @@ -893,7 +895,7 @@ fi ( cd "$ROOT_DIR" rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::bucket::\{[^}]*\b(?:metadata|metadata_sys|object_lock|policy_sys|replication|tagging|utils|versioning|versioning_sys)\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::bucket::(?:metadata|metadata_sys|object_lock|policy_sys|replication|tagging|utils|versioning|versioning_sys)\s*;|pub\(crate\)\s+use rustfs_ecstore::api::client::object_api_utils\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::com\s*;' \ - rustfs/src/storage/storage_compat.rs || true + rustfs/src/storage/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_STORAGE_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" if [[ -s "$RUSTFS_STORAGE_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then @@ -913,7 +915,7 @@ fi next if $path eq "rpc::PeerS3Client"; print "$ARGV:pub(crate) use rustfs_ecstore::api::$path;\n"; } - ' rustfs/src/storage/storage_compat.rs || true + ' rustfs/src/storage/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE" if [[ -s "$RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE" ]]; then @@ -923,7 +925,7 @@ fi ( cd "$ROOT_DIR" rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::bucket::\{[^}]*\b(?:bandwidth|bucket_target_sys|lifecycle|metadata|metadata_sys|quota|replication|target|utils|versioning|versioning_sys)\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::bucket::(?:bandwidth|bucket_target_sys|lifecycle|metadata|metadata_sys|quota|replication|target|utils|versioning|versioning_sys)\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::\{[^}]*\bstorageclass\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::storageclass\s*;' \ - rustfs/src/admin/storage_compat.rs || true + rustfs/src/admin/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_ADMIN_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" if [[ -s "$RUSTFS_ADMIN_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then @@ -933,7 +935,7 @@ fi ( cd "$ROOT_DIR" rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::bucket::\{[^}]*\b(?:bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning|versioning_sys)\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::bucket::(?:bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning|versioning_sys)\s*;|pub\(crate\)\s+use rustfs_ecstore::api::client::(?:object_api_utils|transition_api)\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::storageclass\s*;' \ - rustfs/src/app/storage_compat.rs || true + rustfs/src/app/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" if [[ -s "$RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then @@ -945,7 +947,7 @@ fi rg -n --no-heading 'rustfs_ecstore::api::(object::(ObjectInfo|ObjectOptions)|error::(Error|Result))' \ rustfs/src/app/storage_compat.rs \ rustfs/src/admin/storage_compat.rs \ - rustfs/src/storage/storage_compat.rs || true + rustfs/src/storage/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_OUTER_COMPAT_FACADE_ALIAS_HITS_FILE" if [[ -s "$RUSTFS_OUTER_COMPAT_FACADE_ALIAS_HITS_FILE" ]]; then @@ -957,7 +959,7 @@ fi rg -n --no-heading '(fn .*rustfs_ecstore::api::(bucket::metadata_sys::BucketMetadataSys|bucket::object_lock::objectlock_sys::ObjectLockBlockReason|bucket::lifecycle::tier_sweeper::Jentry|bucket::bandwidth::monitor::Monitor|notification::NotificationSys)|-> .*rustfs_ecstore::api::(bucket::metadata_sys::BucketMetadataSys|bucket::object_lock::objectlock_sys::ObjectLockBlockReason|bucket::lifecycle::tier_sweeper::Jentry|bucket::bandwidth::monitor::Monitor|notification::NotificationSys)|: &?rustfs_ecstore::api::(bucket::metadata_sys::BucketMetadataSys|bucket::object_lock::objectlock_sys::ObjectLockBlockReason|bucket::lifecycle::tier_sweeper::Jentry|bucket::bandwidth::monitor::Monitor|notification::NotificationSys))' \ rustfs/src/app/storage_compat.rs \ rustfs/src/admin/storage_compat.rs \ - rustfs/src/storage/storage_compat.rs || true + rustfs/src/storage/storage_compat.rs 2>/dev/null || true ) >"$RUSTFS_OUTER_COMPAT_SIGNATURE_ALIAS_HITS_FILE" if [[ -s "$RUSTFS_OUTER_COMPAT_SIGNATURE_ALIAS_HITS_FILE" ]]; then @@ -967,7 +969,7 @@ fi ( cd "$ROOT_DIR" rg -n --no-heading 'rustfs_ecstore::api::' rustfs/src/storage/storage_compat.rs \ - | rg -v '^[0-9]+:use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' || true + 2>/dev/null | rg -v '^[0-9]+:use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' || true ) >"$RUSTFS_STORAGE_COMPAT_RAW_FACADE_PATH_HITS_FILE" if [[ -s "$RUSTFS_STORAGE_COMPAT_RAW_FACADE_PATH_HITS_FILE" ]]; then @@ -977,7 +979,7 @@ fi ( cd "$ROOT_DIR" rg -n --no-heading 'rustfs_ecstore::api::' rustfs/src/app/storage_compat.rs rustfs/src/admin/storage_compat.rs \ - | rg -v '^[^:]+:[0-9]+:use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' || true + 2>/dev/null | rg -v '^[^:]+:[0-9]+:use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' || true ) >"$RUSTFS_APP_ADMIN_COMPAT_RAW_FACADE_PATH_HITS_FILE" if [[ -s "$RUSTFS_APP_ADMIN_COMPAT_RAW_FACADE_PATH_HITS_FILE" ]]; then