From 2aca607119a90c294927e85753d6d9f8f39fa1ef Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Fri, 26 Jun 2026 07:52:49 +0800 Subject: [PATCH] refactor: move storage owner boundary aggregation (#3890) --- docs/architecture/migration-progress.md | 40 +- rustfs/src/storage/mod.rs | 938 +---------------- rustfs/src/storage/storage_api.rs | 951 ++++++++++++++++++ scripts/check_architecture_migration_rules.sh | 21 +- 4 files changed, 999 insertions(+), 951 deletions(-) create mode 100644 rustfs/src/storage/storage_api.rs diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 0deeadb27..6fd5be313 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-remaining-external-storage-boundaries` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222`. -- Based on: API-222 branch; branch routes remaining external runtime, test, and fuzz storage symbols through local `storage_api` boundaries. +- Branch: `overtrue/arch-storage-owner-storage-api-boundary` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223`. +- Based on: API-223 branch; branch moves the storage owner root ECStore facade and storage contract aggregation into the owner-local `storage_api` boundary. - PR type for this branch: `consumer-migration` - Runtime behavior changes: none. - Rust code changes: route replication pool, outbound TLS generation, runtime @@ -60,7 +60,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block test ECStore plus storage contract imports through crate-local `storage_api` boundaries, and remaining IAM, notify, OBS metrics, Swift, S3 Select, e2e, and fuzz ECStore/storage contract imports through local `storage_api` - boundaries. + boundaries, plus storage owner root ECStore facade and storage contract + aggregation through `rustfs/src/storage/storage_api.rs`. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade @@ -70,8 +71,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block event-bridge thin module regressions, plus IAM runtime-source bypasses; accept the reviewed AppContext resolver reverse dependencies in the layer baseline, and block direct admin AppContext resolver consumers outside the - admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, and reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries. -- Docs changes: record the API-136 through API-223 owner facade and lifecycle + admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, and reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, and reject storage module root direct ECStore/storage-api imports outside the owner-local `storage_api` boundary. +- Docs changes: record the API-136 through API-224 owner facade and lifecycle runtime-source cleanup. ## Phase 0 Tasks @@ -5245,14 +5246,33 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block migration and layer guards, diff hygiene, residual external boundary scan, Rust risk scan, fast PR gate, and full PR gate before PR. +- [x] `API-224` Move storage owner root storage aggregation into storage_api boundary. + - Do: move the storage owner root ECStore facade modules, storage contract + aliases, and wrapper functions from `rustfs/src/storage/mod.rs` into + `rustfs/src/storage/storage_api.rs`, then re-export the same owner symbols + from the storage module root. + - Acceptance: `rustfs/src/storage/mod.rs` no longer imports ECStore or + `rustfs_storage_api` directly, and migration rules reject direct storage + source bypasses outside the reviewed owner-local boundary. + - Must preserve: storage owner public crate-local symbol paths, startup + storage initialization, disk RPC helper traits, bucket metadata helpers, + replication/tier/global accessors, object IO aliases, and root/app/admin + callers. + - Verification: focused RustFS test compile coverage, formatting, migration + and layer guards, diff hygiene, residual storage owner boundary scan, Rust + risk scan, fast PR gate, and full PR gate before PR. + ## Next PRs -1. `consumer-migration`: continue larger owner/external crate storage-boundary batches after API-223. +1. `consumer-migration`: continue larger owner/external crate storage-boundary batches after API-224. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-224 moves the storage owner root ECStore facade and storage contract aggregation into the owner-local storage_api boundary. | +| Migration preservation | pass | Storage owner crate-local exports and root/app/admin callers keep the same symbols and ECStore/storage implementations. | +| Testing/verification | pass | Focused RustFS test compile coverage and migration guard have passed; formatting, layer guard, residual scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | | Quality/architecture | pass | API-223 moves the remaining external runtime/test/fuzz ECStore and storage contract imports behind local storage_api boundaries. | | Migration preservation | pass | IAM, notify, OBS metrics, Swift, S3 Select, e2e, and fuzz callers keep the same ECStore/storage implementations and behavior. | | Testing/verification | pass | Focused external crate/fuzz compile coverage, formatting, migration/layer guards, residual boundary scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | @@ -7968,6 +7988,6 @@ Notes: ## Handoff Notes - Continue with larger consumer-migration batches outside the cleaned - app/storage/admin/scanner/heal/Swift/runtime/obs/notify/S3 Select/IAM/test - and fuzz boundaries; keep ECStore-owned behavior in ECStore until concrete - behavior is isolated enough for a pure-move slice. + app/storage/admin/scanner/heal/Swift/runtime/obs/notify/S3 Select/IAM/test, + fuzz, and storage owner root boundaries; keep ECStore-owned behavior in + ECStore until concrete behavior is isolated enough for a pure-move slice. diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index c4e318ff8..16e1dcfd7 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -27,13 +27,6 @@ pub(crate) mod sse; pub mod timeout_wrapper; pub mod tonic_service; -pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject; -pub(crate) type StorageGetObjectReader = self::GetObjectReader; -pub(crate) type StorageObjectInfo = self::ObjectInfo; -pub(crate) type StorageObjectOptions = self::ObjectOptions; -pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete; -pub(crate) type StoragePutObjReader = self::PutObjReader; - #[cfg(test)] mod concurrent_fix_test; #[cfg(test)] @@ -47,6 +40,7 @@ mod multi_factor_scheduler_integration_test; pub(crate) mod runtime_sources; #[cfg(test)] mod sse_test; +mod storage_api; pub(crate) use ecfs_extend::*; pub(crate) use sse::{ @@ -54,932 +48,4 @@ pub(crate) use sse::{ sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, }; - -use std::sync::Arc; - -pub(crate) mod ecstore_admin { - pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info}; -} - -pub(crate) mod ecstore_bucket { - pub(crate) use rustfs_ecstore::api::bucket::{ - bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication, - tagging, target, utils, - }; - pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys}; -} - -pub(crate) mod ecstore_capacity { - pub(crate) use rustfs_ecstore::api::capacity::{ - PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, - is_reserved_or_invalid_bucket, - }; -} - -pub(crate) mod ecstore_client { - #[cfg(test)] - pub(crate) use rustfs_ecstore::api::client::transition_api; - pub(crate) use rustfs_ecstore::api::client::{admin_handler_utils, object_api_utils}; -} - -pub(crate) mod ecstore_compression { - pub(crate) use rustfs_ecstore::api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; -} - -pub(crate) mod ecstore_cluster { - pub(crate) use rustfs_ecstore::api::cluster::{ - ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, - ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, - ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, - topology_snapshot_from_endpoint_pools_with_capabilities, - }; -} - -pub(crate) mod ecstore_config { - pub(crate) use rustfs_ecstore::api::config::{ - com, init, init_global_config_sys, set_global_storage_class, storageclass, try_migrate_server_config, - }; -} - -pub(crate) mod ecstore_data_usage { - pub(crate) use rustfs_ecstore::api::data_usage::{ - apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory, - record_bucket_object_write_memory, remove_bucket_usage_from_backend, - }; -} - -#[allow(unused_imports)] -pub(crate) mod ecstore_disk { - pub(crate) use rustfs_ecstore::api::disk::{ - CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, - RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, - WalkDirOptions, - }; - pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce}; -} - -pub(crate) mod ecstore_error { - pub(crate) use rustfs_ecstore::api::error::{ - Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, - }; -} - -pub(crate) mod ecstore_event { - pub(crate) use rustfs_ecstore::api::event::{EventArgs, register_event_dispatch_hook}; -} - -pub(crate) mod ecstore_global { - pub(crate) use rustfs_ecstore::api::global::{ - GLOBAL_BOOT_TIME, GLOBAL_TierConfigMgr, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, - get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr, global_rustfs_port, - is_dist_erasure, new_object_layer_fn, resolve_object_store_handle, set_global_endpoints, set_global_region, - set_global_rustfs_port, set_object_store_resolver, shutdown_background_services, update_erasure_type, - }; -} - -#[allow(unused_imports)] -pub(crate) mod ecstore_layout { - pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools, Endpoints, PoolEndpoints, SetupType}; -} - -pub(crate) mod ecstore_metrics { - pub(crate) use rustfs_ecstore::api::metrics::{CollectMetricsOpts, MetricType, collect_local_metrics}; -} - -#[allow(unused_imports)] -pub(crate) mod ecstore_notification { - pub(crate) use rustfs_ecstore::api::notification::{ - NotificationSys, get_global_notification_sys, new_global_notification_sys, - }; -} - -#[allow(unused_imports)] -pub(crate) mod ecstore_rebalance { - pub(crate) use rustfs_ecstore::api::rebalance::{ - DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, - RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record, - encode_rebalance_stop_propagation_record, - }; -} - -pub(crate) mod ecstore_rio { - #[cfg(test)] - pub(crate) use rustfs_ecstore::api::rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, boxed_reader}; - pub(crate) use rustfs_ecstore::api::rio::{ - DynReader, HashReader, ReadStream, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader, - }; -} - -pub(crate) mod ecstore_rpc { - pub(crate) use rustfs_ecstore::api::rpc::{ - LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, verify_rpc_signature, - }; -} - -pub(crate) mod ecstore_set_disk { - pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class}; -} - -pub(crate) mod ecstore_storage { - pub(crate) use rustfs_ecstore::api::storage::{ - ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_lock_clients, - prewarm_local_disk_id_map, - }; -} - -pub(crate) mod ecstore_tier { - pub(crate) use rustfs_ecstore::api::tier::tier::TierConfigMgr; - #[cfg(test)] - pub(crate) use rustfs_ecstore::api::tier::warm_backend; - pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers}; -} - -pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG; -pub(crate) const BUCKET_LOGGING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LOGGING_CONFIG; -pub(crate) const BUCKET_REQUEST_PAYMENT_CONFIG: &str = ecstore_bucket::metadata::BUCKET_REQUEST_PAYMENT_CONFIG; -pub(crate) const BUCKET_TABLE_CATALOG_META_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_META_PREFIX; -pub(crate) const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = - ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX; -pub(crate) const BUCKET_TABLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_TABLE_CONFIG; -pub(crate) const BUCKET_TABLE_RESERVED_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_RESERVED_PREFIX; -pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; -pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG; -pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE; -pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; -pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL; -pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS; -pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG; -pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC; -pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET; -pub(crate) const TONIC_RPC_PREFIX: &str = ecstore_rpc::TONIC_RPC_PREFIX; -#[cfg(test)] -pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; - -pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata; -#[cfg(test)] -pub(crate) type BucketMetadataSys = ecstore_bucket::metadata_sys::BucketMetadataSys; -pub(crate) type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys; -pub(crate) type BucketBandwidthMonitor = ecstore_bucket::bandwidth::monitor::Monitor; -pub(crate) type CheckPartsResp = ecstore_disk::CheckPartsResp; -pub(crate) type CollectMetricsOpts = ecstore_metrics::CollectMetricsOpts; -pub(crate) type DailyAllTierStats = ecstore_bucket::lifecycle::tier_last_day_stats::DailyAllTierStats; -pub(crate) type DeleteOptions = ecstore_disk::DeleteOptions; -pub(crate) type DiskError = ecstore_disk::error::DiskError; -pub(crate) type DiskInfo = ecstore_disk::DiskInfo; -pub(crate) type DiskInfoOptions = ecstore_disk::DiskInfoOptions; -pub(crate) type DiskResult = ecstore_disk::error::Result; -pub(crate) type DiskStore = ecstore_disk::DiskStore; -#[cfg(test)] -pub(crate) type DisksLayout = ecstore_layout::DisksLayout; -pub(crate) type DynReplicationPool = ecstore_bucket::replication::DynReplicationPool; -pub(crate) type DynReader = ecstore_rio::DynReader; -pub(crate) type ECStore = ecstore_storage::ECStore; -pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; -#[cfg(test)] -pub(crate) type Endpoints = ecstore_layout::Endpoints; -pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; -pub(crate) type EventArgs = ecstore_event::EventArgs; -pub(crate) type ExpiryState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; -pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions; -pub(crate) type FileReader = ecstore_disk::FileReader; -pub(crate) type FileWriter = ecstore_disk::FileWriter; -pub(crate) type FS = ecfs::FS; -pub(crate) type HashReader = ecstore_rio::HashReader; -pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client; -pub(crate) type MetricType = ecstore_metrics::MetricType; -pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo; -pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; -pub(crate) type ObjectStoreResolver = dyn Fn() -> Option> + Send + Sync + 'static; -pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys; -pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; -pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError; -pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo; -pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; -pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; -pub(crate) type ReadOptions = ecstore_disk::ReadOptions; -pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; -pub(crate) type ReplicationStats = ecstore_bucket::replication::ReplicationStats; -pub(crate) type SetupType = ecstore_layout::SetupType; -pub(crate) type StorageError = ecstore_error::StorageError; -pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr; -pub(crate) type Error = ecstore_error::Error; -pub(crate) type Result = ecstore_error::Result; -pub(crate) type UpdateMetadataOpts = ecstore_disk::UpdateMetadataOpts; -pub(crate) type VolumeInfo = ecstore_disk::VolumeInfo; -pub(crate) type WalkDirOptions = ecstore_disk::WalkDirOptions; -pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption; -pub(crate) type WritePlan = ecstore_rio::WritePlan; -#[cfg(test)] -pub(crate) type DecryptReader = ecstore_rio::DecryptReader; -#[cfg(test)] -pub(crate) type EncryptReader = ecstore_rio::EncryptReader; -#[cfg(test)] -pub(crate) type HardLimitReader = ecstore_rio::HardLimitReader; -pub(crate) type NotificationSys = ecstore_notification::NotificationSys; - -pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperties { - ecstore_admin::get_local_server_property().await -} - -pub(crate) async fn init_background_replication(store: Arc) { - ecstore_bucket::replication::init_background_replication(store).await; -} - -pub(crate) async fn all_local_disk() -> Vec { - ecstore_storage::all_local_disk().await -} - -pub(crate) async fn get_bucket_notification_config(bucket: &str) -> Result> { - ecstore_bucket::metadata_sys::get_notification_config(bucket).await -} - -pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { - ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; -} - -pub(crate) fn bucket_metadata_runtime_initialized() -> bool { - ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys().is_some() -} - -pub(crate) fn disk_drive_path(disk: &DiskStore) -> String { - ecstore_disk::DiskAPI::to_string(disk.as_ref()) -} - -pub(crate) fn disk_endpoint(disk: &DiskStore) -> String { - ecstore_disk::DiskAPI::endpoint(disk.as_ref()).to_string() -} - -pub(crate) fn get_global_replication_pool() -> Option> { - ecstore_bucket::replication::get_global_replication_pool() -} - -pub(crate) fn get_global_replication_stats() -> Option> { - ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().cloned() -} - -pub(crate) fn get_global_boot_time() -> Option { - ecstore_global::GLOBAL_BOOT_TIME.get().cloned() -} - -pub(crate) fn get_daily_all_tier_stats() -> DailyAllTierStats { - ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_transition_state().get_daily_all_tier_stats() -} - -pub(crate) fn get_global_expiry_state() -> Arc> { - ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_expiry_state() -} - -pub(crate) async fn try_migrate_bucket_metadata(store: Arc) { - ecstore_bucket::migration::try_migrate_bucket_metadata(store).await; -} - -pub(crate) async fn try_migrate_iam_config(store: Arc) { - ecstore_bucket::migration::try_migrate_iam_config(store).await; -} - -pub(crate) fn init_ecstore_config() { - ecstore_config::init(); -} - -pub(crate) async fn init_global_config_sys(store: Arc) -> Result<()> { - ecstore_config::init_global_config_sys(store).await -} - -pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> { - ecstore_storage::init_local_disks(endpoint_pools).await -} - -pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) { - ecstore_storage::init_lock_clients(endpoint_pools); -} - -pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPools) -> Result<()> { - ecstore_notification::new_global_notification_sys(endpoint_pools).await -} - -pub(crate) async fn read_config(api: Arc, file: &str) -> Result> { - ecstore_config::com::read_config(api, file).await -} - -pub(crate) async fn prewarm_local_disk_id_map() { - ecstore_storage::prewarm_local_disk_id_map().await; -} - -pub(crate) fn replication_queue_current_count() -> Option { - ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| { - stats - .q_cache - .try_lock() - .ok() - .map(|cache| cache.sr_queue_stats.curr.get_current_count()) - }) -} - -pub(crate) async fn save_config(api: Arc, file: &str, data: Vec) -> Result<()> { - ecstore_config::com::save_config(api, file, data).await -} - -pub(crate) fn shutdown_background_services() { - ecstore_global::shutdown_background_services(); -} - -pub(crate) fn set_global_endpoints(endpoints: Vec) { - ecstore_global::set_global_endpoints(endpoints); -} - -pub(crate) fn set_global_region(region: s3s::region::Region) { - ecstore_global::set_global_region(region); -} - -pub(crate) fn set_global_rustfs_port(value: u16) { - ecstore_global::set_global_rustfs_port(value); -} - -pub(crate) async fn try_migrate_server_config(store: Arc) { - ecstore_config::try_migrate_server_config(store).await; -} - -pub(crate) async fn update_erasure_type(setup_type: SetupType) { - ecstore_global::update_erasure_type(setup_type).await; -} - -pub(crate) trait StorageDiskRpcExt { - async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult; - async fn delete_volume(&self, volume: &str) -> DiskResult<()>; - async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult>; - async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) - -> Vec>; - async fn delete_version( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> DiskResult<()>; - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult; - async fn read_version( - &self, - org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> DiskResult; - async fn write_metadata( - &self, - org_volume: &str, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - ) -> DiskResult<()>; - async fn update_metadata( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - opts: &UpdateMetadataOpts, - ) -> DiskResult<()>; - async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; - async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; - async fn stat_volume(&self, volume: &str) -> DiskResult; - async fn list_volumes(&self) -> DiskResult>; - async fn make_volume(&self, volume: &str) -> DiskResult<()>; - async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()>; - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - file_info: rustfs_filemeta::FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> DiskResult; - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult>; - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult; - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>; - async fn rename_part( - &self, - src_volume: &str, - src_path: &str, - dst_volume: &str, - dst_path: &str, - meta: bytes::Bytes, - ) -> DiskResult<()>; - async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; - async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; - async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; - async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult>; - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()>; - async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()>; - async fn read_all(&self, volume: &str, path: &str) -> DiskResult; - async fn append_file(&self, volume: &str, path: &str) -> DiskResult; - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult; -} - -impl StorageDiskRpcExt for T -where - T: ecstore_disk::DiskAPI, -{ - async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult { - ecstore_disk::DiskAPI::disk_info(self, opts).await - } - - async fn delete_volume(&self, volume: &str) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete_volume(self, volume).await - } - - async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult> { - ecstore_disk::DiskAPI::read_multiple(self, req).await - } - - async fn delete_versions( - &self, - volume: &str, - versions: Vec, - opts: DeleteOptions, - ) -> Vec> { - ecstore_disk::DiskAPI::delete_versions(self, volume, versions, opts).await - } - - async fn delete_version( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - force_del_marker: bool, - opts: DeleteOptions, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete_version(self, volume, path, file_info, force_del_marker, opts).await - } - - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult { - ecstore_disk::DiskAPI::read_xl(self, volume, path, read_data).await - } - - async fn read_version( - &self, - org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> DiskResult { - ecstore_disk::DiskAPI::read_version(self, org_volume, volume, path, version_id, opts).await - } - - async fn write_metadata( - &self, - org_volume: &str, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::write_metadata(self, org_volume, volume, path, file_info).await - } - - async fn update_metadata( - &self, - volume: &str, - path: &str, - file_info: rustfs_filemeta::FileInfo, - opts: &UpdateMetadataOpts, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::update_metadata(self, volume, path, file_info, opts).await - } - - async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult { - ecstore_disk::DiskAPI::read_metadata(self, volume, path).await - } - - async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await - } - - async fn stat_volume(&self, volume: &str) -> DiskResult { - ecstore_disk::DiskAPI::stat_volume(self, volume).await - } - - async fn list_volumes(&self) -> DiskResult> { - ecstore_disk::DiskAPI::list_volumes(self).await - } - - async fn make_volume(&self, volume: &str) -> DiskResult<()> { - ecstore_disk::DiskAPI::make_volume(self, volume).await - } - - async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()> { - ecstore_disk::DiskAPI::make_volumes(self, volume).await - } - - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - file_info: rustfs_filemeta::FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> DiskResult { - ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await - } - - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult> { - ecstore_disk::DiskAPI::list_dir(self, origvolume, volume, dir_path, count).await - } - - async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult { - ecstore_disk::DiskAPI::read_file_stream(self, volume, path, offset, length).await - } - - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> { - ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await - } - - async fn rename_part( - &self, - src_volume: &str, - src_path: &str, - dst_volume: &str, - dst_path: &str, - meta: bytes::Bytes, - ) -> DiskResult<()> { - ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await - } - - async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> { - ecstore_disk::DiskAPI::delete(self, volume, path, options).await - } - - async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { - ecstore_disk::DiskAPI::verify_file(self, volume, path, file_info).await - } - - async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { - ecstore_disk::DiskAPI::check_parts(self, volume, path, file_info).await - } - - async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult> { - ecstore_disk::DiskAPI::read_parts(self, bucket, paths).await - } - - async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()> { - ecstore_disk::DiskAPI::walk_dir(self, opts, wr).await - } - - async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> { - ecstore_disk::DiskAPI::write_all(self, volume, path, data).await - } - - async fn read_all(&self, volume: &str, path: &str) -> DiskResult { - ecstore_disk::DiskAPI::read_all(self, volume, path).await - } - - async fn append_file(&self, volume: &str, path: &str) -> DiskResult { - ecstore_disk::DiskAPI::append_file(self, volume, path).await - } - - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult { - ecstore_disk::DiskAPI::create_file(self, origvolume, volume, path, file_size).await - } -} - -pub(crate) trait StoragePeerS3ClientExt { - async fn heal_bucket( - &self, - bucket: &str, - opts: &rustfs_common::heal_channel::HealOpts, - ) -> DiskResult; - async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()>; - async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult>; - async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()>; - async fn get_bucket_info( - &self, - bucket: &str, - opts: &rustfs_storage_api::BucketOptions, - ) -> DiskResult; -} - -impl StoragePeerS3ClientExt for LocalPeerS3Client { - async fn heal_bucket( - &self, - bucket: &str, - opts: &rustfs_common::heal_channel::HealOpts, - ) -> DiskResult { - ecstore_rpc::PeerS3Client::heal_bucket(self, bucket, opts).await - } - - async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()> { - ecstore_rpc::PeerS3Client::make_bucket(self, bucket, opts).await - } - - async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult> { - ecstore_rpc::PeerS3Client::list_bucket(self, opts).await - } - - async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()> { - ecstore_rpc::PeerS3Client::delete_bucket(self, bucket, opts).await - } - - async fn get_bucket_info( - &self, - bucket: &str, - opts: &rustfs_storage_api::BucketOptions, - ) -> DiskResult { - ecstore_rpc::PeerS3Client::get_bucket_info(self, bucket, opts).await - } -} - -pub(crate) async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { - ecstore_bucket::metadata::load_bucket_metadata(api, bucket).await -} - -#[cfg(test)] -pub(crate) fn bucket_metadata_sys_initialized() -> bool { - ecstore_bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_some() -} - -#[cfg(test)] -pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { - ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() -} - -pub(crate) async fn delete_bucket_metadata_config(bucket: &str, config_file: &str) -> Result { - ecstore_bucket::metadata_sys::delete(bucket, config_file).await -} - -pub(crate) async fn get_bucket_metadata(bucket: &str) -> Result> { - ecstore_bucket::metadata_sys::get(bucket).await -} - -pub(crate) async fn get_bucket_accelerate_config( - bucket: &str, -) -> Result<(s3s::dto::AccelerateConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_accelerate_config(bucket).await -} - -pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await -} - -pub(crate) async fn get_bucket_cors_config(bucket: &str) -> Result<(s3s::dto::CORSConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_cors_config(bucket).await -} - -pub(crate) async fn get_bucket_logging_config(bucket: &str) -> Result<(s3s::dto::BucketLoggingStatus, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_logging_config(bucket).await -} - -pub(crate) async fn get_bucket_object_lock_config( - bucket: &str, -) -> Result<(s3s::dto::ObjectLockConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await -} - -pub(crate) async fn get_public_access_block_config( - bucket: &str, -) -> Result<(s3s::dto::PublicAccessBlockConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await -} - -pub(crate) async fn get_bucket_replication_config( - bucket: &str, -) -> Result<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_replication_config(bucket).await -} - -pub(crate) async fn get_bucket_request_payment_config( - bucket: &str, -) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_request_payment_config(bucket).await -} - -pub(crate) async fn get_bucket_sse_config( - bucket: &str, -) -> Result<(s3s::dto::ServerSideEncryptionConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_sse_config(bucket).await -} - -pub(crate) async fn get_bucket_website_config(bucket: &str) -> Result<(s3s::dto::WebsiteConfiguration, time::OffsetDateTime)> { - ecstore_bucket::metadata_sys::get_website_config(bucket).await -} - -pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> { - ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, bm).await -} - -pub(crate) async fn update_bucket_metadata_config( - bucket: &str, - config_file: &str, - data: Vec, -) -> Result { - ecstore_bucket::metadata_sys::update(bucket, config_file, data).await -} - -pub(crate) fn add_object_lock_years(dt: time::OffsetDateTime, years: i32) -> time::OffsetDateTime { - ecstore_bucket::object_lock::objectlock_sys::add_years(dt, years) -} - -pub(crate) fn check_retention_for_modification( - user_defined: &std::collections::HashMap, - new_mode: Option<&str>, - new_retain_until: Option, - bypass_governance: bool, -) -> Option { - ecstore_bucket::object_lock::objectlock_sys::check_retention_for_modification( - user_defined, - new_mode, - new_retain_until, - bypass_governance, - ) -} - -pub(crate) async fn record_replication_proxy(bucket: &str, api: &str, is_err: bool) { - if let Some(stats) = ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() { - stats.inc_proxy(bucket, api, is_err).await; - } -} - -pub(crate) fn decode_tags(tags: &str) -> Vec { - ecstore_bucket::tagging::decode_tags(tags) -} - -pub(crate) fn decode_tags_to_map(tags: &str) -> std::collections::HashMap { - ecstore_bucket::tagging::decode_tags_to_map(tags) -} - -pub(crate) fn encode_tags(tags: Vec) -> String { - ecstore_bucket::tagging::encode_tags(tags) -} - -pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { - ecstore_bucket::utils::serialize(val) -} - -pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { - ecstore_error::is_err_bucket_not_found(err) -} - -pub(crate) fn is_err_object_not_found(err: &Error) -> bool { - ecstore_error::is_err_object_not_found(err) -} - -pub(crate) fn is_err_version_not_found(err: &Error) -> bool { - ecstore_error::is_err_version_not_found(err) -} - -pub(crate) fn is_all_buckets_not_found(errs: &[Option]) -> bool { - ecstore_disk::error_reduce::is_all_buckets_not_found(errs) -} - -pub(crate) fn get_global_lock_client() -> Option> { - ecstore_global::get_global_lock_client() -} - -pub(crate) fn get_global_lock_clients() --> Option<&'static std::collections::HashMap>> { - ecstore_global::get_global_lock_clients() -} - -pub(crate) fn get_global_bucket_monitor() -> Option> { - ecstore_global::get_global_bucket_monitor() -} - -pub(crate) fn get_global_endpoints_opt() -> Option { - ecstore_global::get_global_endpoints_opt() -} - -pub(crate) fn get_global_deployment_id() -> Option { - ecstore_global::get_global_deployment_id() -} - -pub(crate) fn get_global_region() -> Option { - ecstore_global::get_global_region() -} - -pub(crate) fn global_rustfs_port() -> u16 { - ecstore_global::global_rustfs_port() -} - -pub(crate) fn get_global_tier_config_mgr() -> Arc> { - ecstore_global::get_global_tier_config_mgr() -} - -pub(crate) fn new_object_layer_fn() -> Option> { - ecstore_global::new_object_layer_fn() -} - -pub(crate) fn set_object_store_resolver(resolver: Arc) -> bool { - ecstore_global::set_object_store_resolver(resolver) -} - -pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { - ecstore_notification::get_global_notification_sys() -} - -pub(crate) async fn is_dist_erasure() -> bool { - ecstore_global::is_dist_erasure().await -} - -pub(crate) fn resolve_object_store_handle() -> Option> { - ecstore_global::resolve_object_store_handle() -} - -pub(crate) async fn collect_local_metrics( - types: MetricType, - opts: &CollectMetricsOpts, -) -> rustfs_madmin::metrics::RealtimeMetrics { - ecstore_metrics::collect_local_metrics(types, opts).await -} - -pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &http::HeaderMap) -> std::io::Result<()> { - ecstore_rpc::verify_rpc_signature(url, method, headers) -} - -pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { - ecstore_client::object_api_utils::to_s3s_etag(etag) -} - -pub(crate) fn table_catalog_path_hash(value: &str) -> String { - ecstore_bucket::metadata::table_catalog_path_hash(value) -} - -pub(crate) fn get_lock_acquire_timeout() -> std::time::Duration { - ecstore_set_disk::get_lock_acquire_timeout() -} - -#[cfg(test)] -pub(crate) fn boxed_reader(reader: R) -> DynReader -where - R: ecstore_rio::Reader + 'static, -{ - ecstore_rio::boxed_reader(reader) -} - -pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String { - ecstore_rio::compression_metadata_value(algorithm) -} - -pub(crate) fn wrap_reader(reader: R) -> DynReader -where - R: ecstore_rio::ReadStream + 'static, -{ - ecstore_rio::wrap_reader(reader) -} - -pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool { - ecstore_set_disk::is_valid_storage_class(storage_class) -} - -pub(crate) fn register_event_dispatch_hook(hook: F) -> bool -where - F: Fn(EventArgs) + Send + Sync + 'static, -{ - ecstore_event::register_event_dispatch_hook(hook) -} - -pub(crate) fn topology_snapshot_from_endpoint_pools_with_capabilities( - endpoint_pools: &EndpointServerPools, - capabilities: rustfs_storage_api::TopologyCapabilities, - disk_capabilities: rustfs_storage_api::DiskCapabilities, -) -> rustfs_storage_api::TopologySnapshot { - ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(endpoint_pools, capabilities, disk_capabilities) -} - -pub(crate) async fn reload_transition_tier_config(api: Arc) -> std::io::Result<()> { - ecstore_global::GLOBAL_TierConfigMgr.write().await.reload(api).await -} - -pub(crate) async fn all_local_disk_path() -> Vec { - ecstore_storage::all_local_disk_path().await -} - -pub(crate) async fn find_local_disk_by_ref(disk_ref: &str) -> Option { - ecstore_storage::find_local_disk_by_ref(disk_ref).await -} - -pub(crate) trait StorageReplicationConfigExt { - fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool; -} - -impl StorageReplicationConfigExt for s3s::dto::ReplicationConfiguration { - fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool { - ::has_active_rules( - self, prefix, recursive, - ) - } -} - -pub(crate) trait StorageVersioningConfigExt { - fn enabled(&self) -> bool; -} - -impl StorageVersioningConfigExt for s3s::dto::VersioningConfiguration { - fn enabled(&self) -> bool { - ::enabled(self) - } -} - -pub(crate) type GetObjectReader = ::GetObjectReader; -pub(crate) type ObjectInfo = ::ObjectInfo; -pub(crate) type ObjectOptions = ::ObjectOptions; -pub(crate) type PutObjReader = ::PutObjectReader; +pub(crate) use storage_api::*; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs new file mode 100644 index 000000000..3747a4704 --- /dev/null +++ b/rustfs/src/storage/storage_api.rs @@ -0,0 +1,951 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage owner-local boundary for ECStore facade and storage contract symbols. + +use std::sync::Arc; + +pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject; +pub(crate) type StorageGetObjectReader = super::GetObjectReader; +pub(crate) type StorageObjectInfo = super::ObjectInfo; +pub(crate) type StorageObjectOptions = super::ObjectOptions; +pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete; +pub(crate) type StoragePutObjReader = super::PutObjReader; + +pub(crate) mod ecstore_admin { + pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info}; +} + +pub(crate) mod ecstore_bucket { + pub(crate) use rustfs_ecstore::api::bucket::{ + bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication, + tagging, target, utils, + }; + pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys}; +} + +pub(crate) mod ecstore_capacity { + pub(crate) use rustfs_ecstore::api::capacity::{ + PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, + is_reserved_or_invalid_bucket, + }; +} + +pub(crate) mod ecstore_client { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::client::transition_api; + pub(crate) use rustfs_ecstore::api::client::{admin_handler_utils, object_api_utils}; +} + +pub(crate) mod ecstore_compression { + pub(crate) use rustfs_ecstore::api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; +} + +pub(crate) mod ecstore_cluster { + pub(crate) use rustfs_ecstore::api::cluster::{ + ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, + ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, + ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, + topology_snapshot_from_endpoint_pools_with_capabilities, + }; +} + +pub(crate) mod ecstore_config { + pub(crate) use rustfs_ecstore::api::config::{ + com, init, init_global_config_sys, set_global_storage_class, storageclass, try_migrate_server_config, + }; +} + +pub(crate) mod ecstore_data_usage { + pub(crate) use rustfs_ecstore::api::data_usage::{ + apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory, + record_bucket_object_write_memory, remove_bucket_usage_from_backend, + }; +} + +#[allow(unused_imports)] +pub(crate) mod ecstore_disk { + pub(crate) use rustfs_ecstore::api::disk::{ + CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter, + RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, + WalkDirOptions, + }; + pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce}; +} + +pub(crate) mod ecstore_error { + pub(crate) use rustfs_ecstore::api::error::{ + Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, + }; +} + +pub(crate) mod ecstore_event { + pub(crate) use rustfs_ecstore::api::event::{EventArgs, register_event_dispatch_hook}; +} + +pub(crate) mod ecstore_global { + pub(crate) use rustfs_ecstore::api::global::{ + GLOBAL_BOOT_TIME, GLOBAL_TierConfigMgr, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, + get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr, global_rustfs_port, + is_dist_erasure, new_object_layer_fn, resolve_object_store_handle, set_global_endpoints, set_global_region, + set_global_rustfs_port, set_object_store_resolver, shutdown_background_services, update_erasure_type, + }; +} + +#[allow(unused_imports)] +pub(crate) mod ecstore_layout { + pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools, Endpoints, PoolEndpoints, SetupType}; +} + +pub(crate) mod ecstore_metrics { + pub(crate) use rustfs_ecstore::api::metrics::{CollectMetricsOpts, MetricType, collect_local_metrics}; +} + +#[allow(unused_imports)] +pub(crate) mod ecstore_notification { + pub(crate) use rustfs_ecstore::api::notification::{ + NotificationSys, get_global_notification_sys, new_global_notification_sys, + }; +} + +#[allow(unused_imports)] +pub(crate) mod ecstore_rebalance { + pub(crate) use rustfs_ecstore::api::rebalance::{ + DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, + RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record, + encode_rebalance_stop_propagation_record, + }; +} + +pub(crate) mod ecstore_rio { + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, boxed_reader}; + pub(crate) use rustfs_ecstore::api::rio::{ + DynReader, HashReader, ReadStream, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader, + }; +} + +pub(crate) mod ecstore_rpc { + pub(crate) use rustfs_ecstore::api::rpc::{ + LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, + SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, verify_rpc_signature, + }; +} + +pub(crate) mod ecstore_set_disk { + pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class}; +} + +pub(crate) mod ecstore_storage { + pub(crate) use rustfs_ecstore::api::storage::{ + ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_lock_clients, + prewarm_local_disk_id_map, + }; +} + +pub(crate) mod ecstore_tier { + pub(crate) use rustfs_ecstore::api::tier::tier::TierConfigMgr; + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::tier::warm_backend; + pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers}; +} + +pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG; +pub(crate) const BUCKET_LOGGING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LOGGING_CONFIG; +pub(crate) const BUCKET_REQUEST_PAYMENT_CONFIG: &str = ecstore_bucket::metadata::BUCKET_REQUEST_PAYMENT_CONFIG; +pub(crate) const BUCKET_TABLE_CATALOG_META_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_META_PREFIX; +pub(crate) const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = + ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX; +pub(crate) const BUCKET_TABLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_TABLE_CONFIG; +pub(crate) const BUCKET_TABLE_RESERVED_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_RESERVED_PREFIX; +pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; +pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG; +pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE; +pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; +pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL; +pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS; +pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG; +pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC; +pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET; +pub(crate) const TONIC_RPC_PREFIX: &str = ecstore_rpc::TONIC_RPC_PREFIX; +#[cfg(test)] +pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; + +pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata; +#[cfg(test)] +pub(crate) type BucketMetadataSys = ecstore_bucket::metadata_sys::BucketMetadataSys; +pub(crate) type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys; +pub(crate) type BucketBandwidthMonitor = ecstore_bucket::bandwidth::monitor::Monitor; +pub(crate) type CheckPartsResp = ecstore_disk::CheckPartsResp; +pub(crate) type CollectMetricsOpts = ecstore_metrics::CollectMetricsOpts; +pub(crate) type DailyAllTierStats = ecstore_bucket::lifecycle::tier_last_day_stats::DailyAllTierStats; +pub(crate) type DeleteOptions = ecstore_disk::DeleteOptions; +pub(crate) type DiskError = ecstore_disk::error::DiskError; +pub(crate) type DiskInfo = ecstore_disk::DiskInfo; +pub(crate) type DiskInfoOptions = ecstore_disk::DiskInfoOptions; +pub(crate) type DiskResult = ecstore_disk::error::Result; +pub(crate) type DiskStore = ecstore_disk::DiskStore; +#[cfg(test)] +pub(crate) type DisksLayout = ecstore_layout::DisksLayout; +pub(crate) type DynReplicationPool = ecstore_bucket::replication::DynReplicationPool; +pub(crate) type DynReader = ecstore_rio::DynReader; +pub(crate) type ECStore = ecstore_storage::ECStore; +pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; +#[cfg(test)] +pub(crate) type Endpoints = ecstore_layout::Endpoints; +pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; +pub(crate) type EventArgs = ecstore_event::EventArgs; +pub(crate) type ExpiryState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; +pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions; +pub(crate) type FileReader = ecstore_disk::FileReader; +pub(crate) type FileWriter = ecstore_disk::FileWriter; +pub(crate) type FS = super::ecfs::FS; +pub(crate) type HashReader = ecstore_rio::HashReader; +pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client; +pub(crate) type MetricType = ecstore_metrics::MetricType; +pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo; +pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; +pub(crate) type ObjectStoreResolver = dyn Fn() -> Option> + Send + Sync + 'static; +pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys; +pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints; +pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError; +pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo; +pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq; +pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp; +pub(crate) type ReadOptions = ecstore_disk::ReadOptions; +pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp; +pub(crate) type ReplicationStats = ecstore_bucket::replication::ReplicationStats; +pub(crate) type SetupType = ecstore_layout::SetupType; +pub(crate) type StorageError = ecstore_error::StorageError; +pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr; +pub(crate) type Error = ecstore_error::Error; +pub(crate) type Result = ecstore_error::Result; +pub(crate) type UpdateMetadataOpts = ecstore_disk::UpdateMetadataOpts; +pub(crate) type VolumeInfo = ecstore_disk::VolumeInfo; +pub(crate) type WalkDirOptions = ecstore_disk::WalkDirOptions; +pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption; +pub(crate) type WritePlan = ecstore_rio::WritePlan; +#[cfg(test)] +pub(crate) type DecryptReader = ecstore_rio::DecryptReader; +#[cfg(test)] +pub(crate) type EncryptReader = ecstore_rio::EncryptReader; +#[cfg(test)] +pub(crate) type HardLimitReader = ecstore_rio::HardLimitReader; +pub(crate) type NotificationSys = ecstore_notification::NotificationSys; + +pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperties { + ecstore_admin::get_local_server_property().await +} + +pub(crate) async fn init_background_replication(store: Arc) { + ecstore_bucket::replication::init_background_replication(store).await; +} + +pub(crate) async fn all_local_disk() -> Vec { + ecstore_storage::all_local_disk().await +} + +pub(crate) async fn get_bucket_notification_config(bucket: &str) -> Result> { + ecstore_bucket::metadata_sys::get_notification_config(bucket).await +} + +pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { + ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; +} + +pub(crate) fn bucket_metadata_runtime_initialized() -> bool { + ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys().is_some() +} + +pub(crate) fn disk_drive_path(disk: &DiskStore) -> String { + ecstore_disk::DiskAPI::to_string(disk.as_ref()) +} + +pub(crate) fn disk_endpoint(disk: &DiskStore) -> String { + ecstore_disk::DiskAPI::endpoint(disk.as_ref()).to_string() +} + +pub(crate) fn get_global_replication_pool() -> Option> { + ecstore_bucket::replication::get_global_replication_pool() +} + +pub(crate) fn get_global_replication_stats() -> Option> { + ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().cloned() +} + +pub(crate) fn get_global_boot_time() -> Option { + ecstore_global::GLOBAL_BOOT_TIME.get().cloned() +} + +pub(crate) fn get_daily_all_tier_stats() -> DailyAllTierStats { + ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_transition_state().get_daily_all_tier_stats() +} + +pub(crate) fn get_global_expiry_state() -> Arc> { + ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_expiry_state() +} + +pub(crate) async fn try_migrate_bucket_metadata(store: Arc) { + ecstore_bucket::migration::try_migrate_bucket_metadata(store).await; +} + +pub(crate) async fn try_migrate_iam_config(store: Arc) { + ecstore_bucket::migration::try_migrate_iam_config(store).await; +} + +pub(crate) fn init_ecstore_config() { + ecstore_config::init(); +} + +pub(crate) async fn init_global_config_sys(store: Arc) -> Result<()> { + ecstore_config::init_global_config_sys(store).await +} + +pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> { + ecstore_storage::init_local_disks(endpoint_pools).await +} + +pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) { + ecstore_storage::init_lock_clients(endpoint_pools); +} + +pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPools) -> Result<()> { + ecstore_notification::new_global_notification_sys(endpoint_pools).await +} + +pub(crate) async fn read_config(api: Arc, file: &str) -> Result> { + ecstore_config::com::read_config(api, file).await +} + +pub(crate) async fn prewarm_local_disk_id_map() { + ecstore_storage::prewarm_local_disk_id_map().await; +} + +pub(crate) fn replication_queue_current_count() -> Option { + ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| { + stats + .q_cache + .try_lock() + .ok() + .map(|cache| cache.sr_queue_stats.curr.get_current_count()) + }) +} + +pub(crate) async fn save_config(api: Arc, file: &str, data: Vec) -> Result<()> { + ecstore_config::com::save_config(api, file, data).await +} + +pub(crate) fn shutdown_background_services() { + ecstore_global::shutdown_background_services(); +} + +pub(crate) fn set_global_endpoints(endpoints: Vec) { + ecstore_global::set_global_endpoints(endpoints); +} + +pub(crate) fn set_global_region(region: s3s::region::Region) { + ecstore_global::set_global_region(region); +} + +pub(crate) fn set_global_rustfs_port(value: u16) { + ecstore_global::set_global_rustfs_port(value); +} + +pub(crate) async fn try_migrate_server_config(store: Arc) { + ecstore_config::try_migrate_server_config(store).await; +} + +pub(crate) async fn update_erasure_type(setup_type: SetupType) { + ecstore_global::update_erasure_type(setup_type).await; +} + +pub(crate) trait StorageDiskRpcExt { + async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult; + async fn delete_volume(&self, volume: &str) -> DiskResult<()>; + async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult>; + async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) + -> Vec>; + async fn delete_version( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()>; + async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult; + async fn read_version( + &self, + org_volume: &str, + volume: &str, + path: &str, + version_id: &str, + opts: &ReadOptions, + ) -> DiskResult; + async fn write_metadata( + &self, + org_volume: &str, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + ) -> DiskResult<()>; + async fn update_metadata( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + opts: &UpdateMetadataOpts, + ) -> DiskResult<()>; + async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult; + async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()>; + async fn stat_volume(&self, volume: &str) -> DiskResult; + async fn list_volumes(&self) -> DiskResult>; + async fn make_volume(&self, volume: &str) -> DiskResult<()>; + async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()>; + async fn rename_data( + &self, + src_volume: &str, + src_path: &str, + file_info: rustfs_filemeta::FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> DiskResult; + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult>; + async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult; + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()>; + async fn rename_part( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: bytes::Bytes, + ) -> DiskResult<()>; + async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()>; + async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; + async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult; + async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult>; + async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()>; + async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()>; + async fn read_all(&self, volume: &str, path: &str) -> DiskResult; + async fn append_file(&self, volume: &str, path: &str) -> DiskResult; + async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult; +} + +impl StorageDiskRpcExt for T +where + T: ecstore_disk::DiskAPI, +{ + async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult { + ecstore_disk::DiskAPI::disk_info(self, opts).await + } + + async fn delete_volume(&self, volume: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_volume(self, volume).await + } + + async fn read_multiple(&self, req: ReadMultipleReq) -> DiskResult> { + ecstore_disk::DiskAPI::read_multiple(self, req).await + } + + async fn delete_versions( + &self, + volume: &str, + versions: Vec, + opts: DeleteOptions, + ) -> Vec> { + ecstore_disk::DiskAPI::delete_versions(self, volume, versions, opts).await + } + + async fn delete_version( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_version(self, volume, path, file_info, force_del_marker, opts).await + } + + async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> DiskResult { + ecstore_disk::DiskAPI::read_xl(self, volume, path, read_data).await + } + + async fn read_version( + &self, + org_volume: &str, + volume: &str, + path: &str, + version_id: &str, + opts: &ReadOptions, + ) -> DiskResult { + ecstore_disk::DiskAPI::read_version(self, org_volume, volume, path, version_id, opts).await + } + + async fn write_metadata( + &self, + org_volume: &str, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::write_metadata(self, org_volume, volume, path, file_info).await + } + + async fn update_metadata( + &self, + volume: &str, + path: &str, + file_info: rustfs_filemeta::FileInfo, + opts: &UpdateMetadataOpts, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::update_metadata(self, volume, path, file_info, opts).await + } + + async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::read_metadata(self, volume, path).await + } + + async fn delete_paths(&self, volume: &str, paths: &[String]) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete_paths(self, volume, paths).await + } + + async fn stat_volume(&self, volume: &str) -> DiskResult { + ecstore_disk::DiskAPI::stat_volume(self, volume).await + } + + async fn list_volumes(&self) -> DiskResult> { + ecstore_disk::DiskAPI::list_volumes(self).await + } + + async fn make_volume(&self, volume: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::make_volume(self, volume).await + } + + async fn make_volumes(&self, volume: Vec<&str>) -> DiskResult<()> { + ecstore_disk::DiskAPI::make_volumes(self, volume).await + } + + async fn rename_data( + &self, + src_volume: &str, + src_path: &str, + file_info: rustfs_filemeta::FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> DiskResult { + ecstore_disk::DiskAPI::rename_data(self, src_volume, src_path, file_info, dst_volume, dst_path).await + } + + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> DiskResult> { + ecstore_disk::DiskAPI::list_dir(self, origvolume, volume, dir_path, count).await + } + + async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> DiskResult { + ecstore_disk::DiskAPI::read_file_stream(self, volume, path, offset, length).await + } + + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> DiskResult<()> { + ecstore_disk::DiskAPI::rename_file(self, src_volume, src_path, dst_volume, dst_path).await + } + + async fn rename_part( + &self, + src_volume: &str, + src_path: &str, + dst_volume: &str, + dst_path: &str, + meta: bytes::Bytes, + ) -> DiskResult<()> { + ecstore_disk::DiskAPI::rename_part(self, src_volume, src_path, dst_volume, dst_path, meta).await + } + + async fn delete(&self, volume: &str, path: &str, options: DeleteOptions) -> DiskResult<()> { + ecstore_disk::DiskAPI::delete(self, volume, path, options).await + } + + async fn verify_file(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { + ecstore_disk::DiskAPI::verify_file(self, volume, path, file_info).await + } + + async fn check_parts(&self, volume: &str, path: &str, file_info: &rustfs_filemeta::FileInfo) -> DiskResult { + ecstore_disk::DiskAPI::check_parts(self, volume, path, file_info).await + } + + async fn read_parts(&self, bucket: &str, paths: &[String]) -> DiskResult> { + ecstore_disk::DiskAPI::read_parts(self, bucket, paths).await + } + + async fn walk_dir(&self, opts: WalkDirOptions, wr: &mut W) -> DiskResult<()> { + ecstore_disk::DiskAPI::walk_dir(self, opts, wr).await + } + + async fn write_all(&self, volume: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> { + ecstore_disk::DiskAPI::write_all(self, volume, path, data).await + } + + async fn read_all(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::read_all(self, volume, path).await + } + + async fn append_file(&self, volume: &str, path: &str) -> DiskResult { + ecstore_disk::DiskAPI::append_file(self, volume, path).await + } + + async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: i64) -> DiskResult { + ecstore_disk::DiskAPI::create_file(self, origvolume, volume, path, file_size).await + } +} + +pub(crate) trait StoragePeerS3ClientExt { + async fn heal_bucket( + &self, + bucket: &str, + opts: &rustfs_common::heal_channel::HealOpts, + ) -> DiskResult; + async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()>; + async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult>; + async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()>; + async fn get_bucket_info( + &self, + bucket: &str, + opts: &rustfs_storage_api::BucketOptions, + ) -> DiskResult; +} + +impl StoragePeerS3ClientExt for LocalPeerS3Client { + async fn heal_bucket( + &self, + bucket: &str, + opts: &rustfs_common::heal_channel::HealOpts, + ) -> DiskResult { + ecstore_rpc::PeerS3Client::heal_bucket(self, bucket, opts).await + } + + async fn make_bucket(&self, bucket: &str, opts: &rustfs_storage_api::MakeBucketOptions) -> DiskResult<()> { + ecstore_rpc::PeerS3Client::make_bucket(self, bucket, opts).await + } + + async fn list_bucket(&self, opts: &rustfs_storage_api::BucketOptions) -> DiskResult> { + ecstore_rpc::PeerS3Client::list_bucket(self, opts).await + } + + async fn delete_bucket(&self, bucket: &str, opts: &rustfs_storage_api::DeleteBucketOptions) -> DiskResult<()> { + ecstore_rpc::PeerS3Client::delete_bucket(self, bucket, opts).await + } + + async fn get_bucket_info( + &self, + bucket: &str, + opts: &rustfs_storage_api::BucketOptions, + ) -> DiskResult { + ecstore_rpc::PeerS3Client::get_bucket_info(self, bucket, opts).await + } +} + +pub(crate) async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { + ecstore_bucket::metadata::load_bucket_metadata(api, bucket).await +} + +#[cfg(test)] +pub(crate) fn bucket_metadata_sys_initialized() -> bool { + ecstore_bucket::metadata_sys::GLOBAL_BucketMetadataSys.get().is_some() +} + +#[cfg(test)] +pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { + ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() +} + +pub(crate) async fn delete_bucket_metadata_config(bucket: &str, config_file: &str) -> Result { + ecstore_bucket::metadata_sys::delete(bucket, config_file).await +} + +pub(crate) async fn get_bucket_metadata(bucket: &str) -> Result> { + ecstore_bucket::metadata_sys::get(bucket).await +} + +pub(crate) async fn get_bucket_accelerate_config( + bucket: &str, +) -> Result<(s3s::dto::AccelerateConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_accelerate_config(bucket).await +} + +pub(crate) async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await +} + +pub(crate) async fn get_bucket_cors_config(bucket: &str) -> Result<(s3s::dto::CORSConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_cors_config(bucket).await +} + +pub(crate) async fn get_bucket_logging_config(bucket: &str) -> Result<(s3s::dto::BucketLoggingStatus, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_logging_config(bucket).await +} + +pub(crate) async fn get_bucket_object_lock_config( + bucket: &str, +) -> Result<(s3s::dto::ObjectLockConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await +} + +pub(crate) async fn get_public_access_block_config( + bucket: &str, +) -> Result<(s3s::dto::PublicAccessBlockConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await +} + +pub(crate) async fn get_bucket_replication_config( + bucket: &str, +) -> Result<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_replication_config(bucket).await +} + +pub(crate) async fn get_bucket_request_payment_config( + bucket: &str, +) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_request_payment_config(bucket).await +} + +pub(crate) async fn get_bucket_sse_config( + bucket: &str, +) -> Result<(s3s::dto::ServerSideEncryptionConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_sse_config(bucket).await +} + +pub(crate) async fn get_bucket_website_config(bucket: &str) -> Result<(s3s::dto::WebsiteConfiguration, time::OffsetDateTime)> { + ecstore_bucket::metadata_sys::get_website_config(bucket).await +} + +pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> { + ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, bm).await +} + +pub(crate) async fn update_bucket_metadata_config( + bucket: &str, + config_file: &str, + data: Vec, +) -> Result { + ecstore_bucket::metadata_sys::update(bucket, config_file, data).await +} + +pub(crate) fn add_object_lock_years(dt: time::OffsetDateTime, years: i32) -> time::OffsetDateTime { + ecstore_bucket::object_lock::objectlock_sys::add_years(dt, years) +} + +pub(crate) fn check_retention_for_modification( + user_defined: &std::collections::HashMap, + new_mode: Option<&str>, + new_retain_until: Option, + bypass_governance: bool, +) -> Option { + ecstore_bucket::object_lock::objectlock_sys::check_retention_for_modification( + user_defined, + new_mode, + new_retain_until, + bypass_governance, + ) +} + +pub(crate) async fn record_replication_proxy(bucket: &str, api: &str, is_err: bool) { + if let Some(stats) = ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get() { + stats.inc_proxy(bucket, api, is_err).await; + } +} + +pub(crate) fn decode_tags(tags: &str) -> Vec { + ecstore_bucket::tagging::decode_tags(tags) +} + +pub(crate) fn decode_tags_to_map(tags: &str) -> std::collections::HashMap { + ecstore_bucket::tagging::decode_tags_to_map(tags) +} + +pub(crate) fn encode_tags(tags: Vec) -> String { + ecstore_bucket::tagging::encode_tags(tags) +} + +pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { + ecstore_bucket::utils::serialize(val) +} + +pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool { + ecstore_error::is_err_bucket_not_found(err) +} + +pub(crate) fn is_err_object_not_found(err: &Error) -> bool { + ecstore_error::is_err_object_not_found(err) +} + +pub(crate) fn is_err_version_not_found(err: &Error) -> bool { + ecstore_error::is_err_version_not_found(err) +} + +pub(crate) fn is_all_buckets_not_found(errs: &[Option]) -> bool { + ecstore_disk::error_reduce::is_all_buckets_not_found(errs) +} + +pub(crate) fn get_global_lock_client() -> Option> { + ecstore_global::get_global_lock_client() +} + +pub(crate) fn get_global_lock_clients() +-> Option<&'static std::collections::HashMap>> { + ecstore_global::get_global_lock_clients() +} + +pub(crate) fn get_global_bucket_monitor() -> Option> { + ecstore_global::get_global_bucket_monitor() +} + +pub(crate) fn get_global_endpoints_opt() -> Option { + ecstore_global::get_global_endpoints_opt() +} + +pub(crate) fn get_global_deployment_id() -> Option { + ecstore_global::get_global_deployment_id() +} + +pub(crate) fn get_global_region() -> Option { + ecstore_global::get_global_region() +} + +pub(crate) fn global_rustfs_port() -> u16 { + ecstore_global::global_rustfs_port() +} + +pub(crate) fn get_global_tier_config_mgr() -> Arc> { + ecstore_global::get_global_tier_config_mgr() +} + +pub(crate) fn new_object_layer_fn() -> Option> { + ecstore_global::new_object_layer_fn() +} + +pub(crate) fn set_object_store_resolver(resolver: Arc) -> bool { + ecstore_global::set_object_store_resolver(resolver) +} + +pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { + ecstore_notification::get_global_notification_sys() +} + +pub(crate) async fn is_dist_erasure() -> bool { + ecstore_global::is_dist_erasure().await +} + +pub(crate) fn resolve_object_store_handle() -> Option> { + ecstore_global::resolve_object_store_handle() +} + +pub(crate) async fn collect_local_metrics( + types: MetricType, + opts: &CollectMetricsOpts, +) -> rustfs_madmin::metrics::RealtimeMetrics { + ecstore_metrics::collect_local_metrics(types, opts).await +} + +pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &http::HeaderMap) -> std::io::Result<()> { + ecstore_rpc::verify_rpc_signature(url, method, headers) +} + +pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { + ecstore_client::object_api_utils::to_s3s_etag(etag) +} + +pub(crate) fn table_catalog_path_hash(value: &str) -> String { + ecstore_bucket::metadata::table_catalog_path_hash(value) +} + +pub(crate) fn get_lock_acquire_timeout() -> std::time::Duration { + ecstore_set_disk::get_lock_acquire_timeout() +} + +#[cfg(test)] +pub(crate) fn boxed_reader(reader: R) -> DynReader +where + R: ecstore_rio::Reader + 'static, +{ + ecstore_rio::boxed_reader(reader) +} + +pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String { + ecstore_rio::compression_metadata_value(algorithm) +} + +pub(crate) fn wrap_reader(reader: R) -> DynReader +where + R: ecstore_rio::ReadStream + 'static, +{ + ecstore_rio::wrap_reader(reader) +} + +pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool { + ecstore_set_disk::is_valid_storage_class(storage_class) +} + +pub(crate) fn register_event_dispatch_hook(hook: F) -> bool +where + F: Fn(EventArgs) + Send + Sync + 'static, +{ + ecstore_event::register_event_dispatch_hook(hook) +} + +pub(crate) fn topology_snapshot_from_endpoint_pools_with_capabilities( + endpoint_pools: &EndpointServerPools, + capabilities: rustfs_storage_api::TopologyCapabilities, + disk_capabilities: rustfs_storage_api::DiskCapabilities, +) -> rustfs_storage_api::TopologySnapshot { + ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(endpoint_pools, capabilities, disk_capabilities) +} + +pub(crate) async fn reload_transition_tier_config(api: Arc) -> std::io::Result<()> { + ecstore_global::GLOBAL_TierConfigMgr.write().await.reload(api).await +} + +pub(crate) async fn all_local_disk_path() -> Vec { + ecstore_storage::all_local_disk_path().await +} + +pub(crate) async fn find_local_disk_by_ref(disk_ref: &str) -> Option { + ecstore_storage::find_local_disk_by_ref(disk_ref).await +} + +pub(crate) trait StorageReplicationConfigExt { + fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool; +} + +impl StorageReplicationConfigExt for s3s::dto::ReplicationConfiguration { + fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool { + ::has_active_rules( + self, prefix, recursive, + ) + } +} + +pub(crate) trait StorageVersioningConfigExt { + fn enabled(&self) -> bool; +} + +impl StorageVersioningConfigExt for s3s::dto::VersioningConfiguration { + fn enabled(&self) -> bool { + ::enabled(self) + } +} + +pub(crate) type GetObjectReader = ::GetObjectReader; +pub(crate) type ObjectInfo = ::ObjectInfo; +pub(crate) type ObjectOptions = ::ObjectOptions; +pub(crate) type PutObjReader = ::PutObjectReader; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 89bd669a3..4d960e4be 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -87,6 +87,7 @@ RUSTFS_LOCAL_COMPAT_GLOB_EXPORT_HITS_FILE="${TMP_DIR}/rustfs_local_compat_glob_e RUSTFS_ADMIN_CONFIG_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_admin_config_storage_compat_module_hits.txt" RUSTFS_STORAGE_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_storage_bucket_storage_compat_module_hits.txt" RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_compat_reexport_hits.txt" +RUSTFS_STORAGE_MOD_DIRECT_STORAGE_SOURCE_HITS_FILE="${TMP_DIR}/rustfs_storage_mod_direct_storage_source_hits.txt" RUSTFS_ADMIN_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_admin_bucket_storage_compat_module_hits.txt" RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_app_bucket_storage_compat_module_hits.txt" RUSTFS_OUTER_COMPAT_FACADE_ALIAS_HITS_FILE="${TMP_DIR}/rustfs_outer_compat_facade_alias_hits.txt" @@ -762,7 +763,7 @@ fi --glob '!**/ecstore_test_compat/**' \ --glob '!**/ecstore_fuzz_compat.rs' \ --glob '!target/**' \ - | rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs):' || true + | rg -v '^(rustfs/src/(admin/mod|app/mod)\.rs|rustfs/src/storage/storage_api\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs):' || true ) | cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE" @@ -969,6 +970,16 @@ if [[ -s "$RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE" ]]; then report_failure "RustFS storage owner compatibility must use local aliases or wrappers instead of re-exporting ECStore API symbols except temporary traits: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename 'rustfs_ecstore::|^use rustfs_storage_api|rustfs_storage_api::' \ + rustfs/src/storage/mod.rs || true +) >"$RUSTFS_STORAGE_MOD_DIRECT_STORAGE_SOURCE_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_MOD_DIRECT_STORAGE_SOURCE_HITS_FILE" ]]; then + report_failure "RustFS storage module root must route ECStore and storage-api symbols through rustfs/src/storage/storage_api.rs: $(paste -sd '; ' "$RUSTFS_STORAGE_MOD_DIRECT_STORAGE_SOURCE_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::bucket::\{[^}]*\b(?:bandwidth|bucket_target_sys|lifecycle|metadata|metadata_sys|quota|replication|target|utils|versioning|versioning_sys)\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::bucket::(?:bandwidth|bucket_target_sys|lifecycle|metadata|metadata_sys|quota|replication|target|utils|versioning|versioning_sys)\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::\{[^}]*\bstorageclass\b[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::config::storageclass\s*;' \ @@ -1111,7 +1122,7 @@ fi --glob '!**/ecstore_compat.rs' \ --glob '!**/ecstore_test_compat.rs' \ --glob '!**/ecstore_test_compat/**' | - rg -v '^(fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true + rg -v '^(fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/storage_api\.rs):' || true ) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then @@ -1137,7 +1148,7 @@ fi crates/e2e_test/src/replication_extension_test.rs \ rustfs/src/admin/mod.rs \ rustfs/src/app/mod.rs \ - rustfs/src/storage/mod.rs \ + rustfs/src/storage/storage_api.rs \ fuzz/fuzz_targets/bucket_validation.rs \ fuzz/fuzz_targets/path_containment.rs || true ) >"$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE" @@ -1165,7 +1176,7 @@ fi crates/e2e_test/src/replication_extension_test.rs \ rustfs/src/admin/mod.rs \ rustfs/src/app/mod.rs \ - rustfs/src/storage/mod.rs \ + rustfs/src/storage/storage_api.rs \ fuzz/fuzz_targets/bucket_validation.rs \ fuzz/fuzz_targets/path_containment.rs || true ) >"$COMPLETED_OWNER_BARE_FACADE_IMPORT_HITS_FILE" @@ -1193,7 +1204,7 @@ fi crates/e2e_test/src/replication_extension_test.rs \ rustfs/src/admin/mod.rs \ rustfs/src/app/mod.rs \ - rustfs/src/storage/mod.rs \ + rustfs/src/storage/storage_api.rs \ fuzz/fuzz_targets/bucket_validation.rs \ fuzz/fuzz_targets/path_containment.rs | rg -v '^[^:]+:[0-9]+:\s*(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::' || true