From 0a5b1b1b3ab9dab852f1950c68329fdd580be85d Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 27 Jun 2026 09:03:20 +0800 Subject: [PATCH] refactor: consolidate ecstore owner module layout (#3934) * refactor: shrink ecstore root owner facades * refactor: remove ecstore core store root shims * refactor: move ecstore erasure owner modules * refactor: remove ecstore root rpc facade * refactor: move ecstore services domain modules --- crates/ecstore/src/api/mod.rs | 34 +++--- .../ecstore/src/bucket/bucket_target_sys.rs | 2 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 16 +-- .../src/bucket/lifecycle/tier_sweeper.rs | 2 +- .../bucket/replication/replication_pool.rs | 2 +- .../replication/replication_resyncer.rs | 4 +- .../bucket/replication/replication_state.rs | 2 +- crates/ecstore/src/cluster/control_plane.rs | 4 +- crates/ecstore/src/cluster/rpc/client.rs | 4 +- crates/ecstore/src/cluster/rpc/http_auth.rs | 6 +- .../cluster/rpc/internode_data_transport.rs | 2 +- .../src/cluster/rpc/peer_rest_client.rs | 8 +- .../ecstore/src/cluster/rpc/peer_s3_client.rs | 16 +-- crates/ecstore/src/cluster/rpc/remote_disk.rs | 42 ++++--- .../ecstore/src/cluster/rpc/remote_locker.rs | 4 +- crates/ecstore/src/config/com.rs | 6 +- crates/ecstore/src/core/mod.rs | 21 ++++ crates/ecstore/src/core/pools.rs | 23 ++-- crates/ecstore/src/core/pools_test.rs | 2 +- crates/ecstore/src/core/sets.rs | 8 +- .../ecstore/src/data_movement/backpressure.rs | 2 +- crates/ecstore/src/data_movement/mod.rs | 2 + crates/ecstore/src/data_usage/mod.rs | 2 +- .../src/diagnostics/admin_server_info.rs | 6 +- crates/ecstore/src/diagnostics/mod.rs | 16 +++ crates/ecstore/src/disk/disk_store.rs | 2 +- crates/ecstore/src/disk/local.rs | 4 +- crates/ecstore/src/disk/mod.rs | 6 +- .../codec}/bridge.rs | 4 +- .../{erasure_codec => erasure/codec}/mod.rs | 0 .../codec}/workspace.rs | 0 .../coding}/bitrot.rs | 0 .../coding}/decode.rs | 14 +-- .../coding}/decode_reader.rs | 10 +- .../coding}/encode.rs | 8 +- .../coding}/erasure.rs | 0 .../coding}/heal.rs | 10 +- .../{erasure_coding => erasure/coding}/mod.rs | 0 .../disks_layout_facade.rs => erasure/mod.rs} | 3 +- crates/ecstore/src/io_support/bitrot.rs | 2 +- crates/ecstore/src/io_support/mod.rs | 17 +++ crates/ecstore/src/layout/endpoints.rs | 2 +- crates/ecstore/src/layout/endpoints_facade.rs | 1 - crates/ecstore/src/layout/pool_space.rs | 4 +- crates/ecstore/src/lib.rs | 61 ++-------- crates/ecstore/src/metadata/bucket.rs | 2 +- crates/ecstore/src/metadata/bucket_sys.rs | 2 +- crates/ecstore/src/object_api/mod.rs | 4 +- crates/ecstore/src/object_api/readers.rs | 100 +++++++-------- crates/ecstore/src/object_api/types.rs | 6 +- crates/ecstore/src/rpc/mod.rs | 24 ---- crates/ecstore/src/runtime/global.rs | 6 +- crates/ecstore/src/runtime/mod.rs | 16 +++ crates/ecstore/src/runtime/sources.rs | 13 +- .../ecstore/src/services/metrics_realtime.rs | 4 +- crates/ecstore/src/services/mod.rs | 20 +++ .../ecstore/src/services/notification_sys.rs | 21 ++-- .../src/{ => services}/rebalance/control.rs | 0 .../src/{ => services}/rebalance/entry.rs | 8 +- .../src/{ => services}/rebalance/meta.rs | 0 .../src/{ => services}/rebalance/migration.rs | 0 .../src/{ => services}/rebalance/mod.rs | 0 .../rebalance/rebalance_unit_tests.rs | 16 +-- .../src/{ => services}/rebalance/runtime.rs | 2 +- .../src/{ => services}/rebalance/types.rs | 0 .../src/{ => services}/rebalance/worker.rs | 4 +- crates/ecstore/src/{ => services}/tier/mod.rs | 0 .../ecstore/src/{ => services}/tier/tier.rs | 38 +++--- .../src/{ => services}/tier/tier_admin.rs | 0 .../src/{ => services}/tier/tier_config.rs | 0 .../src/{ => services}/tier/tier_gen.rs | 2 +- .../src/{ => services}/tier/tier_handlers.rs | 0 .../src/{ => services}/tier/warm_backend.rs | 2 +- .../tier/warm_backend_aliyun.rs | 2 +- .../{ => services}/tier/warm_backend_azure.rs | 2 +- .../{ => services}/tier/warm_backend_gcs.rs | 2 +- .../tier/warm_backend_huaweicloud.rs | 2 +- .../{ => services}/tier/warm_backend_minio.rs | 2 +- .../{ => services}/tier/warm_backend_r2.rs | 2 +- .../tier/warm_backend_rustfs.rs | 2 +- .../{ => services}/tier/warm_backend_s3.rs | 2 +- .../{ => services}/tier/warm_backend_s3sdk.rs | 2 +- .../tier/warm_backend_tencent.rs | 2 +- crates/ecstore/src/set_disk/heal.rs | 4 +- crates/ecstore/src/set_disk/lock.rs | 4 +- crates/ecstore/src/set_disk/mod.rs | 60 +++++---- crates/ecstore/src/set_disk/read.rs | 14 +-- crates/ecstore/src/set_disk/shard_source.rs | 4 +- crates/ecstore/src/store/bucket.rs | 2 +- crates/ecstore/src/store/init.rs | 18 +-- crates/ecstore/src/store/init_format.rs | 2 +- crates/ecstore/src/store/list_objects.rs | 12 +- crates/ecstore/src/store/mod.rs | 35 +++--- crates/ecstore/src/store/peer.rs | 2 +- crates/ecstore/src/store/rebalance.rs | 2 +- docs/architecture/migration-progress.md | 114 +++++++++++++++--- scripts/check_architecture_migration_rules.sh | 92 ++++++++++---- 97 files changed, 614 insertions(+), 445 deletions(-) create mode 100644 crates/ecstore/src/core/mod.rs create mode 100644 crates/ecstore/src/diagnostics/mod.rs rename crates/ecstore/src/{erasure_codec => erasure/codec}/bridge.rs (99%) rename crates/ecstore/src/{erasure_codec => erasure/codec}/mod.rs (100%) rename crates/ecstore/src/{erasure_codec => erasure/codec}/workspace.rs (100%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/bitrot.rs (100%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/decode.rs (99%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/decode_reader.rs (99%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/encode.rs (99%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/erasure.rs (100%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/heal.rs (96%) rename crates/ecstore/src/{erasure_coding => erasure/coding}/mod.rs (100%) rename crates/ecstore/src/{layout/disks_layout_facade.rs => erasure/mod.rs} (92%) create mode 100644 crates/ecstore/src/io_support/mod.rs delete mode 100644 crates/ecstore/src/layout/endpoints_facade.rs delete mode 100644 crates/ecstore/src/rpc/mod.rs create mode 100644 crates/ecstore/src/runtime/mod.rs create mode 100644 crates/ecstore/src/services/mod.rs rename crates/ecstore/src/{ => services}/rebalance/control.rs (100%) rename crates/ecstore/src/{ => services}/rebalance/entry.rs (98%) rename crates/ecstore/src/{ => services}/rebalance/meta.rs (100%) rename crates/ecstore/src/{ => services}/rebalance/migration.rs (100%) rename crates/ecstore/src/{ => services}/rebalance/mod.rs (100%) rename crates/ecstore/src/{ => services}/rebalance/rebalance_unit_tests.rs (99%) rename crates/ecstore/src/{ => services}/rebalance/runtime.rs (99%) rename crates/ecstore/src/{ => services}/rebalance/types.rs (100%) rename crates/ecstore/src/{ => services}/rebalance/worker.rs (99%) rename crates/ecstore/src/{ => services}/tier/mod.rs (100%) rename crates/ecstore/src/{ => services}/tier/tier.rs (97%) rename crates/ecstore/src/{ => services}/tier/tier_admin.rs (100%) rename crates/ecstore/src/{ => services}/tier/tier_config.rs (100%) rename crates/ecstore/src/{ => services}/tier/tier_gen.rs (93%) rename crates/ecstore/src/{ => services}/tier/tier_handlers.rs (100%) rename crates/ecstore/src/{ => services}/tier/warm_backend.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_aliyun.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_azure.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_gcs.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_huaweicloud.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_minio.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_r2.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_rustfs.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_s3.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_s3sdk.rs (99%) rename crates/ecstore/src/{ => services}/tier/warm_backend_tencent.rs (99%) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 6e8958073..7a40199d7 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -15,11 +15,11 @@ //! Explicit ECStore public facades for outer crate compatibility boundaries. pub mod admin { - pub use crate::admin_server_info::{get_local_server_property, get_server_info}; + pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info}; } pub mod bitrot { - pub use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; + pub use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_writer}; } pub mod bucket { @@ -34,11 +34,11 @@ pub mod cache { } pub mod capacity { - pub use crate::pools::{ + pub use crate::core::pools::{ PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, path2_bucket_object, path2_bucket_object_with_base_path, }; - pub use crate::store_utils::is_reserved_or_invalid_bucket; + pub use crate::store::utils::is_reserved_or_invalid_bucket; } pub mod client { @@ -56,7 +56,7 @@ pub mod cluster { } pub mod compression { - pub use crate::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; + pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; } pub mod config { @@ -97,7 +97,7 @@ pub mod error { } pub mod erasure { - pub use crate::erasure_coding::{ + pub use crate::erasure::coding::{ BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ReedSolomonEncoder, calc_shard_size, calc_shard_size_legacy, }; @@ -105,11 +105,11 @@ pub mod erasure { pub mod event { pub use crate::event::name::EventName; - pub use crate::event_notification::{EventArgs, register_event_dispatch_hook}; + pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook}; } pub mod global { - pub use crate::global::{ + pub use crate::runtime::global::{ GLOBAL_BOOT_TIME, GLOBAL_LOCAL_DISK_MAP, GLOBAL_TierConfigMgr, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr, global_rustfs_port, is_dist_erasure, is_erasure, is_erasure_sd, is_first_cluster_node_local, new_object_layer_fn, @@ -119,16 +119,16 @@ pub mod global { } pub mod layout { - pub use crate::disks_layout::DisksLayout; - pub use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType}; + pub use crate::layout::disks_layout::DisksLayout; + pub use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints, SetupType}; } pub mod metrics { - pub use crate::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}; + pub use crate::services::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}; } pub mod notification { - pub use crate::notification_sys::{ + pub use crate::services::notification_sys::{ NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys, }; } @@ -141,7 +141,7 @@ pub mod object { } pub mod rebalance { - pub use crate::rebalance::{ + pub use crate::services::rebalance::{ DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record, @@ -149,14 +149,14 @@ pub mod rebalance { } pub mod rio { - pub use crate::rio::{ + pub use crate::io_support::rio::{ DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan, boxed_reader, compression_metadata_value, wrap_reader, }; } pub mod rpc { - pub use crate::rpc::{ + pub use crate::cluster::rpc::{ LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature, @@ -168,7 +168,7 @@ pub mod set_disk { } pub mod store_list { - pub use crate::store_list_objects::{ListPathOptions, max_keys_plus_one}; + pub use crate::store::list_objects::{ListPathOptions, max_keys_plus_one}; } pub mod storage { @@ -179,5 +179,5 @@ pub mod storage { } pub mod tier { - pub use crate::tier::{tier, tier_admin, tier_config, tier_handlers, warm_backend}; + pub use crate::services::tier::{tier, tier_admin, tier_config, tier_handlers, warm_backend}; } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 1db50ead2..239b7a2de 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -21,7 +21,7 @@ use crate::bucket::target::ARN; use crate::bucket::target::BucketTargetType; use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials}; use crate::bucket::versioning_sys::BucketVersioningSys; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use aws_credential_types::Credentials as SdkCredentials; use aws_sdk_s3::config::Region as SdkRegion; use aws_sdk_s3::config::SharedHttpClient; diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index f007e2908..28b8c1dfc 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -32,9 +32,10 @@ use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_MULTIPART_BUCKET, ST use crate::error::Error; use crate::error::StorageError; use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; -use crate::event_notification::{EventArgs, send_event}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; +use crate::services::event_notification::{EventArgs, send_event}; +use crate::services::tier::warm_backend::WarmBackendGetOpts; use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks}; use crate::storage_api_contracts::{ lifecycle::ExpirationOptions, @@ -44,7 +45,6 @@ use crate::storage_api_contracts::{ range::HTTPRangeSpec, }; use crate::store::ECStore; -use crate::tier::warm_backend::WarmBackendGetOpts; use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use futures::Future; use http::HeaderMap; @@ -622,7 +622,7 @@ impl ExpiryState { } async fn worker(rx: &mut Receiver>, api: Arc, stats: Arc) { - let cancel_token = crate::global::get_background_services_cancel_token().unwrap_or_else(|| { + let cancel_token = crate::runtime::global::get_background_services_cancel_token().unwrap_or_else(|| { static FALLBACK: std::sync::OnceLock = std::sync::OnceLock::new(); FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new) }); @@ -1376,7 +1376,7 @@ fn spawn_tier_free_version_recovery_once(api: Arc) { } tokio::spawn(async move { - let cancel_token = crate::global::get_background_services_cancel_token() + let cancel_token = crate::runtime::global::get_background_services_cancel_token() .cloned() .unwrap_or_else(CancellationToken::new); let mut interval = tokio::time::interval(StdDuration::from_secs(60)); @@ -1451,7 +1451,7 @@ fn spawn_tier_delete_journal_recovery_once(api: Arc) { } tokio::spawn(async move { - let cancel_token = crate::global::get_background_services_cancel_token() + let cancel_token = crate::runtime::global::get_background_services_cancel_token() .cloned() .unwrap_or_else(CancellationToken::new); run_tier_delete_journal_recovery_loop(api, cancel_token).await; @@ -2889,10 +2889,10 @@ mod tests { use crate::bucket::metadata_sys; use crate::disk::RUSTFS_META_MULTIPART_BUCKET; use crate::disk::endpoint::Endpoint; - use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::error::is_err_invalid_upload_id; + use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader}; - use crate::runtime_sources; + use crate::runtime::sources as runtime_sources; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; use crate::storage_api_contracts::{ bucket::{BucketOperations, BucketOptions, MakeBucketOptions}, diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 44e5983a4..db3ffb1ef 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -22,7 +22,7 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp; use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts}; use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry; use crate::client::signer_error::error_chain_contains_signer_header_marker; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::lifecycle::TransitionedObject; use crate::store::ECStore; use rustfs_utils::get_env_usize; diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 50d5bb751..5449d522b 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -28,7 +28,7 @@ use crate::config::com::{read_config, save_config}; use crate::disk::BUCKET_META_PREFIX; use crate::error::Error as EcstoreError; use crate::object_api::{ObjectInfo, ObjectOptions}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::object::DeletedObject; use crate::storage_api_contracts::object::EcstoreObjectIO; use lazy_static::lazy_static; diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 6bf1047cc..16cca6eb6 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -27,9 +27,9 @@ use crate::client::api_get_options::{AdvancedGetOptions, StatObjectOptions}; use crate::config::com::save_config; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found}; -use crate::event_notification::{EventArgs, send_event}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; +use crate::services::event_notification::{EventArgs, send_event}; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::{ list::{ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, diff --git a/crates/ecstore/src/bucket/replication/replication_state.rs b/crates/ecstore/src/bucket/replication/replication_state.rs index e2116672f..798c4a503 100644 --- a/crates/ecstore/src/bucket/replication/replication_state.rs +++ b/crates/ecstore/src/bucket/replication/replication_state.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::error::Error; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use rustfs_filemeta::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; diff --git a/crates/ecstore/src/cluster/control_plane.rs b/crates/ecstore/src/cluster/control_plane.rs index 3e790d476..e8bfa877d 100644 --- a/crates/ecstore/src/cluster/control_plane.rs +++ b/crates/ecstore/src/cluster/control_plane.rs @@ -20,8 +20,8 @@ use crate::storage_api_contracts::topology::{ }; use crate::{ - endpoints::EndpointServerPools, layout::endpoint::{Endpoint, EndpointType}, + layout::endpoints::EndpointServerPools, }; const ENDPOINT_TYPE_LABEL: &str = "endpoint_type"; @@ -434,7 +434,7 @@ mod tests { use super::*; use std::collections::BTreeSet; - use crate::endpoints::{Endpoints, PoolEndpoints}; + use crate::layout::endpoints::{Endpoints, PoolEndpoints}; #[test] fn topology_snapshot_maps_endpoint_sets_without_local_paths() { diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index 521a784a6..2ffe49579 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster::rpc::{TONIC_RPC_PREFIX, gen_signature_headers}; use crate::disk::error::{DiskError, Error as DiskErrorType}; -use crate::rpc::{TONIC_RPC_PREFIX, gen_signature_headers}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use http::Method; use rustfs_protos::{create_new_channel, proto_gen::node_service::node_service_client::NodeServiceClient}; use std::{error::Error, io::ErrorKind}; diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index 9beb270eb..9d072652d 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers}; +use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers}; use base64::Engine as _; use base64::engine::general_purpose; use hmac::{Hmac, KeyInit, Mac}; @@ -163,8 +163,8 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) -> #[cfg(test)] mod tests { use super::*; - use crate::rpc::context_propagation::REQUEST_ID_HEADER; - use crate::runtime_sources; + use crate::cluster::rpc::context_propagation::REQUEST_ID_HEADER; + use crate::runtime::sources as runtime_sources; use http::{HeaderMap, Method}; use std::io::{self, Write}; use std::sync::{Arc, Mutex}; diff --git a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs index 144d79c14..b21be3695 100644 --- a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster::rpc::build_auth_headers; use crate::disk::error::{Error, Result}; use crate::disk::{FileReader, FileWriter}; -use crate::rpc::build_auth_headers; use async_trait::async_trait; use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE}; use rustfs_config::{ diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 8d42f4eb8..23227652e 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::error::{Error, Result}; -use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::{ disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout}, - endpoints::EndpointServerPools, - metrics_realtime::{CollectMetricsOpts, MetricType}, - runtime_sources, + layout::endpoints::EndpointServerPools, + runtime::sources as runtime_sources, + services::metrics_realtime::{CollectMetricsOpts, MetricType}, }; use rmp_serde::{Deserializer, Serializer}; use rustfs_madmin::{ diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index 6d0cc9251..a6520b084 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -13,23 +13,23 @@ // limitations under the License. use crate::bucket::metadata_sys; +use crate::cluster::rpc::client::{ + TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client, +}; use crate::disk::error::DiskError; use crate::disk::error::{Error, Result}; use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs}; use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration}; -use crate::rpc::client::{ - TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client, -}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::bucket::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions}; use crate::store::all_local_disk; -use crate::store_utils::is_reserved_or_invalid_bucket; +use crate::store::utils::is_reserved_or_invalid_bucket; use crate::{ disk::{ self, VolumeInfo, disk_store::{DiskHealthTracker, get_drive_active_check_interval, get_drive_active_check_timeout}, }, - endpoints::{EndpointServerPools, Node}, + layout::endpoints::{EndpointServerPools, Node}, }; use async_trait::async_trait; use futures::future::join_all; @@ -1102,8 +1102,8 @@ mod tests { use crate::disk::disk_store::LocalDiskWrapper; use crate::disk::endpoint::Endpoint; use crate::disk::local::LocalDisk; - use crate::endpoints::{Endpoints, PoolEndpoints}; - use crate::global::reset_local_disk_test_state; + use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + use crate::runtime::global::reset_local_disk_test_state; use crate::store::init_local_disks; use rustfs_filemeta::FileInfo; use serial_test::serial; diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index cf4f14a0e..93fe6b811 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster::rpc::client::{ + TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client, +}; +use crate::cluster::rpc::internode_data_transport::{ + InternodeDataTransport, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest, +}; use crate::disk::error::{Error, Result}; use crate::disk::{ CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, @@ -25,10 +31,6 @@ use crate::disk::{ health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state}, }; use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard}; -use crate::rpc::client::{ - TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client, -}; -use crate::rpc::internode_data_transport::{InternodeDataTransport, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest}; use crate::set_disk::DEFAULT_READ_BUFFER_SIZE; use bytes::Bytes; use futures::lock::Mutex; @@ -183,14 +185,14 @@ impl RemoteDisk { if attempt > 1 && let Some(classification) = last_retry_classification { - crate::rpc::runtime_sources::record_remote_disk_open_write_retry_success(classification); + crate::cluster::rpc::runtime_sources::record_remote_disk_open_write_retry_success(classification); } return Ok(writer); } Err(err) if attempt < REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS && Self::is_retryable_open_write_error(&err) => { if let Some(classification) = err.internode_http_error_kind() { let classification = classification.metric_label(); - crate::rpc::runtime_sources::record_remote_disk_open_write_retry(classification); + crate::cluster::rpc::runtime_sources::record_remote_disk_open_write_retry(classification); last_retry_classification = Some(classification); } debug!( @@ -2106,7 +2108,7 @@ impl DiskAPI for RemoteDisk { let data_len = data.len(); let disk = self.disk_ref().await; let mut client = self.get_client().await.map_err(|err| { - crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error(); Error::other(format!("can not get client, err: {err}")) })?; let request = Request::new(WriteAllRequest { @@ -2116,19 +2118,19 @@ impl DiskAPI for RemoteDisk { data, }); - crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_request(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request(); let response = match client.write_all(request).await { Ok(response) => response.into_inner(), Err(err) => { - crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error(); return Err(err.into()); } }; - crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len); if !response.success { - crate::rpc::runtime_sources::record_remote_disk_grpc_write_all_error(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error(); return Err(response.error.unwrap_or_default().into()); } @@ -2157,7 +2159,7 @@ impl DiskAPI for RemoteDisk { || async { let disk = self.disk_ref().await; let mut client = self.get_client().await.map_err(|err| { - crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error(); Error::other(format!("can not get client, err: {err}")) })?; let request = Request::new(ReadAllRequest { @@ -2166,21 +2168,21 @@ impl DiskAPI for RemoteDisk { path: path.to_string(), }); - crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_request(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request(); let response = match client.read_all(request).await { Ok(response) => response.into_inner(), Err(err) => { - crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error(); return Err(err.into()); } }; if !response.success { - crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_error(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error(); return Err(response.error.unwrap_or_default().into()); } - crate::rpc::runtime_sources::record_remote_disk_grpc_read_all_recv_bytes(response.data.len()); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_recv_bytes(response.data.len()); Ok(response.data) }, get_max_timeout_duration(), @@ -2228,8 +2230,8 @@ impl DiskAPI for RemoteDisk { #[cfg(test)] mod tests { use super::*; - use crate::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport}; - use crate::runtime_sources; + use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport}; + use crate::runtime::sources as runtime_sources; use serde_json::Value; use std::io::{self as std_io, Write}; use std::pin::Pin; @@ -2995,7 +2997,7 @@ mod tests { OpenWriteTestStep::Success, ]); let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await; - crate::rpc::runtime_sources::reset_internode_metrics_for_test(); + crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test(); let _created = remote_disk .create_file("orig-bucket", "bucket", "object/part.1", 4096) @@ -3004,7 +3006,7 @@ mod tests { let calls = transport.calls(); assert_eq!(calls.len(), 2, "create_file should retry exactly once"); - let snapshot = crate::rpc::runtime_sources::internode_metrics_snapshot_for_test(); + let snapshot = crate::cluster::rpc::runtime_sources::internode_metrics_snapshot_for_test(); assert_eq!(snapshot.outgoing_requests_total, 0); } diff --git a/crates/ecstore/src/cluster/rpc/remote_locker.rs b/crates/ecstore/src/cluster/rpc/remote_locker.rs index 2a34922d1..593026c6d 100644 --- a/crates/ecstore/src/cluster/rpc/remote_locker.rs +++ b/crates/ecstore/src/cluster/rpc/remote_locker.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; +use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use async_trait::async_trait; use bytes::Bytes; use rustfs_lock::{ @@ -536,7 +536,7 @@ impl LockClient for RemoteClient { #[cfg(test)] mod tests { use super::*; - use crate::runtime_sources; + use crate::runtime::sources as runtime_sources; use rustfs_lock::{ObjectKey, types::LockPriority}; use tokio::net::TcpListener; use tokio::task::JoinHandle; diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index da3e59efa..025096a2d 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -16,7 +16,7 @@ use crate::config::{audit, notify, oidc, storageclass}; use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::{ admin::StorageAdminApi, object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete}, @@ -1313,10 +1313,10 @@ mod tests { }; use crate::config::{audit, notify, oidc}; use crate::disk::endpoint::Endpoint; - use crate::endpoints::SetupType; use crate::error::{Error, Result}; + use crate::layout::endpoints::SetupType; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; - use crate::runtime_sources; + use crate::runtime::sources as runtime_sources; use crate::set_disk::SetDisks; use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, range::HTTPRangeSpec}; use http::HeaderMap; diff --git a/crates/ecstore/src/core/mod.rs b/crates/ecstore/src/core/mod.rs new file mode 100644 index 000000000..cdfdebf33 --- /dev/null +++ b/crates/ecstore/src/core/mod.rs @@ -0,0 +1,21 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) mod pools; +pub(crate) mod sets; + +#[cfg(test)] +mod pools_test; +#[cfg(test)] +mod store_test; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index ae36aedc0..fa2d740bc 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -27,18 +27,18 @@ use crate::bucket::{ use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; use crate::config::com::{CONFIG_PREFIX, read_config, read_config_no_lock, save_config, save_config_with_opts}; use crate::data_movement; -use crate::data_movement_backpressure::{self, DataMovementOperation}; +use crate::data_movement::backpressure::{self, DataMovementOperation}; use crate::data_usage::DATA_USAGE_CACHE_NAME; use crate::disk::error::DiskError; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; -use crate::endpoints::EndpointServerPools; use crate::error::{Error, Result}; use crate::error::{ StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; +use crate::layout::endpoints::EndpointServerPools; use crate::object_api::{GetObjectReader, ObjectOptions}; -use crate::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; +use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::storage_api_contracts::{ admin::StorageAdminApi, @@ -47,7 +47,7 @@ use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, object::{ObjectIO as _, ObjectOperations as _}, }; -use crate::{sets::Sets, store::ECStore}; +use crate::{core::sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; use http::HeaderMap; @@ -3123,12 +3123,9 @@ impl ECStore { return; } - if let Err(err) = data_movement_backpressure::wait_for_data_movement_admission( - DataMovementOperation::Decommission, - idx, - &callback_rx, - ) - .await + if let Err(err) = + backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &callback_rx) + .await { if matches!(err, Error::OperationCanceled) { return; @@ -4926,9 +4923,9 @@ mod pools_tests { }; use crate::data_movement; use crate::disk::endpoint::Endpoint; - use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::error::Error; - use crate::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; + use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo}; use rustfs_rio::Index; use std::sync::{ diff --git a/crates/ecstore/src/core/pools_test.rs b/crates/ecstore/src/core/pools_test.rs index 4ac19c0a7..11313568a 100644 --- a/crates/ecstore/src/core/pools_test.rs +++ b/crates/ecstore/src/core/pools_test.rs @@ -14,7 +14,7 @@ #[cfg(test)] mod capacity_dedup_tests { - use crate::pools::{ + use crate::core::pools::{ fallback_free_capacity_dedup, fallback_total_capacity_dedup, get_total_usable_capacity, get_total_usable_capacity_free, }; diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index ab5c68491..2b35b5678 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -30,12 +30,12 @@ use crate::{ format::{DistributionAlgoVersion, FormatV3}, new_disk, }, - endpoints::{Endpoints, PoolEndpoints}, error::StorageError, + layout::endpoints::{Endpoints, PoolEndpoints}, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, - runtime_sources, + runtime::sources as runtime_sources, set_disk::SetDisks, - store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, + store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; use futures::{ future::join_all, @@ -1090,8 +1090,8 @@ async fn init_storage_disks_with_errors( #[cfg(test)] mod tests { use super::*; - use crate::endpoints::SetupType; use crate::layout::endpoint::Endpoint; + use crate::layout::endpoints::SetupType; use crate::storage_api_contracts::heal::HealOperations as _; use crate::storage_api_contracts::list::ListOperations as _; use rustfs_lock::client::local::LocalClient; diff --git a/crates/ecstore/src/data_movement/backpressure.rs b/crates/ecstore/src/data_movement/backpressure.rs index 7d91a7204..f0406e474 100644 --- a/crates/ecstore/src/data_movement/backpressure.rs +++ b/crates/ecstore/src/data_movement/backpressure.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::error::{Error, Result}; -use crate::runtime_sources::{self, WorkloadSnapshotProviderRef}; +use crate::runtime::sources::{self as runtime_sources, WorkloadSnapshotProviderRef}; use metrics::{counter, histogram}; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass}; use std::time::{Duration, Instant}; diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 94f02e033..4380000f8 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub(crate) mod backpressure; + use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::set_disk::SetDisks; diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 1414c30c3..9d2176b50 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -20,7 +20,7 @@ use crate::{ config::com::read_config, disk::DiskAPI, error::{Error, classify_system_path_failure_reason}, - runtime_sources, + runtime::sources as runtime_sources, store::ECStore, }; pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path}; diff --git a/crates/ecstore/src/diagnostics/admin_server_info.rs b/crates/ecstore/src/diagnostics/admin_server_info.rs index 4eeee67ab..1ba2291f3 100644 --- a/crates/ecstore/src/diagnostics/admin_server_info.rs +++ b/crates/ecstore/src/diagnostics/admin_server_info.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::cluster::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend}; use crate::error::{Error, Result}; -use crate::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; -use crate::{disk::endpoint::Endpoint, runtime_sources}; +use crate::{disk::endpoint::Endpoint, runtime::sources as runtime_sources}; use crate::data_usage::load_data_usage_cache; use crate::storage_api_contracts::admin::StorageAdminApi; @@ -389,7 +389,7 @@ pub fn get_commit_id() -> String { mod tests { use serial_test::serial; - use crate::runtime_sources; + use crate::runtime::sources as runtime_sources; use super::get_server_info; diff --git a/crates/ecstore/src/diagnostics/mod.rs b/crates/ecstore/src/diagnostics/mod.rs new file mode 100644 index 000000000..39efcdb25 --- /dev/null +++ b/crates/ecstore/src/diagnostics/mod.rs @@ -0,0 +1,16 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) mod admin_server_info; +pub(crate) mod get; diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index fe7822ae5..785d08581 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -23,7 +23,7 @@ use crate::disk::{ }, local::{LocalDisk, ScanGuard}, }; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use bytes::Bytes; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 0ce4bd645..80f8a24b2 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -28,8 +28,8 @@ use crate::disk::{ os, os::{check_path_length, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source}, }; -use crate::erasure_coding::bitrot_verify; -use crate::runtime_sources; +use crate::erasure::coding::bitrot_verify; +use crate::runtime::sources as runtime_sources; use bytes::Bytes; use metrics::counter; use parking_lot::RwLock as ParkingLotRwLock; diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index fe6f4100f..d7270e011 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -33,11 +33,11 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; pub const STORAGE_FORMAT_FILE: &str = "xl.meta"; pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp"; +use crate::cluster::rpc::RemoteDisk; +use crate::cluster::rpc::build_internode_data_transport_from_env; use crate::disk::disk_store::LocalDiskWrapper; use crate::disk::health_state::RuntimeDriveHealthState; use crate::disk::local::ScanGuard; -use crate::rpc::RemoteDisk; -use crate::rpc::build_internode_data_transport_from_env; use bytes::Bytes; use endpoint::Endpoint; use error::DiskError; @@ -1170,7 +1170,7 @@ mod tests { cleanup: false, health_check: false, }, - Arc::new(crate::rpc::TcpHttpInternodeDataTransport), + Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport), ) .await .unwrap(); diff --git a/crates/ecstore/src/erasure_codec/bridge.rs b/crates/ecstore/src/erasure/codec/bridge.rs similarity index 99% rename from crates/ecstore/src/erasure_codec/bridge.rs rename to crates/ecstore/src/erasure/codec/bridge.rs index 8765f765d..56bea5cad 100644 --- a/crates/ecstore/src/erasure_codec/bridge.rs +++ b/crates/ecstore/src/erasure/codec/bridge.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::erasure_codec::workspace::RustfsCodecDecodeWorkspace; -use crate::erasure_coding::Erasure; +use crate::erasure::codec::workspace::RustfsCodecDecodeWorkspace; +use crate::erasure::coding::Erasure; use reed_solomon_erasure::galois_8::ReedSolomon; use std::io; diff --git a/crates/ecstore/src/erasure_codec/mod.rs b/crates/ecstore/src/erasure/codec/mod.rs similarity index 100% rename from crates/ecstore/src/erasure_codec/mod.rs rename to crates/ecstore/src/erasure/codec/mod.rs diff --git a/crates/ecstore/src/erasure_codec/workspace.rs b/crates/ecstore/src/erasure/codec/workspace.rs similarity index 100% rename from crates/ecstore/src/erasure_codec/workspace.rs rename to crates/ecstore/src/erasure/codec/workspace.rs diff --git a/crates/ecstore/src/erasure_coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs similarity index 100% rename from crates/ecstore/src/erasure_coding/bitrot.rs rename to crates/ecstore/src/erasure/coding/bitrot.rs diff --git a/crates/ecstore/src/erasure_coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs similarity index 99% rename from crates/ecstore/src/erasure_coding/decode.rs rename to crates/ecstore/src/erasure/coding/decode.rs index e28ca41ce..3a1f92a08 100644 --- a/crates/ecstore/src/erasure_coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::disk::error::Error; -use crate::disk::error_reduce::reduce_errs; -use crate::erasure_codec::workspace::ShardBufferPool; -use crate::erasure_coding::{BitrotReader, Erasure}; -use crate::get_diagnostics::{ +use crate::diagnostics::get::{ GET_OBJECT_PATH_LEGACY_DUPLEX, GET_SHARD_READ_ERROR_MISSING, GET_SHARD_READ_ERROR_NONE, GET_SHARD_READ_OUTCOME_ERROR, GET_SHARD_READ_OUTCOME_MISSING, GET_SHARD_READ_OUTCOME_SUCCESS, GET_SHARD_ROLE_DATA, GET_SHARD_ROLE_PARITY, GET_STAGE_EMIT, GET_STAGE_RANGE, GET_STAGE_RECONSTRUCT, GET_STAGE_STRIPE_READ, GET_STAGE_STRIPE_READ_FIRST_SHARD, GET_STAGE_STRIPE_READ_QUORUM, GetObjectFailureReason, classify_io_error, record_get_object_pipeline_failure, }; +use crate::disk::error::Error; +use crate::disk::error_reduce::reduce_errs; +use crate::erasure::codec::workspace::ShardBufferPool; +use crate::erasure::coding::{BitrotReader, Erasure}; use crate::set_disk::shard_source::{ShardReadCost, ShardStripeSource, StripeReadState}; use futures::stream::{FuturesUnordered, StreamExt}; use pin_project_lite::pin_project; @@ -870,7 +870,7 @@ mod tests { use super::*; use crate::{ disk::error::DiskError, - erasure_coding::{BitrotReader, BitrotWriter}, + erasure::coding::{BitrotReader, BitrotWriter}, }; use rustfs_utils::HashAlgorithm; use std::io::Cursor; @@ -1051,7 +1051,7 @@ mod tests { const PARITY_SHARDS: usize = 2; const BLOCK_SIZE: usize = 1024 * 1024; - use crate::rio::CompressReader; + use crate::io_support::rio::CompressReader; use rustfs_utils::CompressionAlgorithm; use tokio::io::AsyncReadExt; diff --git a/crates/ecstore/src/erasure_coding/decode_reader.rs b/crates/ecstore/src/erasure/coding/decode_reader.rs similarity index 99% rename from crates/ecstore/src/erasure_coding/decode_reader.rs rename to crates/ecstore/src/erasure/coding/decode_reader.rs index 21bb019b0..df032b811 100644 --- a/crates/ecstore/src/erasure_coding/decode_reader.rs +++ b/crates/ecstore/src/erasure/coding/decode_reader.rs @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::disk::error::Error as DiskError; -use crate::erasure_codec::bridge::ErasureDecodeEngine; -use crate::get_diagnostics::{ +use crate::diagnostics::get::{ GET_OBJECT_PATH_CODEC_STREAMING, GET_READER_BUFFER_OUTPUT, GET_READER_BUFFER_PREFETCH, GET_READER_POLL_PENDING, GET_READER_POLL_READY_DATA, GET_READER_POLL_READY_EMPTY, GET_READER_POLL_READY_ERROR, GET_READER_PREFETCH_DIRECT, GET_READER_PREFETCH_EOF, GET_READER_PREFETCH_ERROR_DEFERRED, GET_READER_PREFETCH_ERROR_IMMEDIATE, GET_READER_PREFETCH_STORED, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_FILL, GET_STAGE_OUTPUT_LOCK_WAIT, GET_STAGE_OUTPUT_POLL, GET_STAGE_RECONSTRUCT, GET_STAGE_STRIPE_READ, }; +use crate::disk::error::Error as DiskError; +use crate::erasure::codec::bridge::ErasureDecodeEngine; use crate::set_disk::shard_source::{ShardStripeSource, StripeReadState}; use std::io; use std::io::ErrorKind; @@ -476,10 +476,10 @@ fn emit_data_shards(state: &StripeReadState, data_shards: usize, block_size: usi #[cfg(test)] mod tests { use super::*; - use crate::erasure_codec::bridge::{ + use crate::erasure::codec::bridge::{ CodecStreamingDecodeEngine, ErasureDecodeEngine, LegacyEcDecodeEngine, RustfsCodecDecodeEngine, }; - use crate::erasure_coding::Erasure; + use crate::erasure::coding::Erasure; use crate::set_disk::shard_source::{ShardSlot, StripeReadState}; use std::collections::VecDeque; use std::sync::Arc; diff --git a/crates/ecstore/src/erasure_coding/encode.rs b/crates/ecstore/src/erasure/coding/encode.rs similarity index 99% rename from crates/ecstore/src/erasure_coding/encode.rs rename to crates/ecstore/src/erasure/coding/encode.rs index e2ffa81cc..02f8d1496 100644 --- a/crates/ecstore/src/erasure_coding/encode.rs +++ b/crates/ecstore/src/erasure/coding/encode.rs @@ -16,9 +16,9 @@ use crate::disk::error::Error; use crate::disk::error_reduce::{ OBJECT_OP_IGNORED_ERRS, WriteQuorumFailureSummary, build_write_quorum_failure_summary, reduce_write_quorum_errs, }; -use crate::erasure_coding::BitrotWriterWrapper; -use crate::erasure_coding::Erasure; -use crate::runtime_sources; +use crate::erasure::coding::BitrotWriterWrapper; +use crate::erasure::coding::Erasure; +use crate::runtime::sources as runtime_sources; use bytes::{Bytes, BytesMut}; use futures::StreamExt; use futures::stream::FuturesUnordered; @@ -666,7 +666,7 @@ impl Erasure { #[cfg(test)] mod tests { use super::*; - use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter}; + use crate::erasure::coding::{BitrotWriterWrapper, CustomWriter}; use rustfs_rio::HardLimitReader; use rustfs_utils::HashAlgorithm; use std::io::Cursor; diff --git a/crates/ecstore/src/erasure_coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs similarity index 100% rename from crates/ecstore/src/erasure_coding/erasure.rs rename to crates/ecstore/src/erasure/coding/erasure.rs diff --git a/crates/ecstore/src/erasure_coding/heal.rs b/crates/ecstore/src/erasure/coding/heal.rs similarity index 96% rename from crates/ecstore/src/erasure_coding/heal.rs rename to crates/ecstore/src/erasure/coding/heal.rs index 4d5874b58..1ad53c599 100644 --- a/crates/ecstore/src/erasure_coding/heal.rs +++ b/crates/ecstore/src/erasure/coding/heal.rs @@ -13,10 +13,10 @@ // limitations under the License. use crate::disk::error::{Error, Result}; -use crate::erasure_coding::BitrotReader; -use crate::erasure_coding::BitrotWriterWrapper; -use crate::erasure_coding::decode::ParallelReader; -use crate::erasure_coding::encode::MultiWriter; +use crate::erasure::coding::BitrotReader; +use crate::erasure::coding::BitrotWriterWrapper; +use crate::erasure::coding::decode::ParallelReader; +use crate::erasure::coding::encode::MultiWriter; use bytes::Bytes; use tokio::io::AsyncRead; use tracing::{info, warn}; @@ -90,7 +90,7 @@ impl super::Erasure { #[cfg(test)] mod tests { use super::*; - use crate::erasure_coding::{CustomWriter, Erasure}; + use crate::erasure::coding::{CustomWriter, Erasure}; use rustfs_utils::HashAlgorithm; use std::io::Cursor; diff --git a/crates/ecstore/src/erasure_coding/mod.rs b/crates/ecstore/src/erasure/coding/mod.rs similarity index 100% rename from crates/ecstore/src/erasure_coding/mod.rs rename to crates/ecstore/src/erasure/coding/mod.rs diff --git a/crates/ecstore/src/layout/disks_layout_facade.rs b/crates/ecstore/src/erasure/mod.rs similarity index 92% rename from crates/ecstore/src/layout/disks_layout_facade.rs rename to crates/ecstore/src/erasure/mod.rs index 34fdd3729..a8dcd0a43 100644 --- a/crates/ecstore/src/layout/disks_layout_facade.rs +++ b/crates/ecstore/src/erasure/mod.rs @@ -12,4 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use crate::layout::disks_layout::*; +pub(crate) mod codec; +pub(crate) mod coding; diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index d3a599a3b..3cb3c6042 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::disk::{self, DiskAPI as _, DiskStore, error::DiskError}; -use crate::erasure_coding::{BitrotReader, BitrotWriterWrapper, CustomWriter}; +use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter}; use bytes::Bytes; use rustfs_utils::HashAlgorithm; use std::io::Cursor; diff --git a/crates/ecstore/src/io_support/mod.rs b/crates/ecstore/src/io_support/mod.rs new file mode 100644 index 000000000..7aaefea25 --- /dev/null +++ b/crates/ecstore/src/io_support/mod.rs @@ -0,0 +1,17 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) mod bitrot; +pub(crate) mod compress; +pub(crate) mod rio; diff --git a/crates/ecstore/src/layout/endpoints.rs b/crates/ecstore/src/layout/endpoints.rs index d09afc4a5..e65fc14fc 100644 --- a/crates/ecstore/src/layout/endpoints.rs +++ b/crates/ecstore/src/layout/endpoints.rs @@ -13,11 +13,11 @@ // limitations under the License. use crate::{ - global::global_rustfs_port, layout::{ disks_layout::DisksLayout, endpoint::{Endpoint, EndpointType}, }, + runtime::global::global_rustfs_port, }; use rustfs_config::{DEFAULT_UNSAFE_BYPASS_DISK_CHECK, ENV_MINIO_CI, ENV_UNSAFE_BYPASS_DISK_CHECK}; use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host}; diff --git a/crates/ecstore/src/layout/endpoints_facade.rs b/crates/ecstore/src/layout/endpoints_facade.rs deleted file mode 100644 index a29173960..000000000 --- a/crates/ecstore/src/layout/endpoints_facade.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::layout::endpoints::*; diff --git a/crates/ecstore/src/layout/pool_space.rs b/crates/ecstore/src/layout/pool_space.rs index e7806f8cb..54a84aab8 100644 --- a/crates/ecstore/src/layout/pool_space.rs +++ b/crates/ecstore/src/layout/pool_space.rs @@ -17,8 +17,8 @@ use std::slice::Iter; use crate::bucket::utils::is_meta_bucketname; use crate::disk::DiskInfo; use crate::error::{Error, Result}; -use crate::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES}; -use crate::runtime_sources; +use crate::runtime::global::{DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES}; +use crate::runtime::sources as runtime_sources; #[derive(Debug, Default, Clone)] pub struct PoolAvailableSpace { diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 0fe686eec..0e4c42d92 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -13,75 +13,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -extern crate core; - -#[path = "diagnostics/admin_server_info.rs"] -mod admin_server_info; pub mod api; -#[path = "services/batch_processor.rs"] -mod batch_processor; -#[path = "io_support/bitrot.rs"] -mod bitrot; mod bucket; mod cache_value; mod cluster; -#[path = "io_support/compress.rs"] -mod compress; mod config; +mod core; mod data_movement; -#[path = "data_movement/backpressure.rs"] -mod data_movement_backpressure; mod data_usage; +mod diagnostics; mod disk; -#[path = "layout/disks_layout_facade.rs"] -mod disks_layout; -#[path = "layout/endpoints_facade.rs"] -mod endpoints; -mod erasure_codec; -mod erasure_coding; +mod erasure; mod error; -#[path = "diagnostics/get.rs"] -mod get_diagnostics; -#[path = "runtime/global.rs"] -mod global; +mod io_support; pub(crate) mod layout; -#[path = "services/metrics_realtime.rs"] -mod metrics_realtime; -#[path = "services/notification_sys.rs"] -mod notification_sys; mod object_api; -#[path = "core/pools.rs"] -mod pools; -mod rebalance; -#[path = "io_support/rio.rs"] -mod rio; -mod rpc; -#[path = "runtime/sources.rs"] -mod runtime_sources; +mod runtime; +mod services; mod set_disk; -#[path = "core/sets.rs"] -mod sets; mod storage_api_contracts; mod store; -#[path = "store/init_format.rs"] -mod store_init; -#[path = "store/list_objects.rs"] -mod store_list_objects; -#[path = "store/utils.rs"] -mod store_utils; // pub mod checksum; mod client; mod event; -#[path = "services/event_notification.rs"] -mod event_notification; -#[cfg(test)] -#[path = "core/pools_test.rs"] -mod pools_test; -#[cfg(test)] -#[path = "core/store_test.rs"] -mod store_test; -mod tier; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use std::sync::Arc; @@ -91,7 +46,7 @@ pub type WorkloadAdmissionSnapshotProviderRef = Arc std::result::Result<(), WorkloadAdmissionSnapshotProviderRef> { - runtime_sources::set_workload_admission_snapshot_provider(provider) + runtime::sources::set_workload_admission_snapshot_provider(provider) } #[cfg(test)] @@ -99,6 +54,6 @@ mod rio_tests { #[test] fn uses_expected_rio_backend() { let expected = if cfg!(feature = "rio-v2") { "rio-v2" } else { "legacy-rio" }; - assert_eq!(crate::rio::backend_name(), expected); + assert_eq!(crate::io_support::rio::backend_name(), expected); } } diff --git a/crates/ecstore/src/metadata/bucket.rs b/crates/ecstore/src/metadata/bucket.rs index 194153eb3..3fe4ecec6 100644 --- a/crates/ecstore/src/metadata/bucket.rs +++ b/crates/ecstore/src/metadata/bucket.rs @@ -20,7 +20,7 @@ use crate::bucket::utils::deserialize; use crate::config::com::{read_config, save_config}; use crate::disk::BUCKET_META_PREFIX; use crate::error::{Error, Result}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::store::ECStore; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use rustfs_policy::policy::BucketPolicy; diff --git a/crates/ecstore/src/metadata/bucket_sys.rs b/crates/ecstore/src/metadata/bucket_sys.rs index af3f2c64c..58ad5258d 100644 --- a/crates/ecstore/src/metadata/bucket_sys.rs +++ b/crates/ecstore/src/metadata/bucket_sys.rs @@ -19,7 +19,7 @@ use crate::bucket::bucket_target_sys::BucketTargetSys; use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse}; use crate::bucket::utils::{deserialize, is_meta_bucketname}; use crate::error::{Error, Result, is_err_bucket_not_found}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::heal::HealOperations as _; use crate::store::ECStore; use futures::future::join_all; diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 66ddaaaa4..5bbefe5b0 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -16,12 +16,12 @@ use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::versioning::VersioningApi as _; use crate::config::storageclass; use crate::error::{Error, Result}; -use crate::rio::{HashReader, LimitReader}; +use crate::io_support::rio::{HashReader, LimitReader}; use crate::storage_api_contracts::{ lifecycle::{ExpirationOptions, TransitionedObject}, range::HTTPRangeSpec, }; -use crate::store_utils::clean_metadata; +use crate::store::utils::clean_metadata; use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions}; use bytes::Bytes; use http::{HeaderMap, HeaderValue}; diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 1412713ec..b5812d451 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -21,7 +21,7 @@ use sha2::Sha256; use std::collections::HashMap; use std::env; -use crate::rio::Index; +use crate::io_support::rio::Index; const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key"; @@ -123,7 +123,7 @@ fn restore_request_active(opts: &ObjectOptions) -> bool { } fn decode_compression_index(index: Option<&bytes::Bytes>) -> Option { - crate::rio::decode_compression_index_bytes(index?) + crate::io_support::rio::decode_compression_index_bytes(index?) } fn get_compressed_offsets(oi: &ObjectInfo, offset: i64) -> (i64, i64, usize, i64, u64) { @@ -270,7 +270,7 @@ struct EncryptionMaterial { key_bytes: [u8; 32], base_nonce: [u8; 12], key_kind: EncryptionKeyKind, - reader_backend: crate::rio::ReadEncryptionBackend, + reader_backend: crate::io_support::rio::ReadEncryptionBackend, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -287,7 +287,7 @@ enum ReadTransform { }, Compressed { algorithm: CompressionAlgorithm, - backend: crate::rio::ReadCompressionBackend, + backend: crate::io_support::rio::ReadCompressionBackend, decompressed_offset: usize, decompressed_length: i64, total_plaintext_size: usize, @@ -301,7 +301,7 @@ enum ReadTransform { plaintext_offset: usize, plaintext_length: i64, total_plaintext_size: usize, - compression: Option<(CompressionAlgorithm, crate::rio::ReadCompressionBackend)>, + compression: Option<(CompressionAlgorithm, crate::io_support::rio::ReadCompressionBackend)>, }, } @@ -398,7 +398,7 @@ impl ReadPlan { let (requested_offset, requested_length) = rs.get_offset_length(plaintext_size)?; #[cfg(feature = "rio-v2")] { - if encryption_backend == crate::rio::ReadEncryptionBackend::Legacy { + if encryption_backend == crate::io_support::rio::ReadEncryptionBackend::Legacy { ( 0, oi.size, @@ -526,7 +526,7 @@ impl ReadPlan { decompressed_length, total_plaintext_size, } => { - let dec_reader = crate::rio::decompression_reader(reader, algorithm, backend); + let dec_reader = crate::io_support::rio::decompression_reader(reader, algorithm, backend); #[cfg(feature = "rio-v2")] let dec_reader = StreamConsumer::new(dec_reader); let final_reader: Box = if decompressed_offset > 0 @@ -588,13 +588,13 @@ impl ReadPlan { let _ = sequence_number; let decrypted_reader: Box = if is_multipart { match material.key_kind { - EncryptionKeyKind::Object => crate::rio::decrypt_multipart_reader_with_object_key( + EncryptionKeyKind::Object => crate::io_support::rio::decrypt_multipart_reader_with_object_key( reader, material.key_bytes, part_numbers, sequence_number, ), - EncryptionKeyKind::Direct => crate::rio::decrypt_multipart_reader( + EncryptionKeyKind::Direct => crate::io_support::rio::decrypt_multipart_reader( reader, material.key_bytes, material.base_nonce, @@ -606,9 +606,9 @@ impl ReadPlan { } else { match material.key_kind { EncryptionKeyKind::Object => { - crate::rio::decrypt_reader_with_object_key(reader, material.key_bytes, sequence_number) + crate::io_support::rio::decrypt_reader_with_object_key(reader, material.key_bytes, sequence_number) } - EncryptionKeyKind::Direct => crate::rio::decrypt_reader( + EncryptionKeyKind::Direct => crate::io_support::rio::decrypt_reader( reader, material.key_bytes, material.base_nonce, @@ -627,7 +627,8 @@ impl ReadPlan { let final_reader: Box = if let Some((algo, compression_backend)) = compression { - let decompressed_reader = crate::rio::decompression_reader(decrypted_reader, algo, compression_backend); + let decompressed_reader = + crate::io_support::rio::decompression_reader(decrypted_reader, algo, compression_backend); #[cfg(feature = "rio-v2")] let decompressed_reader = StreamConsumer::new(decompressed_reader); if plaintext_offset > 0 || plaintext_length != total_plaintext_size_i64 { @@ -1276,7 +1277,7 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap) -> R key_bytes: object_key, base_nonce: [0u8; 12], key_kind: EncryptionKeyKind::Object, - reader_backend: crate::rio::ReadEncryptionBackend::V2, + reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2, }); } @@ -1284,7 +1285,7 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap) -> R key_bytes, base_nonce: generate_ssec_nonce(&oi.bucket, &oi.name), key_kind: EncryptionKeyKind::Direct, - reader_backend: crate::rio::ReadEncryptionBackend::Legacy, + reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy, }) } @@ -1308,7 +1309,7 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap let kms_context: Option> = None; let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref()); - let decrypted_key = if let Some(service) = crate::runtime_sources::object_encryption_service().await { + let decrypted_key = if let Some(service) = crate::runtime::sources::object_encryption_service().await { #[cfg(feature = "rio-v2")] let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) { service.decrypt_legacy_data_key(&encrypted_dek).await @@ -1331,7 +1332,7 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap key_bytes: object_key, base_nonce: [0u8; 12], key_kind: EncryptionKeyKind::Object, - reader_backend: crate::rio::ReadEncryptionBackend::V2, + reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2, }); } @@ -1349,7 +1350,7 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap key_bytes: decrypted_key, base_nonce, key_kind: EncryptionKeyKind::Direct, - reader_backend: crate::rio::ReadEncryptionBackend::Legacy, + reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy, }) } @@ -1932,12 +1933,13 @@ mod tests { async fn test_ranged_decompress_reader_with_rio_v2_s2_stream() { let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); let mut compressed = Vec::new(); - crate::rio::CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + crate::io_support::rio::CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) .read_to_end(&mut compressed) .await .expect("compress plaintext into rio_v2 stream"); - let decompress_reader = crate::rio::DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let decompress_reader = + crate::io_support::rio::DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); let mut ranged_reader = RangedDecompressReader::new(decompress_reader, 5, 7, plaintext.len()).expect("create ranged reader"); @@ -2322,7 +2324,7 @@ mod tests { let user_defined = { let object_key = [0x41; 32]; let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt managed object"); @@ -2344,7 +2346,7 @@ mod tests { #[cfg(not(feature = "rio-v2"))] let user_defined = { let base_nonce = [0x11; 12]; - crate::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) + crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) .read_to_end(&mut encrypted) .await .expect("encrypt managed object"); @@ -2399,7 +2401,7 @@ mod tests { let user_defined = { let object_key = [0x43; 32]; let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt managed ranged object"); @@ -2421,7 +2423,7 @@ mod tests { #[cfg(not(feature = "rio-v2"))] let user_defined = { let base_nonce = [0x13; 12]; - crate::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) + crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) .read_to_end(&mut encrypted) .await .expect("encrypt managed ranged object"); @@ -2486,7 +2488,7 @@ mod tests { let user_defined = { let object_key = [0x42; 32]; let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt managed object with local fallback key"); @@ -2508,7 +2510,7 @@ mod tests { #[cfg(not(feature = "rio-v2"))] let user_defined = { let base_nonce = [0x12; 12]; - crate::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) + crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) .read_to_end(&mut encrypted) .await .expect("encrypt managed object with local fallback key"); @@ -2561,7 +2563,7 @@ mod tests { let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); let mut encrypted = Vec::new(); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt managed object"); @@ -2614,7 +2616,7 @@ mod tests { #[tokio::test] async fn test_get_object_reader_compressed_range_returns_physical_offset_from_index() { - let mut index = crate::rio::Index::new(); + let mut index = crate::io_support::rio::Index::new(); index.add(0, 0).unwrap(); index.add(1_048_576, 2_097_152).unwrap(); @@ -2635,7 +2637,7 @@ mod tests { ..Default::default() }; - let decoded = crate::rio::decode_compression_index_bytes(object_info.parts[0].index.as_ref().unwrap()) + let decoded = crate::io_support::rio::decode_compression_index_bytes(object_info.parts[0].index.as_ref().unwrap()) .expect("headerless MinIO-style compression index should decode"); let (compressed_offset, uncompressed_offset) = decoded.find(2_097_152).expect("seek into decoded index"); assert!(compressed_offset > 0); @@ -2668,7 +2670,7 @@ mod tests { #[tokio::test] async fn test_read_plan_compressed_range_tracks_storage_and_visible_offsets() { - let mut index = crate::rio::Index::new(); + let mut index = crate::io_support::rio::Index::new(); index.add(0, 0).unwrap(); index.add(1_048_576, 2_097_152).unwrap(); @@ -2744,10 +2746,10 @@ mod tests { #[cfg(feature = "rio-v2")] #[tokio::test] async fn test_read_plan_accepts_minio_headerless_compression_index() { - let mut index = crate::rio::Index::new(); + let mut index = crate::io_support::rio::Index::new(); index.add(0, 0).unwrap(); index.add(1_048_576, 2_097_152).unwrap(); - let headerless_index = crate::rio::compression_index_storage_bytes(&index); + let headerless_index = crate::io_support::rio::compression_index_storage_bytes(&index); assert!( !headerless_index.starts_with(&[0x50, 0x2A, 0x4D, 0x18]), "rio_v2 should store MinIO-style headerless compression indexes" @@ -2776,7 +2778,7 @@ mod tests { end: 2_097_161, }; - let decoded = crate::rio::decode_compression_index_bytes(object_info.parts[0].index.as_ref().unwrap()) + let decoded = crate::io_support::rio::decode_compression_index_bytes(object_info.parts[0].index.as_ref().unwrap()) .expect("headerless MinIO-style compression index should decode"); let (compressed_offset, uncompressed_offset) = decoded.find(range.start).expect("seek into decoded index"); assert!(compressed_offset > 0); @@ -2798,11 +2800,11 @@ mod tests { #[cfg(feature = "rio-v2")] #[test] fn test_get_compressed_offsets_aligns_encrypted_ranges_to_dare_packages() { - let mut index = crate::rio::Index::new(); + let mut index = crate::io_support::rio::Index::new(); index.add(0, 0).unwrap(); index.add(200_000, 2_097_152).unwrap(); - let stored_index = crate::rio::compression_index_storage_bytes(&index); - let expected_comp_off = crate::rio::decode_compression_index_bytes(&stored_index) + let stored_index = crate::io_support::rio::compression_index_storage_bytes(&index); + let expected_comp_off = crate::io_support::rio::decode_compression_index_bytes(&stored_index) .expect("decode stored index") .find(2_097_152) .expect("find offset in stored index") @@ -2823,7 +2825,7 @@ mod tests { ("x-amz-server-side-encryption-customer-original-size".to_string(), "4194304".to_string()), ( "x-minio-internal-compression".to_string(), - crate::rio::compression_metadata_value(CompressionAlgorithm::default()), + crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), ), ("x-minio-internal-actual-size".to_string(), "4194304".to_string()), ])), @@ -2904,7 +2906,7 @@ mod tests { let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); let mut encrypted = Vec::new(); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt object with rio-v2 object key"); @@ -3029,7 +3031,7 @@ mod tests { let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); let mut encrypted = Vec::new(); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt large ranged object"); @@ -3159,7 +3161,7 @@ mod tests { #[cfg(feature = "rio-v2")] #[tokio::test] async fn test_get_object_reader_uses_compression_index_for_encrypted_ranges() { - use crate::rio::TryGetIndex; + use crate::io_support::rio::TryGetIndex; let plaintext: Vec = (0..(10 * 1024 * 1024 + 123_456)) .map(|i| (((i as u64).wrapping_mul(1_103_515_245).wrapping_add(12_345) >> 16) & 0xFF) as u8) @@ -3169,8 +3171,10 @@ mod tests { let bucket = "bucket"; let object = "compressed-large-object"; let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); - let mut compressor = - crate::rio::CompressReader::with_encrypted_padding(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressor = crate::io_support::rio::CompressReader::with_encrypted_padding( + Cursor::new(plaintext.clone()), + CompressionAlgorithm::default(), + ); let mut compressed = Vec::new(); compressor .read_to_end(&mut compressed) @@ -3181,9 +3185,9 @@ mod tests { .try_get_index() .cloned() .expect("large rio_v2 encrypted+compressed object should expose a compression index"); - let stored_index = crate::rio::compression_index_storage_bytes(&index); - let decoded_index = - crate::rio::decode_compression_index_bytes(&stored_index).expect("decode stored encrypted compression index"); + let stored_index = crate::io_support::rio::compression_index_storage_bytes(&index); + let decoded_index = crate::io_support::rio::decode_compression_index_bytes(&stored_index) + .expect("decode stored encrypted compression index"); let range = HTTPRangeSpec { is_suffix_length: false, @@ -3206,7 +3210,7 @@ mod tests { assert!(expected_decrypt_skip >= 0); let mut encrypted = Vec::new(); - crate::rio::EncryptReader::new_with_object_key(Cursor::new(compressed.clone()), object_key) + crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(compressed.clone()), object_key) .read_to_end(&mut encrypted) .await .expect("encrypt compressed plaintext"); @@ -3221,7 +3225,7 @@ mod tests { assert!(chunk_offset + 4 + chunk_len <= compressed.len()); let mut decrypted_tail = Vec::new(); - crate::rio::DecryptReader::new_with_object_key_and_sequence( + crate::io_support::rio::DecryptReader::new_with_object_key_and_sequence( Cursor::new(encrypted[expected_storage_offset..].to_vec()), object_key, expected_sequence_number, @@ -3235,7 +3239,7 @@ mod tests { "package-aligned decryption plus decrypt_skip must land on the indexed S2 chunk boundary" ); - let mut direct_reader = crate::rio::DecompressReader::new( + let mut direct_reader = crate::io_support::rio::DecompressReader::new( Cursor::new(decrypted_tail[expected_decrypt_skip as usize..].to_vec()), CompressionAlgorithm::default(), ); @@ -3279,7 +3283,7 @@ mod tests { ), ( "x-minio-internal-compression".to_string(), - crate::rio::compression_metadata_value(CompressionAlgorithm::default()), + crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), ), ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()), ])), diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index e22fae71d..ba8f12265 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -222,14 +222,14 @@ impl ObjectInfo { Ok((algorithm, compressed)) } - pub fn compression_read_plan(&self) -> Result<(CompressionAlgorithm, crate::rio::ReadCompressionBackend, bool)> { + pub fn compression_read_plan(&self) -> Result<(CompressionAlgorithm, crate::io_support::rio::ReadCompressionBackend, bool)> { let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); if let Some(scheme) = scheme { - let (algorithm, backend) = crate::rio::compression_scheme_to_read_plan(&scheme)?; + let (algorithm, backend) = crate::io_support::rio::compression_scheme_to_read_plan(&scheme)?; Ok((algorithm, backend, true)) } else { - Ok((CompressionAlgorithm::None, crate::rio::ReadCompressionBackend::Legacy, false)) + Ok((CompressionAlgorithm::None, crate::io_support::rio::ReadCompressionBackend::Legacy, false)) } } diff --git a/crates/ecstore/src/rpc/mod.rs b/crates/ecstore/src/rpc/mod.rs deleted file mode 100644 index 3c9d686b2..000000000 --- a/crates/ecstore/src/rpc/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(test)] -pub(crate) use crate::cluster::rpc::TcpHttpInternodeDataTransport; -pub(crate) use crate::cluster::rpc::heal_bucket_local_on_disks; -pub use crate::cluster::rpc::{ - LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, RemoteClient, RemoteDisk, S3PeerSys, - SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, TonicInterceptor, build_auth_headers, - build_internode_data_transport_from_env, gen_signature_headers, gen_tonic_signature_interceptor, - node_service_time_out_client, node_service_time_out_client_no_auth, verify_rpc_signature, -}; -pub(crate) use crate::cluster::rpc::{client, context_propagation, internode_data_transport, runtime_sources}; diff --git a/crates/ecstore/src/runtime/global.rs b/crates/ecstore/src/runtime/global.rs index cf1b37f6f..e79cbd096 100644 --- a/crates/ecstore/src/runtime/global.rs +++ b/crates/ecstore/src/runtime/global.rs @@ -16,10 +16,10 @@ use crate::bucket::bandwidth::monitor::Monitor; use crate::{ bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys, disk::DiskStore, - endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, - event_notification::EventNotifier, + layout::endpoints::{EndpointServerPools, PoolEndpoints, SetupType}, + services::event_notification::EventNotifier, + services::tier::tier::TierConfigMgr, store::ECStore, - tier::tier::TierConfigMgr, }; use lazy_static::lazy_static; use rustfs_lock::client::LockClient; diff --git a/crates/ecstore/src/runtime/mod.rs b/crates/ecstore/src/runtime/mod.rs new file mode 100644 index 000000000..276571f3d --- /dev/null +++ b/crates/ecstore/src/runtime/mod.rs @@ -0,0 +1,16 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) mod global; +pub(crate) mod sources; diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index 67eccffe3..948fb4d40 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -21,17 +21,14 @@ use std::{ use crate::bucket::bandwidth::monitor::Monitor; use crate::disk::endpoint::Endpoint; use crate::{ - batch_processor::{GlobalBatchProcessors, get_global_processors}, bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, GLOBAL_ExpiryState, GLOBAL_TransitionState, TransitionState}, bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}, bucket::replication::{DynReplicationPool, GLOBAL_REPLICATION_POOL, GLOBAL_REPLICATION_STATS, ReplicationStats}, config::{get_global_storage_class, set_global_storage_class, storageclass}, disk::{DiskAPI, DiskOption, DiskStore, new_disk}, - endpoints::EndpointServerPools, - endpoints::SetupType, error::Result, - event_notification::EventNotifier, - global::{ + layout::endpoints::{EndpointServerPools, SetupType}, + runtime::global::{ GLOBAL_BOOT_TIME, GLOBAL_EventNotifier, GLOBAL_IsErasureSD, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_LifecycleSys, GLOBAL_LocalNodeName, GLOBAL_RootDiskThreshold, GLOBAL_TierConfigMgr, TypeLocalDiskSetDrives, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints, @@ -39,9 +36,11 @@ use crate::{ init_global_bucket_monitor, is_dist_erasure, is_erasure, is_first_cluster_node_local, resolve_object_store_handle, set_global_deployment_id, set_global_lock_client, set_global_lock_clients, set_object_layer, update_erasure_type, }, - notification_sys::{NotificationSys, get_global_notification_sys}, + services::batch_processor::{GlobalBatchProcessors, get_global_processors}, + services::event_notification::EventNotifier, + services::notification_sys::{NotificationSys, get_global_notification_sys}, + services::tier::tier::TierConfigMgr, store::ECStore, - tier::tier::TierConfigMgr, }; use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST}; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; diff --git a/crates/ecstore/src/services/metrics_realtime.rs b/crates/ecstore/src/services/metrics_realtime.rs index c1d9d6e13..e5a5210d5 100644 --- a/crates/ecstore/src/services/metrics_realtime.rs +++ b/crates/ecstore/src/services/metrics_realtime.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin_server_info::get_local_server_property; -use crate::runtime_sources; +use crate::diagnostics::admin_server_info::get_local_server_property; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::admin::StorageAdminApi; use chrono::Utc; use rustfs_common::{heal_channel::DriveState, metrics::global_metrics}; diff --git a/crates/ecstore/src/services/mod.rs b/crates/ecstore/src/services/mod.rs new file mode 100644 index 000000000..c250df3df --- /dev/null +++ b/crates/ecstore/src/services/mod.rs @@ -0,0 +1,20 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) mod batch_processor; +pub(crate) mod event_notification; +pub(crate) mod metrics_realtime; +pub(crate) mod notification_sys; +pub(crate) mod rebalance; +pub(crate) mod tier; diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 2801bade7..9e46709ab 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin_server_info::get_commit_id; -use crate::endpoints::EndpointServerPools; +use crate::cluster::rpc::PeerRestClient; +use crate::diagnostics::admin_server_info::get_commit_id; use crate::error::{Error, Result}; -use crate::metrics_realtime::{CollectMetricsOpts, MetricType}; -use crate::rebalance::RebalSaveOpt; -use crate::rpc::PeerRestClient; -use crate::runtime_sources; +use crate::layout::endpoints::EndpointServerPools; +use crate::runtime::sources as runtime_sources; +use crate::services::metrics_realtime::{CollectMetricsOpts, MetricType}; +use crate::services::rebalance::RebalSaveOpt; use crate::storage_api_contracts::admin::StorageAdminApi; use futures::future::join_all; use lazy_static::lazy_static; @@ -232,7 +232,12 @@ impl NotificationSys { futures.push(async move { if let Some(client) = client { match client - .signal_service(crate::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC, &sub_sys, false, SystemTime::UNIX_EPOCH) + .signal_service( + crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC, + &sub_sys, + false, + SystemTime::UNIX_EPOCH, + ) .await { Ok(_) => NotificationPeerErr { @@ -261,7 +266,7 @@ impl NotificationSys { futures.push(async move { if let Some(client) = client { match client - .signal_service(crate::rpc::SERVICE_SIGNAL_REFRESH_CONFIG, "", false, SystemTime::UNIX_EPOCH) + .signal_service(crate::cluster::rpc::SERVICE_SIGNAL_REFRESH_CONFIG, "", false, SystemTime::UNIX_EPOCH) .await { Ok(_) => NotificationPeerErr { diff --git a/crates/ecstore/src/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs similarity index 100% rename from crates/ecstore/src/rebalance/control.rs rename to crates/ecstore/src/services/rebalance/control.rs diff --git a/crates/ecstore/src/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs similarity index 98% rename from crates/ecstore/src/rebalance/entry.rs rename to crates/ecstore/src/services/rebalance/entry.rs index 8545633b5..a484a6bbc 100644 --- a/crates/ecstore/src/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -29,11 +29,11 @@ use super::{ EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_ENTRY, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, }; +use crate::core::pools::ListCallback; use crate::data_movement; -use crate::data_movement_backpressure::{self, DataMovementOperation}; +use crate::data_movement::backpressure::{self, DataMovementOperation}; use crate::error::{Error, Result}; use crate::object_api::{GetObjectReader, ObjectOptions}; -use crate::pools::ListCallback; use crate::set_disk::SetDisks; use crate::storage_api_contracts::object::ObjectOperations as _; use crate::store::ECStore; @@ -113,7 +113,7 @@ impl ECStore { let mut expired: usize = 0; let mut stats_updates = Vec::with_capacity(fivs.versions.len()); for version in fivs.versions.iter() { - if crate::pools::should_skip_lifecycle_for_data_movement( + if crate::core::pools::should_skip_lifecycle_for_data_movement( self.clone(), &bucket, version, @@ -401,7 +401,7 @@ impl ECStore { return; } - if let Err(err) = data_movement_backpressure::wait_for_data_movement_admission( + if let Err(err) = backpressure::wait_for_data_movement_admission( DataMovementOperation::Rebalance, pool_index, &callback_rx, diff --git a/crates/ecstore/src/rebalance/meta.rs b/crates/ecstore/src/services/rebalance/meta.rs similarity index 100% rename from crates/ecstore/src/rebalance/meta.rs rename to crates/ecstore/src/services/rebalance/meta.rs diff --git a/crates/ecstore/src/rebalance/migration.rs b/crates/ecstore/src/services/rebalance/migration.rs similarity index 100% rename from crates/ecstore/src/rebalance/migration.rs rename to crates/ecstore/src/services/rebalance/migration.rs diff --git a/crates/ecstore/src/rebalance/mod.rs b/crates/ecstore/src/services/rebalance/mod.rs similarity index 100% rename from crates/ecstore/src/rebalance/mod.rs rename to crates/ecstore/src/services/rebalance/mod.rs diff --git a/crates/ecstore/src/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs similarity index 99% rename from crates/ecstore/src/rebalance/rebalance_unit_tests.rs rename to crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs index 6d6609fd4..dee8fb2d8 100644 --- a/crates/ecstore/src/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs @@ -2408,22 +2408,22 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() { ..Default::default() }; - let endpoint_pools: crate::endpoints::EndpointServerPools = Vec::new().into(); + let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into(); let store = Arc::new(crate::store::ECStore { id: uuid::Uuid::new_v4(), disk_map: std::collections::HashMap::new(), pools: Vec::new(), - peer_sys: crate::rpc::S3PeerSys::new(&endpoint_pools), - pool_meta: tokio::sync::RwLock::new(crate::pools::PoolMeta::default()), + peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools), + pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()), rebalance_meta: tokio::sync::RwLock::new(Some(active_meta)), decommission_cancelers: tokio::sync::RwLock::new(Vec::new()), start_gate: tokio::sync::Mutex::new(()), pool_meta_save_gate: tokio::sync::Mutex::new(()), - local_disk_map: crate::global::GLOBAL_LOCAL_DISK_MAP.clone(), - local_disk_id_map: crate::global::GLOBAL_LOCAL_DISK_ID_MAP.clone(), - local_disk_set_drives: crate::global::GLOBAL_LOCAL_DISK_SET_DRIVES.clone(), - tier_config_mgr: crate::tier::tier::TierConfigMgr::new(), - event_notifier: crate::event_notification::EventNotifier::new(), + local_disk_map: crate::runtime::global::GLOBAL_LOCAL_DISK_MAP.clone(), + local_disk_id_map: crate::runtime::global::GLOBAL_LOCAL_DISK_ID_MAP.clone(), + local_disk_set_drives: crate::runtime::global::GLOBAL_LOCAL_DISK_SET_DRIVES.clone(), + tier_config_mgr: crate::services::tier::tier::TierConfigMgr::new(), + event_notifier: crate::services::event_notification::EventNotifier::new(), bucket_monitor: std::sync::OnceLock::new(), }); diff --git a/crates/ecstore/src/rebalance/runtime.rs b/crates/ecstore/src/services/rebalance/runtime.rs similarity index 99% rename from crates/ecstore/src/rebalance/runtime.rs rename to crates/ecstore/src/services/rebalance/runtime.rs index 5dbd6b511..70d3df0f8 100644 --- a/crates/ecstore/src/rebalance/runtime.rs +++ b/crates/ecstore/src/services/rebalance/runtime.rs @@ -14,7 +14,7 @@ use super::{ RebalanceBucketOutcome, }; use crate::error::{Error, Result}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::store::ECStore; use std::collections::HashSet; use std::sync::Arc; diff --git a/crates/ecstore/src/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs similarity index 100% rename from crates/ecstore/src/rebalance/types.rs rename to crates/ecstore/src/services/rebalance/types.rs diff --git a/crates/ecstore/src/rebalance/worker.rs b/crates/ecstore/src/services/rebalance/worker.rs similarity index 99% rename from crates/ecstore/src/rebalance/worker.rs rename to crates/ecstore/src/services/rebalance/worker.rs index cb80e4ee7..a6bb01c2e 100644 --- a/crates/ecstore/src/rebalance/worker.rs +++ b/crates/ecstore/src/services/rebalance/worker.rs @@ -5,12 +5,12 @@ use super::{ REBALANCE_MIGRATION_RETRY_BASE_DELAY, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, Result, }; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; +use crate::core::pools::ListCallback; use crate::disk::error::DiskError; use crate::error::{ Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down, }; -use crate::pools::ListCallback; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use rand::RngExt as _; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; diff --git a/crates/ecstore/src/tier/mod.rs b/crates/ecstore/src/services/tier/mod.rs similarity index 100% rename from crates/ecstore/src/tier/mod.rs rename to crates/ecstore/src/services/tier/mod.rs diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs similarity index 97% rename from crates/ecstore/src/tier/tier.rs rename to crates/ecstore/src/services/tier/tier.rs index 1abb72e67..1df0b5ffd 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -38,21 +38,21 @@ use tracing::{debug, error, info, warn}; use crate::client::admin_handler_utils::AdminError; use crate::error::{Error, Result, StorageError}; -use crate::storage_api_contracts::{ - object::{DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, ObjectIO, ObjectOperations, ObjectToDelete}, - range::HTTPRangeSpec, -}; -use crate::tier::{ +use crate::services::tier::{ tier_admin::TierCreds, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_ALREADY_EXISTS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND}, warm_backend::{check_warm_backend, new_warm_backend}, }; +use crate::storage_api_contracts::{ + object::{DeletedObject, EcstoreObjectIO, EcstoreObjectOperations, ObjectIO, ObjectOperations, ObjectToDelete}, + range::HTTPRangeSpec, +}; use crate::{ config::com::{CONFIG_PREFIX, read_config}, disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, - runtime_sources, + runtime::sources as runtime_sources, store::ECStore, }; use rustfs_filemeta::FileInfo; @@ -266,7 +266,7 @@ fn tier_type_from_hint(hint: Option<&str>) -> Option { } } -fn external_tier_s3_from_internal(s3: &crate::tier::tier_config::TierS3) -> ExternalTierS3 { +fn external_tier_s3_from_internal(s3: &crate::services::tier::tier_config::TierS3) -> ExternalTierS3 { ExternalTierS3 { endpoint: s3.endpoint.clone(), access_key: s3.access_key.clone(), @@ -536,7 +536,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul .s3 .as_ref() .ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?; - cfg.s3 = Some(crate::tier::tier_config::TierS3 { + cfg.s3 = Some(crate::services::tier::tier_config::TierS3 { name: cfg.name.clone(), endpoint: s3.endpoint.clone(), access_key: s3.access_key.clone(), @@ -557,7 +557,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul .azure .as_ref() .ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?; - cfg.azure = Some(crate::tier::tier_config::TierAzure { + cfg.azure = Some(crate::services::tier::tier_config::TierAzure { name: cfg.name.clone(), endpoint: az.endpoint.clone(), access_key: az.account_name.clone(), @@ -566,7 +566,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul prefix: az.prefix.clone(), region: az.region.clone(), storage_class: az.storage_class.clone(), - sp_auth: crate::tier::tier_config::ServicePrincipalAuth { + sp_auth: crate::services::tier::tier_config::ServicePrincipalAuth { tenant_id: az.sp_auth.tenant_id.clone(), client_id: az.sp_auth.client_id.clone(), client_secret: az.sp_auth.client_secret.clone(), @@ -578,7 +578,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul .gcs .as_ref() .ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?; - cfg.gcs = Some(crate::tier::tier_config::TierGCS { + cfg.gcs = Some(crate::services::tier::tier_config::TierGCS { name: cfg.name.clone(), endpoint: gcs.endpoint.clone(), creds: gcs.creds.clone(), @@ -593,7 +593,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul .compatible_backend .as_ref() .ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?; - cfg.minio = Some(crate::tier::tier_config::TierMinIO { + cfg.minio = Some(crate::services::tier::tier_config::TierMinIO { name: cfg.name.clone(), endpoint: m.endpoint.clone(), access_key: m.access_key.clone(), @@ -605,7 +605,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul } TierType::RustFS => { let m = decode_legacy_s3_like(&cfg.name, &ext)?; - cfg.rustfs = Some(crate::tier::tier_config::TierRustFS { + cfg.rustfs = Some(crate::services::tier::tier_config::TierRustFS { name: cfg.name.clone(), endpoint: m.endpoint, access_key: m.access_key, @@ -618,7 +618,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul } TierType::Aliyun => { let m = decode_legacy_s3_like(&cfg.name, &ext)?; - cfg.aliyun = Some(crate::tier::tier_config::TierAliyun { + cfg.aliyun = Some(crate::services::tier::tier_config::TierAliyun { name: cfg.name.clone(), endpoint: m.endpoint, access_key: m.access_key, @@ -630,7 +630,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul } TierType::Tencent => { let m = decode_legacy_s3_like(&cfg.name, &ext)?; - cfg.tencent = Some(crate::tier::tier_config::TierTencent { + cfg.tencent = Some(crate::services::tier::tier_config::TierTencent { name: cfg.name.clone(), endpoint: m.endpoint, access_key: m.access_key, @@ -642,7 +642,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul } TierType::Huaweicloud => { let m = decode_legacy_s3_like(&cfg.name, &ext)?; - cfg.huaweicloud = Some(crate::tier::tier_config::TierHuaweicloud { + cfg.huaweicloud = Some(crate::services::tier::tier_config::TierHuaweicloud { name: cfg.name.clone(), endpoint: m.endpoint, access_key: m.access_key, @@ -654,7 +654,7 @@ fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Resul } TierType::R2 => { let m = decode_legacy_s3_like(&cfg.name, &ext)?; - cfg.r2 = Some(crate::tier::tier_config::TierR2 { + cfg.r2 = Some(crate::services::tier::tier_config::TierR2 { name: cfg.name.clone(), endpoint: m.endpoint, access_key: m.access_key, @@ -1342,7 +1342,7 @@ mod tests { version: "v1".to_string(), tier_type: TierType::S3, name: name.to_string(), - s3: Some(crate::tier::tier_config::TierS3 { + s3: Some(crate::services::tier::tier_config::TierS3 { name: name.to_string(), endpoint: "https://example-s3.invalid".to_string(), access_key: "ak".to_string(), @@ -1393,7 +1393,7 @@ mod tests { version: "v1".to_string(), tier_type: TierType::RustFS, name: "COLD-B".to_string(), - rustfs: Some(crate::tier::tier_config::TierRustFS { + rustfs: Some(crate::services::tier::tier_config::TierRustFS { name: "COLD-B".to_string(), endpoint: "https://example-compat.invalid".to_string(), access_key: "ak".to_string(), diff --git a/crates/ecstore/src/tier/tier_admin.rs b/crates/ecstore/src/services/tier/tier_admin.rs similarity index 100% rename from crates/ecstore/src/tier/tier_admin.rs rename to crates/ecstore/src/services/tier/tier_admin.rs diff --git a/crates/ecstore/src/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs similarity index 100% rename from crates/ecstore/src/tier/tier_config.rs rename to crates/ecstore/src/services/tier/tier_config.rs diff --git a/crates/ecstore/src/tier/tier_gen.rs b/crates/ecstore/src/services/tier/tier_gen.rs similarity index 93% rename from crates/ecstore/src/tier/tier_gen.rs rename to crates/ecstore/src/services/tier/tier_gen.rs index 3efe1891e..400466b47 100644 --- a/crates/ecstore/src/tier/tier_gen.rs +++ b/crates/ecstore/src/services/tier/tier_gen.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::tier::tier::TierConfigMgr; +use crate::services::tier::tier::TierConfigMgr; #[allow(dead_code)] impl TierConfigMgr { diff --git a/crates/ecstore/src/tier/tier_handlers.rs b/crates/ecstore/src/services/tier/tier_handlers.rs similarity index 100% rename from crates/ecstore/src/tier/tier_handlers.rs rename to crates/ecstore/src/services/tier/tier_handlers.rs diff --git a/crates/ecstore/src/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend.rs rename to crates/ecstore/src/services/tier/warm_backend.rs index 3a3b0e965..0bbec9c0b 100644 --- a/crates/ecstore/src/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -24,7 +24,7 @@ use crate::client::{ transition_api::{ReadCloser, ReaderImpl}, }; use crate::error::is_err_bucket_not_found; -use crate::tier::{ +use crate::services::tier::{ tier::ERR_TIER_TYPE_UNSUPPORTED, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR}, diff --git a/crates/ecstore/src/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_aliyun.rs rename to crates/ecstore/src/services/tier/warm_backend_aliyun.rs index 515a54152..7ab709ec0 100644 --- a/crates/ecstore/src/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierAliyun, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_azure.rs rename to crates/ecstore/src/services/tier/warm_backend_azure.rs index bd5cffbb9..f05045f44 100644 --- a/crates/ecstore/src/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierAzure, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_gcs.rs rename to crates/ecstore/src/services/tier/warm_backend_gcs.rs index f2d19b898..425e2507f 100644 --- a/crates/ecstore/src/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -33,7 +33,7 @@ use crate::client::{ api_put_object::PutObjectOptions, transition_api::{Options, ReadCloser, ReaderImpl}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierGCS, warm_backend::{WarmBackend, WarmBackendGetOpts}, }; diff --git a/crates/ecstore/src/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_huaweicloud.rs rename to crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index 29df0bef4..2fab4b9d7 100644 --- a/crates/ecstore/src/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierHuaweicloud, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_minio.rs rename to crates/ecstore/src/services/tier/warm_backend_minio.rs index 044d377ca..6767d5065 100644 --- a/crates/ecstore/src/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierMinIO, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/tier/warm_backend_r2.rs b/crates/ecstore/src/services/tier/warm_backend_r2.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_r2.rs rename to crates/ecstore/src/services/tier/warm_backend_r2.rs index f9f2ecb7c..d2842a08a 100644 --- a/crates/ecstore/src/tier/warm_backend_r2.rs +++ b/crates/ecstore/src/services/tier/warm_backend_r2.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierR2, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_rustfs.rs rename to crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 9fb2fb513..fe0b114ee 100644 --- a/crates/ecstore/src/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierRustFS, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_s3.rs rename to crates/ecstore/src/services/tier/warm_backend_s3.rs index 5dd46a222..8fac23841 100644 --- a/crates/ecstore/src/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -32,7 +32,7 @@ use crate::client::{ }; use crate::error::ErrorResponse; use crate::error::error_resp_to_object_err; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierS3, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, }; diff --git a/crates/ecstore/src/tier/warm_backend_s3sdk.rs b/crates/ecstore/src/services/tier/warm_backend_s3sdk.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_s3sdk.rs rename to crates/ecstore/src/services/tier/warm_backend_s3sdk.rs index c4b36c9da..fc1790e0a 100644 --- a/crates/ecstore/src/tier/warm_backend_s3sdk.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3sdk.rs @@ -35,7 +35,7 @@ use crate::client::{ }; use crate::error::ErrorResponse; use crate::error::error_resp_to_object_err; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierS3, warm_backend::{WarmBackend, WarmBackendGetOpts}, }; diff --git a/crates/ecstore/src/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs similarity index 99% rename from crates/ecstore/src/tier/warm_backend_tencent.rs rename to crates/ecstore/src/services/tier/warm_backend_tencent.rs index febfb4bc6..f1e6e94f2 100644 --- a/crates/ecstore/src/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -27,7 +27,7 @@ use crate::client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use crate::tier::{ +use crate::services::tier::{ tier_config::TierTencent, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, warm_backend_s3::WarmBackendS3, diff --git a/crates/ecstore/src/set_disk/heal.rs b/crates/ecstore/src/set_disk/heal.rs index a47c723de..751794af0 100644 --- a/crates/ecstore/src/set_disk/heal.rs +++ b/crates/ecstore/src/set_disk/heal.rs @@ -129,14 +129,14 @@ impl SetDisks { let erasure = if !latest_meta.deleted && !latest_meta.is_remote() { // Initialize erasure coding; use legacy mode for old-version files - erasure_coding::Erasure::new_with_options( + crate::erasure::coding::Erasure::new_with_options( latest_meta.erasure.data_blocks, latest_meta.erasure.parity_blocks, latest_meta.erasure.block_size, latest_meta.uses_legacy_checksum, ) } else { - erasure_coding::Erasure::default() + crate::erasure::coding::Erasure::default() }; result.object_size = diff --git a/crates/ecstore/src/set_disk/lock.rs b/crates/ecstore/src/set_disk/lock.rs index 6b3bc2621..cde9c9af7 100644 --- a/crates/ecstore/src/set_disk/lock.rs +++ b/crates/ecstore/src/set_disk/lock.rs @@ -14,7 +14,7 @@ use super::*; use crate::disk::health_state::DriveMembershipSnapshot; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; impl SetDisks { pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String { @@ -417,7 +417,7 @@ impl SetDisks { #[cfg(test)] mod tests { use super::*; - use crate::store_init::save_format_file; + use crate::store::init_format::save_format_file; use tempfile::TempDir; use tokio::sync::RwLock; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 6718d5b6d..7cc5d16ee 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -15,8 +15,6 @@ #![allow(unused_imports)] #![allow(unused_variables)] -use crate::batch_processor::AsyncBatchProcessor; -use crate::bitrot::{create_bitrot_reader, create_bitrot_writer}; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::bucket::metadata_sys; use crate::bucket::object_lock::objectlock_sys::check_retention_for_modification; @@ -24,6 +22,11 @@ use crate::bucket::replication::check_replicate_delete; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl}; +use crate::cluster::rpc::heal_bucket_local_on_disks; +use crate::diagnostics::get::{ + GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, + GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error, record_get_object_pipeline_failure, +}; use crate::disk::error_reduce::{ BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs, reduce_write_quorum_errs, @@ -33,19 +36,16 @@ use crate::disk::{ conv_part_err_to_int, has_part_err, }; use crate::disk::{STORAGE_FORMAT_FILE, count_part_not_success}; -use crate::erasure_codec::bridge::{ +use crate::erasure::codec::bridge::{ CodecStreamingDecodeEngine, GET_CODEC_STREAMING_ENGINE_LEGACY, GET_CODEC_STREAMING_ENGINE_RUSTFS, }; -use crate::erasure_coding; +use crate::erasure::coding; use crate::error::{Error, Result, is_err_version_not_found}; use crate::error::{GenericError, ObjectApiError, is_err_object_not_found}; -use crate::get_diagnostics::{ - GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, - GET_STAGE_EMIT, GET_STAGE_METADATA, classify_storage_error, record_get_object_pipeline_failure, -}; +use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_writer}; use crate::object_api::ObjectOptions; -use crate::rpc::heal_bucket_local_on_disks; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; +use crate::services::batch_processor::AsyncBatchProcessor; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -56,7 +56,7 @@ use crate::storage_api_contracts::{ object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete}, range::HTTPRangeSpec, }; -use crate::store_utils::is_reserved_or_invalid_bucket; +use crate::store::utils::is_reserved_or_invalid_bucket; use crate::{ bucket::lifecycle::bucket_lifecycle_ops::{ LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts, @@ -69,10 +69,10 @@ use crate::{ UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, }, error::{StorageError, to_object_err}, - // event::name::EventName, - event_notification::{EventArgs, send_event}, object_api::{GetObjectReader, ObjectInfo, PutObjReader}, - store_init::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file}, + // event::name::EventName, + services::event_notification::{EventArgs, send_event}, + store::init_format::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file}, }; use bytes::Bytes; use bytesize::ByteSize; @@ -174,7 +174,7 @@ const ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_MULTIP const DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 128 * 1024 * 1024; static CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); -use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; +use crate::io_support::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; pub const MAX_PARTS_COUNT: usize = 10000; @@ -403,7 +403,7 @@ fn get_codec_streaming_engine() -> GetCodecStreamingEngine { } } -fn build_get_codec_streaming_decode_engine(erasure: erasure_coding::Erasure) -> std::io::Result { +fn build_get_codec_streaming_decode_engine(erasure: coding::Erasure) -> std::io::Result { match get_codec_streaming_engine() { GetCodecStreamingEngine::Legacy => Ok(CodecStreamingDecodeEngine::legacy(erasure)), GetCodecStreamingEngine::Rustfs => CodecStreamingDecodeEngine::rustfs(&erasure), @@ -1417,7 +1417,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap()); let result: Result = async { - let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = + crate::erasure::coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let is_inline_buffer = runtime_sources::storage_class_should_inline(erasure.shard_file_size(data.size()), opts.versioned); @@ -1573,7 +1574,10 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string()); } - let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes); + let index_op = data + .stream + .try_get_index() + .map(crate::io_support::rio::compression_index_storage_bytes); //TODO: userDefined @@ -3746,7 +3750,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp()); let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}")); - let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = + crate::erasure::coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let writer_setup_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now); let mut writers = Vec::with_capacity(shuffle_disks.len()); @@ -3869,7 +3874,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ))); } - let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes); + let index_op = data + .stream + .try_get_index() + .map(crate::io_support::rio::compression_index_storage_bytes); let mut etag = data.stream.try_resolve_etag().unwrap_or_default(); @@ -4850,7 +4858,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { } }; - let endpoints = crate::endpoints::Endpoints::from(self.set_endpoints.clone()); + let endpoints = crate::layout::endpoints::Endpoints::from(self.set_endpoints.clone()); let before_drives = crate::layout::set_heal::formats_to_drives_info(&endpoints, &formats, &errs); let mut result = HealResultItem { heal_item_type: HealItemType::Metadata.to_string(), @@ -5778,14 +5786,14 @@ mod tests { use crate::disk::endpoint::Endpoint; use crate::disk::error::DiskError; use crate::disk::health_state::RuntimeDriveHealthState; - use crate::endpoints::SetupType; + use crate::layout::endpoints::SetupType; use crate::object_api::ObjectInfo; use crate::storage_api_contracts::{ heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart, namespace::NamespaceLocking as _, object::ObjectOperations as _, }; - use crate::store_init::save_format_file; - use crate::store_list_objects::ListPathOptions; + use crate::store::init_format::save_format_file; + use crate::store::list_objects::ListPathOptions; use rustfs_filemeta::ErasureInfo; use rustfs_filemeta::MetaCacheEntry; use rustfs_filemeta::ReplicationState; @@ -7141,7 +7149,7 @@ mod tests { let err = set_disks .list_path( CancellationToken::new(), - crate::store_list_objects::ListPathOptions { + crate::store::list_objects::ListPathOptions { bucket: "bucket".to_string(), recursive: true, ..Default::default() @@ -7822,7 +7830,7 @@ mod tests { fi.size = payload.len() as i64; fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None); - let erasure = erasure_coding::Erasure::new_with_options( + let erasure = crate::erasure::coding::Erasure::new_with_options( fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 4ec237c49..0c82cb12e 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use crate::get_diagnostics::{ +use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, @@ -1688,7 +1688,7 @@ impl SetDisks { object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds" ); - let erasure = erasure_coding::Erasure::new_with_options( + let erasure = crate::erasure::coding::Erasure::new_with_options( fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, @@ -1978,7 +1978,7 @@ impl SetDisks { return Err(Error::other("codec streaming reader only supports single-part plain objects")); } - let erasure = erasure_coding::Erasure::new_with_options( + let erasure = crate::erasure::coding::Erasure::new_with_options( fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, @@ -2075,7 +2075,7 @@ impl SetDisks { .await; } - let source = erasure_coding::decode::ParallelReader::new_with_metrics_path_and_read_costs( + let source = crate::erasure::coding::decode::ParallelReader::new_with_metrics_path_and_read_costs( readers, erasure.clone(), 0, @@ -2084,8 +2084,8 @@ impl SetDisks { read_costs, ); let engine = build_get_codec_streaming_decode_engine(erasure)?; - let reader = erasure_coding::decode_reader::ErasureDecodeReader::new(source, engine, part_length)?; - Ok(Box::new(erasure_coding::decode_reader::SyncErasureDecodeReader::new(reader))) + let reader = crate::erasure::coding::decode_reader::ErasureDecodeReader::new(source, engine, part_length)?; + Ok(Box::new(crate::erasure::coding::decode_reader::SyncErasureDecodeReader::new(reader))) } } @@ -3017,7 +3017,7 @@ mod tests { #[test] fn codec_streaming_decode_engine_builder_selects_rustfs() { temp_env::with_var(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS), || { - let erasure = erasure_coding::Erasure::new(4, 2, 32); + let erasure = crate::erasure::coding::Erasure::new(4, 2, 32); let engine = build_get_codec_streaming_decode_engine(erasure).expect("engine should be built"); assert!(matches!(engine, CodecStreamingDecodeEngine::Rustfs(_))); diff --git a/crates/ecstore/src/set_disk/shard_source.rs b/crates/ecstore/src/set_disk/shard_source.rs index f7887671a..34b3336a6 100644 --- a/crates/ecstore/src/set_disk/shard_source.rs +++ b/crates/ecstore/src/set_disk/shard_source.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::disk::error::Error; -use crate::get_diagnostics::{ +use crate::diagnostics::get::{ GET_SHARD_READ_COST_LOCAL, GET_SHARD_READ_COST_REMOTE, GET_SHARD_READ_COST_SAME_NODE, GET_SHARD_READ_COST_UNKNOWN, }; +use crate::disk::error::Error; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ShardReadCost { diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index fbeb67b1b..6c15d810f 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -17,7 +17,7 @@ use crate::bucket::{ metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix}, utils::is_meta_bucketname, }; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::namespace::NamespaceLocking as _; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index c12dffd75..3eab8b9e0 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -13,9 +13,9 @@ // limitations under the License. use super::*; +use crate::core::pools::local_decommission_queue_prefix; use crate::error::is_err_decommission_running; -use crate::pools::local_decommission_queue_prefix; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use tracing::{debug, error, info, warn}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -25,7 +25,7 @@ const EVENT_DECOMMISSION_RESUME_FAILED: &str = "decommission_resume_failed"; const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry"; const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status"; -fn pool_first_endpoint_is_local(pool: &crate::endpoints::PoolEndpoints) -> bool { +fn pool_first_endpoint_is_local(pool: &crate::layout::endpoints::PoolEndpoints) -> bool { pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local) } @@ -202,7 +202,7 @@ impl ECStore { // periodic monitoring until format loading succeeds. Startup RPC // failures can still spawn recovery probes for peers that come up // after this node. - let (disks, errs) = store_init::init_disks( + let (disks, errs) = init_format::init_disks( &pool_eps.endpoints, &DiskOption { cleanup: true, @@ -217,7 +217,7 @@ impl ECStore { let mut times = 0; let mut interval = 1; loop { - match store_init::connect_load_init_formats( + match init_format::connect_load_init_formats( pool_first_is_local, &disks, pool_eps.set_count, @@ -446,7 +446,7 @@ impl ECStore { crate::bucket::lifecycle::bucket_lifecycle_ops::init_background_stale_multipart_upload_cleanup(self.clone()); TransitionState::init(self.clone()).await; - crate::tier::tier::try_migrate_tiering_config(self.clone()).await; + crate::services::tier::tier::try_migrate_tiering_config(self.clone()).await; if let Err(err) = runtime_sources::init_tier_config_mgr(self.clone()).await { info!("TierConfigMgr init error: {}", err); @@ -470,11 +470,11 @@ mod tests { should_resume_local_decommission, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay, }; use crate::{ + core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus}, disk::endpoint::Endpoint, - endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, error::StorageError, - pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus}, - rebalance::RebalanceMeta, + layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, + services::rebalance::RebalanceMeta, }; use std::time::Duration; use time::OffsetDateTime; diff --git a/crates/ecstore/src/store/init_format.rs b/crates/ecstore/src/store/init_format.rs index 6b7969c4a..fccd112d5 100644 --- a/crates/ecstore/src/store/init_format.rs +++ b/crates/ecstore/src/store/init_format.rs @@ -23,7 +23,7 @@ use crate::{ format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, new_disk, }, - endpoints::Endpoints, + layout::endpoints::Endpoints, }; use futures::future::join_all; use rustfs_config::server_config::KVS; diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 760389cc9..c582c854e 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -16,6 +16,7 @@ use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::utils::check_list_objs_args; use crate::bucket::versioning::VersioningApi; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; +use crate::core::sets::Sets; use crate::disk::error::DiskError; use crate::disk::{DiskInfo, DiskStore}; use crate::error::{ @@ -23,7 +24,6 @@ use crate::error::{ }; use crate::object_api::{ObjectInfo, ObjectOptions}; use crate::set_disk::SetDisks; -use crate::sets::Sets; use crate::storage_api_contracts::{ list::{ ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, @@ -32,7 +32,7 @@ use crate::storage_api_contracts::{ object::ObjectOperations as _, }; use crate::store::ECStore; -use crate::store_utils::is_reserved_or_invalid_bucket; +use crate::store::utils::is_reserved_or_invalid_bucket; use futures::future::join_all; use rand::seq::SliceRandom; use rustfs_filemeta::{ @@ -3337,13 +3337,13 @@ mod test { // use crate::disk::MetaCacheEntries; // use crate::disk::MetaCacheEntry; // use crate::disk::WalkDirOptions; - // use crate::endpoints::EndpointServerPools; + // use crate::layout::endpoints::EndpointServerPools; // use crate::error::Error; // use crate::metacache::writer::MetacacheReader; // use crate::set_disk::SetDisks; - // use crate::store_list_objects::ListPathOptions; - // use crate::store_list_objects::WalkOptions; - // use crate::store_list_objects::WalkVersionsSortOrder; + // use crate::store::list_objects::ListPathOptions; + // use crate::store::list_objects::WalkOptions; + // use crate::store::list_objects::WalkVersionsSortOrder; // use futures::future::join_all; // use rustfs_lock::namespace_lock::NsLockMap; // use tokio::sync::broadcast; diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 8cd23a03e..380dccf7b 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -32,7 +32,9 @@ use crate::bucket::utils::check_object_args; use crate::bucket::utils::check_put_object_args; use crate::bucket::utils::check_put_object_part_args; use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname}; +use crate::cluster::rpc::{RemoteClient, S3PeerSys}; use crate::config::storageclass; +use crate::core::pools::PoolMeta; use crate::disk::endpoint::{Endpoint, EndpointType}; use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions}; use crate::error::{Error, Result}; @@ -40,12 +42,11 @@ use crate::error::{ StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_invalid_upload_id, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, to_object_err, }; -use crate::event_notification::EventNotifier; -use crate::global::{DISK_RESERVE_FRACTION, TypeLocalDiskSetDrives}; -use crate::pools::PoolMeta; -use crate::rebalance::RebalanceMeta; -use crate::rpc::RemoteClient; -use crate::runtime_sources; +use crate::runtime::global::{DISK_RESERVE_FRACTION, TypeLocalDiskSetDrives}; +use crate::runtime::sources as runtime_sources; +use crate::services::event_notification::EventNotifier; +use crate::services::rebalance::RebalanceMeta; +use crate::services::tier::tier::TierConfigMgr; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -53,16 +54,13 @@ use crate::storage_api_contracts::{ object::{DeletedObject, ObjectToDelete}, range::HTTPRangeSpec, }; -use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; -use crate::tier::tier::TierConfigMgr; +use crate::store::init_format::{check_disk_fatal_errs, ec_drives_no_config}; use crate::{ bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata}, + core::sets::Sets, disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET}, - endpoints::EndpointServerPools, + layout::endpoints::EndpointServerPools, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, - rpc::S3PeerSys, - sets::Sets, - store_init, }; use futures::future::join_all; use http::HeaderMap; @@ -157,11 +155,14 @@ const MAX_UPLOADS_LIST: usize = 10000; mod bucket; mod heal; mod init; +pub(crate) mod init_format; mod list; +pub(crate) mod list_objects; mod multipart; mod object; mod peer; mod rebalance; +pub(crate) mod utils; use peer::init_local_peer; pub use peer::{ @@ -246,7 +247,7 @@ impl ECStore { /// service singletons. The globals remain the source of truth. impl ECStore { /// Get the notification system - pub fn notification_system(&self) -> Option<&'static crate::notification_sys::NotificationSys> { + pub fn notification_system(&self) -> Option<&'static crate::services::notification_sys::NotificationSys> { runtime_sources::notification_sys() } @@ -266,7 +267,7 @@ impl ECStore { } /// Get the tier config manager - pub fn tier_config_mgr(&self) -> Arc> { + pub fn tier_config_mgr(&self) -> Arc> { runtime_sources::global_tier_config_mgr() } @@ -772,9 +773,9 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore { #[cfg(test)] mod tests { use super::*; - use crate::endpoints::{Endpoints, PoolEndpoints}; - use crate::global::{GLOBAL_LOCAL_DISK_ID_MAP, reset_local_disk_test_state}; - use crate::store_init::{connect_load_init_formats, init_disks}; + use crate::layout::endpoints::{Endpoints, PoolEndpoints}; + use crate::runtime::global::{GLOBAL_LOCAL_DISK_ID_MAP, reset_local_disk_test_state}; + use crate::store::init_format::{connect_load_init_formats, init_disks}; use serial_test::serial; use tempfile::TempDir; diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index b8d623c89..21441efea 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use tracing::{debug, error}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 3960a0df2..06eb916d8 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -14,7 +14,7 @@ use super::*; use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space}; -use crate::runtime_sources; +use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _}; pub(in crate::store) mod support; use support::{ diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index a0b4c4a55..6d1b45e59 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,20 +5,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-ecstore-cluster-rpc-metadata-layout` +- Branch: `overtrue/arch-ecstore-services-domain-batch` - Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/API-243/API-244/API-245/API-246/API-247/API-248/API-249/API-250/API-251/API-252/API-253/API-254/CTX-002`. - Current baseline also includes API-255 from PR #3923, API-256 from PR #3925, and CFG-009 from PR #3927. -- Current phase PR: E-029/E-030 ECStore cluster RPC and metadata owner layout. -- Based on: E-028 batch branch while PR #3931 is pending; rebase onto - `origin/main` after E-019 through E-028 merge before opening this PR. +- Current phase PR: E-035/E-008 ECStore services domain batch. +- Based on: E-034 branch while PR #3932 and follow-up ECStore owner PRs are + pending; rebase onto `origin/main` after E-019 through E-034 merge before + opening this PR. - PR type for this branch: `pure-move`. -- Runtime behavior changes: none expected for E-029/E-030; production changes - are module file moves that keep the existing `rpc`, `bucket::metadata`, - `bucket::metadata_sys`, and `set_disk::metadata` facade paths. -- Rust code changes: move ECStore RPC implementation files under the - `cluster/rpc` owner directory and move bucket/set-disk metadata support files - under the `metadata` owner directory without function-body changes. +- Runtime behavior changes: none expected for E-035/E-008; production changes + only move the ECStore rebalance and tier owner modules under `services` while + retaining the public `api::rebalance` and `api::tier` facades. +- Rust code changes: move the ECStore root `rebalance` and `tier` directories + into `services`, remove the root `lib.rs` declarations, and migrate + crate-internal imports to `services::{rebalance,tier}` without function-body + changes. - CI/script changes: reject restoring ECStore root `store.rs`, `set_disk.rs`, `store_list_objects.rs`, `store_utils.rs`, `store_init.rs`, `disks_layout.rs`, `endpoints.rs`, `storage_api_contracts.rs`, @@ -26,11 +28,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block `notification_sys.rs`, `data_movement.rs`, or `data_movement_backpressure.rs`, `admin_server_info.rs`, `data_usage.rs`, `get_diagnostics.rs`, `global.rs`, `runtime_sources.rs`, `bitrot.rs`, - `compress.rs`, `rio.rs`, `error.rs`, `rebalance.rs`, `pools.rs`, - `sets.rs`, `pools_test.rs`, or `store_test.rs`, keep the ECStore cluster root - module as a re-export-only owner facade, reject restoring ECStore root RPC - support files, keep the ECStore root RPC module as a re-export-only facade, - and reject restoring old bucket/set-disk metadata owner paths; + `compress.rs`, `rio.rs`, `erasure_codec`, `erasure_coding`, `error.rs`, + `rebalance.rs`, `pools.rs`, `sets.rs`, `pools_test.rs`, or `store_test.rs`, + keep the ECStore cluster root module as a re-export-only owner facade, reject + restoring ECStore root RPC modules or root `lib.rs` declaration, reject + ECStore-internal `crate::rpc` consumers, reject restoring old bucket/set-disk + metadata owner paths, and reject + restoring ECStore root `lib.rs` owner path shims for core, store, layout, data + movement, diagnostics, services, I/O support, and runtime modules; reject + restoring ECStore root rebalance or tier service-domain modules or + ECStore-internal `crate::rebalance`/`crate::tier` consumers; lock completed config model ownership against restoring old `rustfs_ecstore::config` model/accessor compatibility paths, and lock ECStore public facades against re-exporting the moved server-config symbols; @@ -77,8 +84,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block rebalance facade modules after E-026, and reject restoring ECStore root core runtime/test modules after E-027, keep the ECStore cluster root module as a re-export-only owner facade after E-028, reject restoring ECStore root RPC - support files after E-029, and reject restoring old bucket/set-disk metadata - owner paths after E-030. + support files after E-029, reject restoring old bucket/set-disk metadata owner + paths after E-030, reject restoring ECStore root owner path shims for + diagnostics, services, I/O support, and runtime modules after E-031, and + reject restoring ECStore root owner path shims for core, store, layout, and + data movement modules after E-032, reject restoring ECStore root erasure + codec or coding modules after E-033/E-004, and reject restoring ECStore root + RPC facade modules or ECStore-internal `crate::rpc` consumers after E-034, + and reject restoring ECStore root rebalance/tier service-domain modules or + ECStore-internal `crate::rebalance`/`crate::tier` consumers after E-035/E-008. ## Phase 0 Tasks @@ -9266,6 +9280,72 @@ Notes: object I/O, notification persistence, S3 Select reads, IAM error mapping, observability metrics, test/fuzz semantics, or ECStore definitions. +- Issue #660 E-032 current slice: + - `cargo check -p rustfs-ecstore --lib`: passed. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs-ecstore --lib layout`: passed. + - `cargo test -p rustfs-ecstore --lib store`: passed. + - `cargo test -p rustfs-ecstore --lib data_movement`: passed. + - `cargo test -p rustfs-ecstore --lib core::`: passed. + - `cargo fmt --all --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `git diff --check overtrue/arch-ecstore-root-owner-facade-shrink...HEAD`: + passed. + - ECStore root owner path shim scan: passed; no root `lib.rs` path shims remain + for core, store, layout, or data movement owner modules. + - Rust risk scan: passed; no new unwrap/expect, numeric casts, string error + public APIs, boxed public errors, production println/eprintln, or relaxed + ordering introduced in changed Rust files. + +- Issue #660 E-033/E-004 current slice: + - `cargo check -p rustfs-ecstore --lib`: passed. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs-ecstore --lib erasure`: passed. + - `cargo fmt --all --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `git diff --check overtrue/arch-ecstore-core-store-layout-shims...HEAD`: + passed. + - ECStore root erasure module scan: passed; no root `erasure_codec` or + `erasure_coding` module directories or root `lib.rs` declarations remain. + - Rust risk scan: passed; no new unwrap/expect, numeric casts, string error + public APIs, boxed public errors, production println/eprintln, or relaxed + ordering introduced in changed Rust files. + +- Issue #660 E-034/E-006/E-009 current slice: + - `cargo check -p rustfs-ecstore --lib`: passed. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs-ecstore --lib cluster::rpc`: passed. + - `cargo fmt --all --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `git diff --check overtrue/arch-ecstore-erasure-owner-layout...HEAD`: + passed. + - ECStore RPC root consumer scan: passed; no ECStore-internal `crate::rpc` + consumers remain. + - ECStore root RPC facade scan: passed; no root `rpc` module or root + `lib.rs` declaration remains. + - Rust risk scan: passed; no new unwrap/expect, numeric casts, string error + public APIs, boxed public errors, production println/eprintln, or relaxed + ordering introduced in changed Rust files. + +- Issue #660 E-035/E-008 current slice: + - `cargo check -p rustfs-ecstore --lib`: passed. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs-ecstore --lib services::rebalance`: passed. + - `cargo test -p rustfs-ecstore --lib services::tier`: passed. + - `cargo fmt --all --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `git diff --check overtrue/arch-ecstore-cluster-rpc-facade-shrink...HEAD`: + passed. + - ECStore services domain scan: passed; no root `rebalance`/`tier` modules + or ECStore-internal `crate::rebalance`/`crate::tier` consumers remain. + - Rust risk scan: passed; no new unwrap/expect, numeric casts, string error + public APIs, boxed public errors, production println/eprintln, or relaxed + ordering introduced in changed Rust files. + ## Handoff Notes - Continue with larger consumer-migration batches outside the cleaned diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index de2ca31f1..02e1edfc8 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -138,12 +138,16 @@ ECSTORE_ROOT_DATA_MOVEMENT_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_data_moveme ECSTORE_ROOT_USAGE_DIAGNOSTICS_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_usage_diagnostics_module_hits.txt" ECSTORE_ROOT_RUNTIME_GLOBAL_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_runtime_global_module_hits.txt" ECSTORE_ROOT_IO_SUPPORT_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_io_support_module_hits.txt" +ECSTORE_ROOT_ERASURE_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_erasure_module_hits.txt" ECSTORE_ROOT_ERROR_REBALANCE_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_error_rebalance_module_hits.txt" +ECSTORE_ROOT_SERVICE_DOMAIN_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_service_domain_module_hits.txt" +ECSTORE_ROOT_SERVICE_DOMAIN_IMPL_HITS_FILE="${TMP_DIR}/ecstore_root_service_domain_impl_hits.txt" ECSTORE_ROOT_CORE_RUNTIME_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_core_runtime_module_hits.txt" ECSTORE_CLUSTER_ROOT_IMPL_HITS_FILE="${TMP_DIR}/ecstore_cluster_root_impl_hits.txt" ECSTORE_ROOT_RPC_SUPPORT_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_rpc_support_module_hits.txt" ECSTORE_ROOT_RPC_IMPL_HITS_FILE="${TMP_DIR}/ecstore_root_rpc_impl_hits.txt" ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE="${TMP_DIR}/ecstore_old_metadata_owner_path_hits.txt" +ECSTORE_ROOT_OWNER_PATH_SHIM_HITS_FILE="${TMP_DIR}/ecstore_root_owner_path_shim_hits.txt" ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt" RUSTFS_LOCAL_COMPAT_OWNER_SELF_PATH_HITS_FILE="${TMP_DIR}/rustfs_local_compat_owner_self_path_hits.txt" RUSTFS_ROOT_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_compat_relative_consumer_hits.txt" @@ -363,42 +367,42 @@ require_source_line \ "startup IAM shim crate-private module" require_source_line \ "crates/ecstore/src/lib.rs" \ - "mod disks_layout;" \ - "ECStore legacy disks-layout compatibility module crate-private visibility" + "mod core;" \ + "ECStore core owner module crate-private visibility" require_source_line \ "crates/ecstore/src/lib.rs" \ - "mod endpoints;" \ - "ECStore legacy endpoint compatibility module crate-private visibility" + "mod data_movement;" \ + "ECStore data movement owner module crate-private visibility" +require_source_line \ + "crates/ecstore/src/lib.rs" \ + "mod erasure;" \ + "ECStore erasure owner module crate-private visibility" require_source_line \ "crates/ecstore/src/lib.rs" \ "mod cluster;" \ "ECStore cluster control-plane owner module crate-private visibility" +require_source_line \ + "crates/ecstore/src/lib.rs" \ + "pub(crate) mod layout;" \ + "ECStore layout owner module crate-private visibility" require_source_line \ "crates/ecstore/src/api/mod.rs" \ "pub mod cluster {" \ "ECStore cluster control-plane public facade" for ecstore_private_module in \ - admin_server_info \ bucket \ cache_value \ client \ - compress \ config \ data_usage \ + diagnostics \ disk \ error \ - event_notification \ - global \ - metrics_realtime \ - notification_sys \ - pools \ - rebalance \ - rio \ - rpc \ + io_support \ + runtime \ + services \ set_disk \ - store \ - store_utils \ - tier; do + store; do require_source_line \ "crates/ecstore/src/lib.rs" \ "mod ${ecstore_private_module};" \ @@ -2912,6 +2916,20 @@ if [[ -s "$ECSTORE_ROOT_IO_SUPPORT_MODULE_HITS_FILE" ]]; then report_failure "ECStore I/O support modules must stay under the io_support owner directory: $(paste -sd '; ' "$ECSTORE_ROOT_IO_SUPPORT_MODULE_HITS_FILE")" fi +( + cd "$ROOT_DIR" + { + [[ -e crates/ecstore/src/erasure_codec ]] && printf '%s\n' 'crates/ecstore/src/erasure_codec' + [[ -e crates/ecstore/src/erasure_coding ]] && printf '%s\n' 'crates/ecstore/src/erasure_coding' + rg -n --with-filename '^mod erasure_(codec|coding);|#\[path = "erasure_(codec|coding)/' crates/ecstore/src/lib.rs || true + true + } +) >"$ECSTORE_ROOT_ERASURE_MODULE_HITS_FILE" + +if [[ -s "$ECSTORE_ROOT_ERASURE_MODULE_HITS_FILE" ]]; then + report_failure "ECStore erasure modules must stay under the erasure owner directory: $(paste -sd '; ' "$ECSTORE_ROOT_ERASURE_MODULE_HITS_FILE")" +fi + ( cd "$ROOT_DIR" { @@ -2925,6 +2943,28 @@ if [[ -s "$ECSTORE_ROOT_ERROR_REBALANCE_MODULE_HITS_FILE" ]]; then report_failure "ECStore error and rebalance modules must stay under owner directories: $(paste -sd '; ' "$ECSTORE_ROOT_ERROR_REBALANCE_MODULE_HITS_FILE")" fi +( + cd "$ROOT_DIR" + { + [[ -e crates/ecstore/src/rebalance ]] && printf '%s\n' 'crates/ecstore/src/rebalance' + [[ -e crates/ecstore/src/tier ]] && printf '%s\n' 'crates/ecstore/src/tier' + rg -n --with-filename '^\s*mod\s+(rebalance|tier);|#\[path = "(rebalance|tier)/' crates/ecstore/src/lib.rs || true + true + } +) >"$ECSTORE_ROOT_SERVICE_DOMAIN_MODULE_HITS_FILE" + +if [[ -s "$ECSTORE_ROOT_SERVICE_DOMAIN_MODULE_HITS_FILE" ]]; then + report_failure "ECStore rebalance and tier modules must stay under the services owner directory: $(paste -sd '; ' "$ECSTORE_ROOT_SERVICE_DOMAIN_MODULE_HITS_FILE")" +fi + +rg -n --with-filename 'crate::(rebalance|tier)\b' \ + "${ROOT_DIR}/crates/ecstore/src" \ + >"$ECSTORE_ROOT_SERVICE_DOMAIN_IMPL_HITS_FILE" || true + +if [[ -s "$ECSTORE_ROOT_SERVICE_DOMAIN_IMPL_HITS_FILE" ]]; then + report_failure "ECStore internal rebalance and tier consumers must use the services owner path: $(paste -sd '; ' "$ECSTORE_ROOT_SERVICE_DOMAIN_IMPL_HITS_FILE")" +fi + ( cd "$ROOT_DIR" { @@ -2960,20 +3000,22 @@ fi [[ -e crates/ecstore/src/rpc/remote_disk.rs ]] && printf '%s\n' 'crates/ecstore/src/rpc/remote_disk.rs' [[ -e crates/ecstore/src/rpc/remote_locker.rs ]] && printf '%s\n' 'crates/ecstore/src/rpc/remote_locker.rs' [[ -e crates/ecstore/src/rpc/runtime_sources.rs ]] && printf '%s\n' 'crates/ecstore/src/rpc/runtime_sources.rs' + [[ -e crates/ecstore/src/rpc/mod.rs ]] && printf '%s\n' 'crates/ecstore/src/rpc/mod.rs' + rg -n --with-filename '^\s*mod\s+rpc;' crates/ecstore/src/lib.rs || true true } ) >"$ECSTORE_ROOT_RPC_SUPPORT_MODULE_HITS_FILE" if [[ -s "$ECSTORE_ROOT_RPC_SUPPORT_MODULE_HITS_FILE" ]]; then - report_failure "ECStore RPC support modules must stay under the cluster/rpc owner directory: $(paste -sd '; ' "$ECSTORE_ROOT_RPC_SUPPORT_MODULE_HITS_FILE")" + report_failure "ECStore RPC modules must stay under the cluster/rpc owner directory: $(paste -sd '; ' "$ECSTORE_ROOT_RPC_SUPPORT_MODULE_HITS_FILE")" fi -rg -n --with-filename '^(pub(?:\(crate\))? )?(struct|enum|fn|trait) |^mod ' \ - "${ROOT_DIR}/crates/ecstore/src/rpc/mod.rs" \ +rg -n --with-filename 'crate::rpc\b' \ + "${ROOT_DIR}/crates/ecstore/src" \ >"$ECSTORE_ROOT_RPC_IMPL_HITS_FILE" || true if [[ -s "$ECSTORE_ROOT_RPC_IMPL_HITS_FILE" ]]; then - report_failure "ECStore root rpc module must only re-export cluster/rpc owner symbols: $(paste -sd '; ' "$ECSTORE_ROOT_RPC_IMPL_HITS_FILE")" + report_failure "ECStore internal RPC consumers must use the cluster/rpc owner path: $(paste -sd '; ' "$ECSTORE_ROOT_RPC_IMPL_HITS_FILE")" fi ( @@ -2991,6 +3033,14 @@ if [[ -s "$ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE" ]]; then report_failure "ECStore metadata modules must stay under the metadata owner directory: $(paste -sd '; ' "$ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE")" fi +rg -n --with-filename '#\[path = "(data_movement|layout|core|store|diagnostics|services|io_support|runtime)/' \ + "${ROOT_DIR}/crates/ecstore/src/lib.rs" \ + >"$ECSTORE_ROOT_OWNER_PATH_SHIM_HITS_FILE" || true + +if [[ -s "$ECSTORE_ROOT_OWNER_PATH_SHIM_HITS_FILE" ]]; then + report_failure "ECStore root lib must not restore owner path shims for data_movement/layout/core/store/diagnostics/services/io_support/runtime modules: $(paste -sd '; ' "$ECSTORE_ROOT_OWNER_PATH_SHIM_HITS_FILE")" +fi + cat >"$ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE" <<'EOF' EOF sort -o "$ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE" "$ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE"