From a6b3e4f5d653ac6b69155548ea508151af92834f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 5 Jul 2026 03:01:14 +0800 Subject: [PATCH] refactor: use canonical Rust module paths (#4269) --- AGENTS.md | 3 ++ .../internode-transport-adapter-rfc.md | 24 ++++++------ .../transport-metrics-and-baseline.md | 2 +- .../bucket.rs => bucket/metadata.rs} | 0 .../bucket_sys.rs => bucket/metadata_sys.rs} | 0 .../metadata_test.rs} | 0 crates/ecstore/src/bucket/mod.rs | 3 -- .../set_disk.rs => set_disk/metadata.rs} | 0 crates/ecstore/src/set_disk/mod.rs | 1 - crates/heal/tests/endpoint_index_test.rs | 2 - .../tests/endpoint_index_test/storage_api.rs | 21 ---------- crates/heal/tests/heal_bug_fixes_test.rs | 1 - .../tests/heal_bug_fixes_test/storage_api.rs | 21 ---------- crates/heal/tests/heal_integration_test.rs | 1 - .../heal_integration_test/storage_api.rs | 27 ------------- crates/heal/tests/storage_api.rs | 31 +++++++++++++++ docs/architecture/global-state-inventory.md | 2 +- rustfs/src/storage/rpc/node_service.rs | 6 --- .../storage/rpc/{ => node_service}/bucket.rs | 0 .../storage/rpc/{ => node_service}/disk.rs | 0 .../storage/rpc/{ => node_service}/event.rs | 0 .../storage/rpc/{ => node_service}/health.rs | 0 .../storage/rpc/{ => node_service}/lock.rs | 0 .../storage/rpc/{ => node_service}/metrics.rs | 0 scripts/check_architecture_migration_rules.sh | 38 ++++++++----------- 25 files changed, 64 insertions(+), 119 deletions(-) rename crates/ecstore/src/{metadata/bucket.rs => bucket/metadata.rs} (100%) rename crates/ecstore/src/{metadata/bucket_sys.rs => bucket/metadata_sys.rs} (100%) rename crates/ecstore/src/{metadata/bucket_test.rs => bucket/metadata_test.rs} (100%) rename crates/ecstore/src/{metadata/set_disk.rs => set_disk/metadata.rs} (100%) delete mode 100644 crates/heal/tests/endpoint_index_test/storage_api.rs delete mode 100644 crates/heal/tests/heal_bug_fixes_test/storage_api.rs delete mode 100644 crates/heal/tests/heal_integration_test/storage_api.rs create mode 100644 crates/heal/tests/storage_api.rs rename rustfs/src/storage/rpc/{ => node_service}/bucket.rs (100%) rename rustfs/src/storage/rpc/{ => node_service}/disk.rs (100%) rename rustfs/src/storage/rpc/{ => node_service}/event.rs (100%) rename rustfs/src/storage/rpc/{ => node_service}/health.rs (100%) rename rustfs/src/storage/rpc/{ => node_service}/lock.rs (100%) rename rustfs/src/storage/rpc/{ => node_service}/metrics.rs (100%) diff --git a/AGENTS.md b/AGENTS.md index 8301a185b..895a4b5fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,9 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a - Prefer direct, local code over extracting one-off helpers. - Extract a helper only when logic is reused or the extraction materially clarifies a non-trivial flow. +- Use Rust's default module file layout (`mod foo;` with `foo.rs` or `foo/mod.rs`/`foo/*.rs`). + Avoid `#[path = "..."]` for module inclusion; move files into the canonical module tree instead. + If an unavoidable generated-code, FFI, or test-fixture exception remains, keep it local and document why the canonical layout cannot work. - Solve only the requested problem; do not add speculative features, configurability, or adjacent improvements. - Prefer editing existing code over rewriting files or reshaping unrelated logic. - Modify only what is required and remove only artifacts introduced by your own changes. diff --git a/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md b/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md index ec861a0cd..215adbe52 100644 --- a/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md +++ b/crates/ecstore/docs/internode-transport/internode-transport-adapter-rfc.md @@ -132,10 +132,10 @@ They should remain on gRPC/TCP: | Area | Client/server code | Examples | Notes | | --- | --- | --- | --- | -| Bucket peer ops | `crates/ecstore/src/rpc/peer_s3_client.rs`, `rustfs/src/storage/rpc/bucket.rs` | `MakeBucket`, `ListBucket`, `DeleteBucket`, `GetBucketInfo`, `HealBucket` | Small metadata/control payloads. | -| Locking | `crates/ecstore/src/rpc/remote_locker.rs`, `rustfs/src/storage/rpc/lock.rs` | `Lock`, `UnLock`, `Refresh`, batch lock/unlock | Latency-sensitive but not bulk data; correctness and timeout semantics matter more than transport bandwidth. | -| Peer/admin state | `crates/ecstore/src/rpc/peer_rest_client.rs`, `rustfs/src/storage/rpc/health.rs`, `metrics.rs`, `event.rs` | `LocalStorageInfo`, `ServerInfo`, `GetMetrics`, `GetLiveEvents`, reload APIs, rebalance APIs | Operational control plane. | -| Disk metadata/control | `crates/ecstore/src/rpc/remote_disk.rs`, `rustfs/src/storage/rpc/disk.rs` | `DiskInfo`, `ReadXL`, `ReadVersion`, `ReadMetadata`, `WriteMetadata`, `RenameFile`, `RenamePart`, `Delete*`, `VerifyFile`, `CheckParts` | Usually metadata, integrity checks, or namespace mutations. | +| Bucket peer ops | `crates/ecstore/src/rpc/peer_s3_client.rs`, `rustfs/src/storage/rpc/node_service/bucket.rs` | `MakeBucket`, `ListBucket`, `DeleteBucket`, `GetBucketInfo`, `HealBucket` | Small metadata/control payloads. | +| Locking | `crates/ecstore/src/rpc/remote_locker.rs`, `rustfs/src/storage/rpc/node_service/lock.rs` | `Lock`, `UnLock`, `Refresh`, batch lock/unlock | Latency-sensitive but not bulk data; correctness and timeout semantics matter more than transport bandwidth. | +| Peer/admin state | `crates/ecstore/src/rpc/peer_rest_client.rs`, `rustfs/src/storage/rpc/node_service/health.rs`, `node_service/metrics.rs`, `node_service/event.rs` | `LocalStorageInfo`, `ServerInfo`, `GetMetrics`, `GetLiveEvents`, reload APIs, rebalance APIs | Operational control plane. | +| Disk metadata/control | `crates/ecstore/src/rpc/remote_disk.rs`, `rustfs/src/storage/rpc/node_service/disk.rs` | `DiskInfo`, `ReadXL`, `ReadVersion`, `ReadMetadata`, `WriteMetadata`, `RenameFile`, `RenamePart`, `Delete*`, `VerifyFile`, `CheckParts` | Usually metadata, integrity checks, or namespace mutations. | | Connection health | `RemoteDisk`, `RemotePeerS3Client`, `PeerRestClient` | TCP connectivity probes and fault/recovery state | Must remain available even if an optional data backend is unavailable. | ### Data plane candidates @@ -177,12 +177,12 @@ Classification: | Path | Owner references | Server references | Classification | Notes | | --- | --- | --- | --- | --- | -| `ReadAll` | `RemoteDisk::read_all`; `crates/ecstore/src/store_init.rs`; heal resume metadata readers | `rustfs/src/storage/rpc/disk.rs::handle_read_all` | Still direct gRPC | Unary `bytes` response. Currently used mostly for metadata/config files; measure before moving. | -| `WriteAll` | `RemoteDisk::write_all`; `crates/ecstore/src/store_init.rs`; heal resume metadata writers | `rustfs/src/storage/rpc/disk.rs::handle_write_all` | Still direct gRPC | Unary `bytes` request. Currently used mostly for metadata/config/checkpoint writes. | -| `ReadMultiple` | `RemoteDisk::read_multiple`; `crates/ecstore/src/set_disk/read.rs::read_multiple_files` | `rustfs/src/storage/rpc/disk.rs::handle_read_multiple` | Still direct gRPC | Returns multiple small file payloads, usually metadata/listing support. Could become large with many entries. | -| `ReadParts` | `RemoteDisk::read_parts`; `crates/ecstore/src/set_disk/read.rs::read_parts`; multipart list/complete paths | `rustfs/src/storage/rpc/disk.rs::handle_read_parts` | Still direct gRPC | Encoded `ObjectPartInfo` metadata, not object data. | -| `RenamePart` | `RemoteDisk::rename_part`; `crates/ecstore/src/set_disk/write.rs::rename_part` | `rustfs/src/storage/rpc/disk.rs::handle_rename_part` | Still direct gRPC | Carries part metadata while committing multipart data already written through stream writers. | -| `ListDir` | `RemoteDisk::list_dir`; multipart/lifecycle metadata listing callers | `rustfs/src/storage/rpc/disk.rs::handle_list_dir` | Still direct gRPC | Directory name listing, metadata/control-plane unless measured otherwise. | +| `ReadAll` | `RemoteDisk::read_all`; `crates/ecstore/src/store_init.rs`; heal resume metadata readers | `rustfs/src/storage/rpc/node_service/disk.rs::handle_read_all` | Still direct gRPC | Unary `bytes` response. Currently used mostly for metadata/config files; measure before moving. | +| `WriteAll` | `RemoteDisk::write_all`; `crates/ecstore/src/store_init.rs`; heal resume metadata writers | `rustfs/src/storage/rpc/node_service/disk.rs::handle_write_all` | Still direct gRPC | Unary `bytes` request. Currently used mostly for metadata/config/checkpoint writes. | +| `ReadMultiple` | `RemoteDisk::read_multiple`; `crates/ecstore/src/set_disk/read.rs::read_multiple_files` | `rustfs/src/storage/rpc/node_service/disk.rs::handle_read_multiple` | Still direct gRPC | Returns multiple small file payloads, usually metadata/listing support. Could become large with many entries. | +| `ReadParts` | `RemoteDisk::read_parts`; `crates/ecstore/src/set_disk/read.rs::read_parts`; multipart list/complete paths | `rustfs/src/storage/rpc/node_service/disk.rs::handle_read_parts` | Still direct gRPC | Encoded `ObjectPartInfo` metadata, not object data. | +| `RenamePart` | `RemoteDisk::rename_part`; `crates/ecstore/src/set_disk/write.rs::rename_part` | `rustfs/src/storage/rpc/node_service/disk.rs::handle_rename_part` | Still direct gRPC | Carries part metadata while committing multipart data already written through stream writers. | +| `ListDir` | `RemoteDisk::list_dir`; multipart/lifecycle metadata listing callers | `rustfs/src/storage/rpc/node_service/disk.rs::handle_list_dir` | Still direct gRPC | Directory name listing, metadata/control-plane unless measured otherwise. | | Legacy gRPC `WalkDir` | `rustfs/src/storage/rpc/node_service.rs::NodeService::walk_dir` | same file | Still direct gRPC | Server implementation remains, but current `RemoteDisk::walk_dir` uses HTTP through the transport. Keep until callers are audited or compatibility policy is set. | ### Metadata/control-plane only @@ -190,7 +190,7 @@ Classification: | Area | Owner references | Classification | Notes | | --- | --- | --- | --- | | Disk metadata and namespace mutations | `RemoteDisk::{read_metadata,write_metadata,update_metadata,read_version,read_xl,rename_data,rename_file,delete*,verify_file,check_parts,disk_info}` | Metadata/control-plane only | These remain on gRPC by design. | -| Peer/bucket/admin operations | `crates/ecstore/src/rpc/{peer_s3_client.rs,peer_rest_client.rs,remote_locker.rs}` and matching `rustfs/src/storage/rpc/*` handlers | Metadata/control-plane only | Not candidates for a data-plane backend without separate measurements. | +| Peer/bucket/admin operations | `crates/ecstore/src/rpc/{peer_s3_client.rs,peer_rest_client.rs,remote_locker.rs}` and matching `rustfs/src/storage/rpc/node_service/*` handlers | Metadata/control-plane only | Not candidates for a data-plane backend without separate measurements. | | Store init and format operations | `crates/ecstore/src/store_init.rs` | Metadata/control-plane only | Uses `ReadAll`/`WriteAll` for small format/config objects. | | Heal orchestration | `crates/heal/src/heal/storage.rs` and `crates/ecstore/src/set_disk.rs::heal_object` | Metadata/control-plane plus covered data reads | Heal object data reads go through `get_object_reader` and then covered shard streams; resume/checkpoint metadata uses direct gRPC disk metadata calls. | @@ -198,7 +198,7 @@ Classification: | Path | Owner references | Classification | Notes | | --- | --- | --- | --- | -| Proto `Write` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/disk.rs::handle_write` | Not relevant | Handler is unimplemented. | +| Proto `Write` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/node_service/disk.rs::handle_write` | Not relevant | Handler is unimplemented. | | Proto `WriteStream` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/node_service.rs::write_stream` | Not relevant | Returns `unimplemented`. | | Proto `ReadAt` | `crates/protos/src/node.proto`; `rustfs/src/storage/rpc/node_service.rs::read_at` | Not relevant | Returns `unimplemented`. | | E2E reliant gRPC helpers | `crates/e2e_test/src/reliant/*` | Not relevant | Test harnesses, not production internode data-path callers. | diff --git a/crates/ecstore/docs/internode-transport/transport-metrics-and-baseline.md b/crates/ecstore/docs/internode-transport/transport-metrics-and-baseline.md index 8580bf082..bca28150b 100644 --- a/crates/ecstore/docs/internode-transport/transport-metrics-and-baseline.md +++ b/crates/ecstore/docs/internode-transport/transport-metrics-and-baseline.md @@ -59,7 +59,7 @@ values as labels. | --- | --- | --- | | Outgoing HTTP stream requests | `crates/rio/src/http_reader.rs` | Outgoing request count, sent bytes for writer bodies, received bytes for reader bodies, and errors for known `/rustfs/rpc/` operations. | | Incoming HTTP stream requests | `rustfs/src/storage/rpc/http_service.rs` | Incoming request count, sent bytes for read and walk streams, received bytes for put streams, and route errors. | -| gRPC `ReadAll` / `WriteAll` | `rustfs/src/storage/rpc/disk.rs` | Incoming request count, sent/received bytes, and errors with the `grpc` backend label. | +| gRPC `ReadAll` / `WriteAll` | `rustfs/src/storage/rpc/node_service/disk.rs` | Incoming request count, sent/received bytes, and errors with the `grpc` backend label. | Known gaps: diff --git a/crates/ecstore/src/metadata/bucket.rs b/crates/ecstore/src/bucket/metadata.rs similarity index 100% rename from crates/ecstore/src/metadata/bucket.rs rename to crates/ecstore/src/bucket/metadata.rs diff --git a/crates/ecstore/src/metadata/bucket_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs similarity index 100% rename from crates/ecstore/src/metadata/bucket_sys.rs rename to crates/ecstore/src/bucket/metadata_sys.rs diff --git a/crates/ecstore/src/metadata/bucket_test.rs b/crates/ecstore/src/bucket/metadata_test.rs similarity index 100% rename from crates/ecstore/src/metadata/bucket_test.rs rename to crates/ecstore/src/bucket/metadata_test.rs diff --git a/crates/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs index c542add56..9fa11cd39 100644 --- a/crates/ecstore/src/bucket/mod.rs +++ b/crates/ecstore/src/bucket/mod.rs @@ -19,12 +19,9 @@ pub mod bandwidth; pub mod bucket_target_sys; pub mod error; pub mod lifecycle; -#[path = "../metadata/bucket.rs"] pub mod metadata; -#[path = "../metadata/bucket_sys.rs"] pub mod metadata_sys; #[cfg(test)] -#[path = "../metadata/bucket_test.rs"] mod metadata_test; pub mod migration; mod msgp_decode; diff --git a/crates/ecstore/src/metadata/set_disk.rs b/crates/ecstore/src/set_disk/metadata.rs similarity index 100% rename from crates/ecstore/src/metadata/set_disk.rs rename to crates/ecstore/src/set_disk/metadata.rs diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index fc5a8a041..3c5ffdb4b 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -446,7 +446,6 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock = OnceLock::new(); mod ctx; mod list; mod lock; -#[path = "../metadata/set_disk.rs"] mod metadata; mod multipart; mod ops; diff --git a/crates/heal/tests/endpoint_index_test.rs b/crates/heal/tests/endpoint_index_test.rs index bbe1ec2d4..52684b9af 100644 --- a/crates/heal/tests/endpoint_index_test.rs +++ b/crates/heal/tests/endpoint_index_test.rs @@ -18,11 +18,9 @@ use std::net::SocketAddr; use tempfile::TempDir; use tokio_util::sync::CancellationToken; -#[path = "endpoint_index_test/storage_api.rs"] mod storage_api; use storage_api::endpoint_index::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks}; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_endpoint_index_settings() -> rustfs_heal::Result<()> { let temp_dir = TempDir::new()?; diff --git a/crates/heal/tests/endpoint_index_test/storage_api.rs b/crates/heal/tests/endpoint_index_test/storage_api.rs deleted file mode 100644 index 83bd07ed9..000000000 --- a/crates/heal/tests/endpoint_index_test/storage_api.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; -pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; - -pub(crate) mod endpoint_index { - pub(crate) use super::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks}; -} diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index 25a256b01..a543f50fb 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -18,7 +18,6 @@ use rustfs_heal::heal::{ utils, }; -#[path = "heal_bug_fixes_test/storage_api.rs"] mod storage_api; use storage_api::bug_fixes::{BucketInfo, DiskStore, Endpoint}; diff --git a/crates/heal/tests/heal_bug_fixes_test/storage_api.rs b/crates/heal/tests/heal_bug_fixes_test/storage_api.rs deleted file mode 100644 index a9f1204f4..000000000 --- a/crates/heal/tests/heal_bug_fixes_test/storage_api.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint}; -use rustfs_storage_api as storage_contracts; - -pub(crate) mod bug_fixes { - pub(crate) use super::storage_contracts::BucketInfo; - pub(crate) use super::{DiskStore, Endpoint}; -} diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index d41cf9fc3..31a97243d 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -30,7 +30,6 @@ use tokio_util::sync::CancellationToken; use tracing::info; use walkdir::WalkDir; -#[path = "heal_integration_test/storage_api.rs"] mod storage_api; use storage_api::integration::{ diff --git a/crates/heal/tests/heal_integration_test/storage_api.rs b/crates/heal/tests/heal_integration_test/storage_api.rs deleted file mode 100644 index 74003556d..000000000 --- a/crates/heal/tests/heal_integration_test/storage_api.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; -pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; -pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; -use rustfs_storage_api as storage_contracts; - -pub(crate) mod integration { - pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations}; - - pub(crate) use super::{ - ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks, - }; -} diff --git a/crates/heal/tests/storage_api.rs b/crates/heal/tests/storage_api.rs new file mode 100644 index 000000000..308e9d114 --- /dev/null +++ b/crates/heal/tests/storage_api.rs @@ -0,0 +1,31 @@ +#[allow(unused_imports)] +pub(crate) mod bug_fixes { + pub(crate) use rustfs_ecstore::api::disk::DiskStore; + pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; + pub(crate) use rustfs_storage_api::BucketInfo; +} + +#[allow(unused_imports)] +pub(crate) mod endpoint_index { + pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; + pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; + pub(crate) use rustfs_ecstore::api::layout::Endpoints; + pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints; + pub(crate) use rustfs_ecstore::api::storage::ECStore; + pub(crate) use rustfs_ecstore::api::storage::init_local_disks; +} + +#[allow(unused_imports)] +pub(crate) mod integration { + pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; + pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; + pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; + pub(crate) use rustfs_ecstore::api::layout::Endpoints; + pub(crate) use rustfs_ecstore::api::layout::PoolEndpoints; + pub(crate) use rustfs_ecstore::api::storage::ECStore; + pub(crate) use rustfs_ecstore::api::storage::init_local_disks; + pub(crate) use rustfs_storage_api::BucketOperations; + pub(crate) use rustfs_storage_api::BucketOptions; + pub(crate) use rustfs_storage_api::ObjectIO; + pub(crate) use rustfs_storage_api::ObjectOperations; +} diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 6960d26a4..1b8b7839c 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -47,7 +47,7 @@ migration PR removes or replaces each item. | `GLOBAL_TIER_CONFIG_MGR`, `GLOBAL_STORAGE_CLASS`, `GLOBAL_CONFIG_SYS`, `GLOBAL_SERVER_CONFIG` | `crates/ecstore/src/config`, `crates/config`, `rustfs/src/app/context/runtime_sources.rs` | Runtime migration target | Tier config manager reads and reloads now use the ECStore runtime-source helper; move remaining config state through config/runtime-source owners only, without combining storage-class behavior or persistence changes. | | `GLOBAL_EVENT_NOTIFIER`, `GLOBAL_NOTIFICATION_SYS` | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs`, and `crates/ecstore/src/services/*` | Runtime migration target | `GLOBAL_EVENT_NOTIFIER` and `GLOBAL_NOTIFICATION_SYS` access now stay behind ECStore runtime-source and notification owner helpers; move remaining notification ownership only through notify/runtime-source boundaries. | | `EVENT_DISPATCH_HOOK` | `crates/ecstore/src/services/event_notification.rs`, RustFS server event bridge, and storage compatibility APIs | Runtime migration target / owner helper | Direct hook storage stays inside the ECStore event-notification owner; RustFS registers the bridge through the storage compatibility facade until event dispatch ownership moves behind an injected notification sink. | -| `GLOBAL_BUCKET_METADATA_SYS` | `crates/ecstore/src/metadata/bucket_sys.rs`, `crates/ecstore/src/runtime/sources.rs`, and RustFS storage compatibility APIs | Runtime migration target | Bucket metadata system direct access now stays inside the ECStore metadata owner; callers use metadata owner helpers or storage/runtime-source compatibility functions until metadata ownership moves behind an injected runtime context. | +| `GLOBAL_BUCKET_METADATA_SYS` | `crates/ecstore/src/bucket/metadata_sys.rs`, `crates/ecstore/src/runtime/sources.rs`, and RustFS storage compatibility APIs | Runtime migration target | Bucket metadata system direct access now stays inside the ECStore metadata owner; callers use metadata owner helpers or storage/runtime-source compatibility functions until metadata ownership moves behind an injected runtime context. | | `GLOBAL_BOOT_TIME`, `GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN`, `GLOBAL_DEPLOYMENT_ID`, `GLOBAL_REGION`, `GLOBAL_RUSTFS_PORT`, `GLOBAL_LOCAL_NODE_NAME_FALLBACK`, `GLOBAL_LOCAL_NODE_NAME_HEX_FALLBACK` | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Boot time, background service cancellation token reads, ECStore local-node-name fallback reads, and deployment ID/region/port reads now stay behind the ECStore runtime-source API; scalar writes remain behind bootstrap owner helpers until ownership handles replace them. | | `WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER` | `crates/ecstore/src/runtime/sources.rs`, RustFS startup background setup, and storage compatibility APIs | Runtime migration target / owner helper | Startup publishes the workload provider through the storage compatibility facade, and ECStore data movement reads it only through the runtime-source helper until workload admission ownership moves into an explicit runtime context. | | `GLOBAL_LOCAL_LOCK_CLIENT`, `GLOBAL_LOCK_CLIENTS`, `GLOBAL_LOCK_MANAGER` | `crates/ecstore/src/runtime/global.rs`, `crates/lock` | Runtime migration target / process-global split | ECStore lock client reads now flow through ECStore `api::runtime` helpers at the RustFS storage facade boundary; preserve lock quorum and lock client selection while keeping the process-level lock manager separate from endpoint-specific clients. | diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 86fabaf6e..7c8309583 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -154,17 +154,11 @@ fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> StorageR Ok(()) } -#[path = "bucket.rs"] mod bucket; -#[path = "disk.rs"] mod disk; -#[path = "event.rs"] mod event; -#[path = "health.rs"] mod health; -#[path = "lock.rs"] mod lock; -#[path = "metrics.rs"] mod metrics; pub struct NodeService { diff --git a/rustfs/src/storage/rpc/bucket.rs b/rustfs/src/storage/rpc/node_service/bucket.rs similarity index 100% rename from rustfs/src/storage/rpc/bucket.rs rename to rustfs/src/storage/rpc/node_service/bucket.rs diff --git a/rustfs/src/storage/rpc/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs similarity index 100% rename from rustfs/src/storage/rpc/disk.rs rename to rustfs/src/storage/rpc/node_service/disk.rs diff --git a/rustfs/src/storage/rpc/event.rs b/rustfs/src/storage/rpc/node_service/event.rs similarity index 100% rename from rustfs/src/storage/rpc/event.rs rename to rustfs/src/storage/rpc/node_service/event.rs diff --git a/rustfs/src/storage/rpc/health.rs b/rustfs/src/storage/rpc/node_service/health.rs similarity index 100% rename from rustfs/src/storage/rpc/health.rs rename to rustfs/src/storage/rpc/node_service/health.rs diff --git a/rustfs/src/storage/rpc/lock.rs b/rustfs/src/storage/rpc/node_service/lock.rs similarity index 100% rename from rustfs/src/storage/rpc/lock.rs rename to rustfs/src/storage/rpc/node_service/lock.rs diff --git a/rustfs/src/storage/rpc/metrics.rs b/rustfs/src/storage/rpc/node_service/metrics.rs similarity index 100% rename from rustfs/src/storage/rpc/metrics.rs rename to rustfs/src/storage/rpc/node_service/metrics.rs diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 3a6077e81..91e04b84b 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -1209,7 +1209,7 @@ fi --glob '!**/ecstore_test_compat/**' \ --glob '!**/ecstore_fuzz_compat.rs' \ --glob '!target/**' \ - | rg -v '^(rustfs/src/(admin/mod|app/mod)\.rs|rustfs/src/storage/storage_api\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs):' || true + | rg -v '^(rustfs/src/(admin/mod|app/mod)\.rs|rustfs/src/storage/storage_api\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs):' || true ) | cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE" @@ -1606,7 +1606,7 @@ fi --glob '!**/ecstore_compat.rs' \ --glob '!**/ecstore_test_compat.rs' \ --glob '!**/ecstore_test_compat/**' | - rg -v '^(fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/storage_api\.rs):' || true + rg -v '^(fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs|crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/storage_api\.rs):' || true ) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then @@ -1617,9 +1617,7 @@ fi cd "$ROOT_DIR" rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \ crates/heal/src/heal/storage_api.rs \ - crates/heal/tests/endpoint_index_test/storage_api.rs \ - crates/heal/tests/heal_bug_fixes_test/storage_api.rs \ - crates/heal/tests/heal_integration_test/storage_api.rs \ + crates/heal/tests/storage_api.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ @@ -1645,9 +1643,7 @@ fi cd "$ROOT_DIR" rg -n --with-filename '^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+\s*;|^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::\*\s*;' \ crates/heal/src/heal/storage_api.rs \ - crates/heal/tests/endpoint_index_test/storage_api.rs \ - crates/heal/tests/heal_bug_fixes_test/storage_api.rs \ - crates/heal/tests/heal_integration_test/storage_api.rs \ + crates/heal/tests/storage_api.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ @@ -1673,9 +1669,7 @@ fi cd "$ROOT_DIR" rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' \ crates/heal/src/heal/storage_api.rs \ - crates/heal/tests/endpoint_index_test/storage_api.rs \ - crates/heal/tests/heal_bug_fixes_test/storage_api.rs \ - crates/heal/tests/heal_integration_test/storage_api.rs \ + crates/heal/tests/storage_api.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ @@ -2036,8 +2030,8 @@ fi rustfs/src/storage/helper.rs \ rustfs/src/storage/concurrency/manager.rs \ rustfs/src/storage/sse.rs \ - rustfs/src/storage/rpc/disk.rs \ - rustfs/src/storage/rpc/health.rs \ + rustfs/src/storage/rpc/node_service/disk.rs \ + rustfs/src/storage/rpc/node_service/health.rs \ rustfs/src/storage/rpc/http_service.rs \ --glob '*.rs' || true ) >"$RUSTFS_STORAGE_RUNTIME_SOURCE_BYPASS_HITS_FILE" @@ -2559,7 +2553,7 @@ fi crates/scanner/tests \ crates/e2e_test/src \ --glob '*.rs' \ - | rg -v '^(crates/e2e_test/src/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs):' || true + | rg -v '^(crates/e2e_test/src/storage_api\.rs|crates/heal/tests/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs):' || true ) >"$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" if [[ -s "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then @@ -2593,7 +2587,7 @@ fi crates/scanner/tests \ fuzz/fuzz_targets \ --glob '*.rs' | - rg -v '^(crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs):' || true + rg -v '^(crates/e2e_test/src/storage_api\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/storage_api\.rs|crates/iam/src/storage_api\.rs|crates/notify/src/storage_api\.rs|crates/obs/src/metrics/storage_api\.rs|crates/protocols/src/swift/storage_api\.rs|crates/s3select-api/src/storage_api\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation_storage_api|path_containment_storage_api)\.rs):' || true ) >"$EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE" if [[ -s "$EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE" ]]; then @@ -3777,7 +3771,7 @@ fi rg -n --with-filename '\bGLOBAL_(BucketMetadataSys|BUCKET_METADATA_SYS)\b' \ crates rustfs fuzz \ --glob '*.rs' | - rg -v '^crates/ecstore/src/metadata/bucket_sys\.rs:' || true + rg -v '^crates/ecstore/src/bucket/metadata_sys\.rs:' || true ) >"$GLOBAL_BUCKET_METADATA_SYS_BYPASS_HITS_FILE" if [[ -s "$GLOBAL_BUCKET_METADATA_SYS_BYPASS_HITS_FILE" ]]; then @@ -4816,7 +4810,7 @@ fi rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \ crates/heal/tests \ --glob '*.rs' | - rg -v '^crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs:' || true + rg -v '^crates/heal/tests/storage_api\.rs:' || true ) >"$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE" if [[ -s "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then @@ -5187,16 +5181,16 @@ require_source_contains \ ( cd "$ROOT_DIR" { - [[ -e crates/ecstore/src/bucket/metadata.rs ]] && printf '%s\n' 'crates/ecstore/src/bucket/metadata.rs' - [[ -e crates/ecstore/src/bucket/metadata_sys.rs ]] && printf '%s\n' 'crates/ecstore/src/bucket/metadata_sys.rs' - [[ -e crates/ecstore/src/bucket/metadata_test.rs ]] && printf '%s\n' 'crates/ecstore/src/bucket/metadata_test.rs' - [[ -e crates/ecstore/src/set_disk/metadata.rs ]] && printf '%s\n' 'crates/ecstore/src/set_disk/metadata.rs' + [[ -e crates/ecstore/src/metadata/bucket.rs ]] && printf '%s\n' 'crates/ecstore/src/metadata/bucket.rs' + [[ -e crates/ecstore/src/metadata/bucket_sys.rs ]] && printf '%s\n' 'crates/ecstore/src/metadata/bucket_sys.rs' + [[ -e crates/ecstore/src/metadata/bucket_test.rs ]] && printf '%s\n' 'crates/ecstore/src/metadata/bucket_test.rs' + [[ -e crates/ecstore/src/metadata/set_disk.rs ]] && printf '%s\n' 'crates/ecstore/src/metadata/set_disk.rs' true } ) >"$ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE" if [[ -s "$ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE" ]]; then - report_failure "ECStore metadata modules must stay under the metadata owner directory: $(paste -sd '; ' "$ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE")" + report_failure "ECStore metadata modules must use canonical bucket/set_disk module directories: $(paste -sd '; ' "$ECSTORE_OLD_METADATA_OWNER_PATH_HITS_FILE")" fi rg -n --with-filename '#\[path = "(data_movement|layout|core|store|diagnostics|services|io_support|runtime)/' \