diff --git a/crates/heal/src/heal/storage_compat.rs b/crates/heal/src/heal/storage_compat.rs index 6d60a1495..75019eda0 100644 --- a/crates/heal/src/heal/storage_compat.rs +++ b/crates/heal/src/heal/storage_compat.rs @@ -27,7 +27,12 @@ pub(crate) use rustfs_ecstore::api::disk::DiskAPI; pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP; #[cfg(test)] -pub(crate) use rustfs_ecstore::api::disk::{DiskOption, new_disk}; +pub(crate) type DiskOption = rustfs_ecstore::api::disk::DiskOption; + +#[cfg(test)] +pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> rustfs_ecstore::api::disk::error::Result { + rustfs_ecstore::api::disk::new_disk(ep, opt).await +} pub type HealObjectInfo = ::ObjectInfo; pub type HealObjectOptions = ::ObjectOptions; diff --git a/crates/heal/tests/common/storage_compat.rs b/crates/heal/tests/common/storage_compat.rs index 7bd273ac0..35d22bc95 100644 --- a/crates/heal/tests/common/storage_compat.rs +++ b/crates/heal/tests/common/storage_compat.rs @@ -12,10 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(unused_imports)] +#![allow(dead_code, unused_imports)] -pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; -pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint}; -pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -pub(crate) use rustfs_ecstore::api::storage::ECStore; -pub(crate) use rustfs_ecstore::api::storage::init_local_disks; +use std::sync::Arc; + +pub(crate) type DiskStore = rustfs_ecstore::api::disk::DiskStore; +pub(crate) type ECStore = rustfs_ecstore::api::storage::ECStore; +pub(crate) type Endpoint = rustfs_ecstore::api::disk::endpoint::Endpoint; +pub(crate) type Endpoints = rustfs_ecstore::api::layout::Endpoints; +pub(crate) type PoolEndpoints = rustfs_ecstore::api::layout::PoolEndpoints; + +pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; + +pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { + rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; +} + +pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::storage::init_local_disks(endpoint_pools).await +} diff --git a/crates/scanner/tests/common/storage_compat.rs b/crates/scanner/tests/common/storage_compat.rs index e405aa7b7..78f8a3b47 100644 --- a/crates/scanner/tests/common/storage_compat.rs +++ b/crates/scanner/tests/common/storage_compat.rs @@ -14,25 +14,71 @@ #![allow(dead_code, unused_imports)] +use std::sync::Arc; +use time::OffsetDateTime; + pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::api::disk::STORAGE_FORMAT_FILE; +pub(crate) type BucketMetadata = rustfs_ecstore::api::bucket::metadata::BucketMetadata; +pub(crate) type BucketVersioningSys = rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; +pub(crate) type DiskOption = rustfs_ecstore::api::disk::DiskOption; +pub(crate) type ECStore = rustfs_ecstore::api::storage::ECStore; +pub(crate) type Endpoint = rustfs_ecstore::api::disk::endpoint::Endpoint; +pub(crate) type Endpoints = rustfs_ecstore::api::layout::Endpoints; +pub(crate) type PoolEndpoints = rustfs_ecstore::api::layout::PoolEndpoints; +pub(crate) type ReadCloser = rustfs_ecstore::api::client::transition_api::ReadCloser; +pub(crate) type ReaderImpl = rustfs_ecstore::api::client::transition_api::ReaderImpl; +pub(crate) type TierConfig = rustfs_ecstore::api::tier::tier_config::TierConfig; +pub(crate) type TierMinIO = rustfs_ecstore::api::tier::tier_config::TierMinIO; +pub(crate) type TierType = rustfs_ecstore::api::tier::tier_config::TierType; pub(crate) type TransitionOptions = rustfs_ecstore::api::bucket::lifecycle::lifecycle::TransitionOptions; +pub(crate) type WarmBackendGetOpts = rustfs_ecstore::api::tier::warm_backend::WarmBackendGetOpts; -pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{ - enqueue_transition_for_existing_objects, init_background_expiry, -}; -pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ - get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata, -}; -pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; -pub(crate) use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path; -pub(crate) use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl}; -pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk}; +pub(crate) use rustfs_ecstore::api::disk::DiskAPI; pub(crate) use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr; -pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -pub(crate) use rustfs_ecstore::api::storage::ECStore; -pub(crate) use rustfs_ecstore::api::storage::init_local_disks; -pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType}; -pub(crate) use rustfs_ecstore::api::tier::warm_backend::WarmBackendGetOpts; -pub(crate) use rustfs_ecstore::api::tier::warm_backend::{WarmBackend, build_transition_put_options}; +pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; +pub(crate) use rustfs_ecstore::api::tier::warm_backend::WarmBackend; +pub(crate) use rustfs_ecstore::api::tier::warm_backend::build_transition_put_options; + +pub(crate) async fn enqueue_transition_for_existing_objects( + api: Arc, + bucket: &str, +) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(api, bucket).await +} + +pub(crate) async fn init_background_expiry(api: Arc) { + rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await; +} + +pub(crate) async fn get_bucket_metadata(bucket: &str) -> rustfs_ecstore::api::error::Result> { + rustfs_ecstore::api::bucket::metadata_sys::get(bucket).await +} + +pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { + rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; +} + +pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::storage::init_local_disks(endpoint_pools).await +} + +pub(crate) async fn new_disk( + ep: &Endpoint, + opt: &DiskOption, +) -> rustfs_ecstore::api::disk::error::Result { + rustfs_ecstore::api::disk::new_disk(ep, opt).await +} + +pub(crate) fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) { + rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path(base_path, path) +} + +pub(crate) async fn update_bucket_metadata( + bucket: &str, + config_file: &str, + data: Vec, +) -> rustfs_ecstore::api::error::Result { + rustfs_ecstore::api::bucket::metadata_sys::update(bucket, config_file, data).await +} diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 6b0e9a527..6b3d55fb9 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -169,6 +169,10 @@ must not restore broad `config`/`global` module imports. Observability must consume data usage through a local DTO projection instead of re-exporting the ECStore data-usage loader. The e2e harness must not restore grouped RPC passthroughs. +Test and fuzz `storage_compat.rs` harnesses must also stay narrow. Heal and +scanner test harnesses must expose ECStore contracts through direct aliases or +local wrappers, and fuzz harnesses must wrap bucket utility entrypoints instead +of restoring grouped ECStore passthrough exports. ECStore ClusterControlPlane read models must stay owned by the crate-private `cluster` module. Public access goes through `rustfs_ecstore::api::cluster` so diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index d2e5a42bb..190bd03c7 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-edge-compat-alias-prune` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083`. -- Stacked on: API-083 admin/app compatibility alias pruning. +- Branch: `overtrue/arch-test-protocol-compat-prune` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084`. +- Stacked on: API-084 edge compatibility alias pruning. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: prune scanner, notify, observability, and e2e edge - compatibility passthroughs into explicit aliases, wrappers, and local DTOs. -- CI/script changes: guard against restoring broad edge compatibility - passthroughs. -- Docs changes: record the API-084 edge compatibility alias boundary. +- Rust code changes: prune heal/scanner test and fuzz compatibility + passthroughs into explicit aliases and local wrappers. +- CI/script changes: guard against restoring grouped test and fuzz + compatibility passthroughs. +- Docs changes: record the API-085 test/fuzz compatibility boundary. ## Phase 0 Tasks @@ -201,6 +201,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block - Verification: focused edge crate compile coverage, edge compatibility residual scans, formatting, diff hygiene, risk scan, architecture guard, pre-commit quality gate, and three-expert review. +- [x] `API-085` Prune test and fuzz compatibility passthrough aliases. + - Completed slice: replace heal/scanner test and fuzz grouped ECStore + compatibility passthroughs with direct type aliases and local wrapper + functions. + - Acceptance: test harnesses keep their existing ECStore setup and lifecycle + helper call sites while exposing only narrow compatibility symbols, and + fuzz targets exercise bucket utility contracts through local wrappers. + - Must preserve: heal test ECStore setup, scanner lifecycle integration setup, + local disk initialization, bucket metadata updates, transition enqueue + behavior, and fuzz validation semantics. + - Verification: focused heal/scanner compile coverage, test/fuzz + compatibility residual scans, formatting, diff hygiene, architecture guard, + pre-commit quality gate, and three-expert review. - [x] `G-012` Inventory placement and repair invariants. - Acceptance: [`placement-repair-invariants.md`](placement-repair-invariants.md) records @@ -3234,14 +3247,24 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | API-084 narrows scanner/notify/obs/e2e edge compatibility passthroughs without adding ECStore ownership cycles or new runtime dependencies. | -| Migration preservation | passed | Scanner lifecycle/replication helpers, notify config persistence, obs usage metrics, and e2e RPC helper behavior remain preserved behind local aliases/wrappers. | -| Testing/verification | passed | Focused edge compile coverage, compatibility residual scans, migration guard, formatting, diff hygiene, added-line risk scan, full pre-commit, and three-expert review passed. | +| Quality/architecture | passed | API-085 narrows heal/scanner test and fuzz compatibility passthroughs without adding ECStore ownership cycles or runtime dependencies. | +| Migration preservation | passed | Heal test setup, scanner lifecycle integration helpers, local disk initialization, bucket metadata updates, transition enqueue behavior, and fuzz validation semantics remain preserved behind aliases/wrappers. | +| Testing/verification | passed | Focused heal/scanner compile coverage, test/fuzz compatibility residual scans, migration guard, formatting, diff hygiene, added-line risk scan, full pre-commit, and three-expert review passed. | ## Verification Notes Passed before push: +- Issue #660 API-085 current slice: + - `cargo check --tests -p rustfs-heal -p rustfs-scanner`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - Test/fuzz grouped compatibility passthrough residual scans: passed. + - Rust added-line risk scan on changed Rust files and guard script: passed. + - `make pre-commit`: passed. + - Issue #660 API-084 current slice: - `cargo check --tests -p rustfs-scanner -p rustfs-notify -p rustfs-obs -p e2e_test`: passed. - `cargo fmt --all --check`: passed. diff --git a/fuzz/fuzz_targets/bucket_validation/storage_compat.rs b/fuzz/fuzz_targets/bucket_validation/storage_compat.rs index 0bc702d6d..92d06ffce 100644 --- a/fuzz/fuzz_targets/bucket_validation/storage_compat.rs +++ b/fuzz/fuzz_targets/bucket_validation/storage_compat.rs @@ -12,6 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_ecstore::api::bucket::utils::{ - check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname, -}; +pub(crate) fn check_bucket_and_object_names(bucket: &str, object: &str) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::bucket::utils::check_bucket_and_object_names(bucket, object) +} + +pub(crate) fn check_list_objs_args( + bucket: &str, + prefix: &str, + marker: &Option, +) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::bucket::utils::check_list_objs_args(bucket, prefix, marker) +} + +pub(crate) fn check_valid_bucket_name_strict(bucket_name: &str) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::bucket::utils::check_valid_bucket_name_strict(bucket_name) +} + +pub(crate) fn is_meta_bucketname(name: &str) -> bool { + rustfs_ecstore::api::bucket::utils::is_meta_bucketname(name) +} diff --git a/fuzz/fuzz_targets/path_containment/storage_compat.rs b/fuzz/fuzz_targets/path_containment/storage_compat.rs index f54cdf0e2..32d7bd4ad 100644 --- a/fuzz/fuzz_targets/path_containment/storage_compat.rs +++ b/fuzz/fuzz_targets/path_containment/storage_compat.rs @@ -12,6 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_ecstore::api::bucket::utils::{ - check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix, -}; +pub(crate) fn check_object_name_for_length_and_slash( + bucket: &str, + object: &str, +) -> rustfs_ecstore::api::error::Result<()> { + rustfs_ecstore::api::bucket::utils::check_object_name_for_length_and_slash(bucket, object) +} + +pub(crate) fn has_bad_path_component(path: &str) -> bool { + rustfs_ecstore::api::bucket::utils::has_bad_path_component(path) +} + +pub(crate) fn is_valid_object_prefix(object: &str) -> bool { + rustfs_ecstore::api::bucket::utils::is_valid_object_prefix(object) +} diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 1bed791c7..b9d3f90b9 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -88,6 +88,7 @@ SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/scanner_bucket_storag NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/notify_storage_compat_module_hits.txt" OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/obs_storage_compat_passthrough_hits.txt" E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE="${TMP_DIR}/e2e_storage_compat_rpc_passthrough_hits.txt" +TEST_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/test_storage_compat_passthrough_hits.txt" PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt" BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt" NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt" @@ -875,6 +876,17 @@ if [[ -s "$E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE" ]]; then report_failure "e2e storage compatibility must wrap RPC helpers instead of re-exporting broad ECStore functions: $(paste -sd '; ' "$E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --no-heading 'pub\(crate\)\s+use rustfs_ecstore::api::(?:bucket::(?:lifecycle|metadata_sys|utils)|capacity|client|disk|layout|storage|tier)::[^;]*\{[^}]*\}\s*;|pub\(crate\)\s+use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys|pub\(crate\)\s+use rustfs_ecstore::api::storage::init_local_disks' \ + crates/heal/tests crates/scanner/tests fuzz/fuzz_targets \ + --glob '*storage_compat.rs' || true +) >"$TEST_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE" + +if [[ -s "$TEST_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE" ]]; then + report_failure "test and fuzz storage compatibility boundaries must use explicit aliases or wrappers instead of grouped ECStore passthroughs: $(paste -sd '; ' "$TEST_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --no-heading '#!\[allow\(unused_imports\)\]' \