diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 549dba626..bb945d35d 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-storage-owner-contract-domain-batch` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/API-243/API-244/CTX-002`. -- Based on: rebased onto current `origin/main` after PR #3910 merged. +- Branch: `overtrue/arch-admin-contract-domain-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/API-243/API-244/API-245/CTX-002`. +- Based on: rebased onto current `origin/main` after PR #3911 merged. - PR type for this branch: `consumer-migration` -- Runtime behavior changes: none expected for API-245; storage owner code still - uses the same storage contracts, now exposed from `contract` domain modules - instead of its flat root facade. -- Rust code changes: segment the storage-owner local `contract` facade into - domain modules, migrate storage-owner consumers to domain contract imports, - reject flat storage-owner contract consumers in migration guardrails, route +- Runtime behavior changes: none expected for API-246; admin code still uses + the same storage contracts, now exposed from admin `contract` domain modules + instead of its flat root facade; rebase-exposed app usecase compile fixes + keep the same stream and data-usage behavior. +- Rust code changes: segment the admin local `contract` facade into domain + modules, migrate admin consumers to domain contract imports, reject flat + admin contract consumers in migration guardrails, route replication pool, outbound TLS generation, runtime region, KMS encryption service, runtime support handles, S3 Select DB, internode RPC metrics, IAM authorization/handler reads, notification @@ -5619,16 +5620,32 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block and topology snapshot construction. - Verification: focused RustFS compile, formatting, migration/layer guards, storage-owner raw contract/root consumer scans, diff hygiene, and Rust risk - scan passed; full PR gate is planned before PR. + scan passed; full PR gate passed before PR. + +- [x] `API-246` Segment admin contract facade by domain module. + - Do: split the admin local `contract` facade into admin, bucket, heal, list, + and object modules, then migrate admin consumers to those modules. + - Acceptance: admin handlers, router, and service code no longer import + storage contract symbols from the flat `contract` root, and migration rules + reject regressions. + - Must preserve: bucket metadata, replication, site replication, rebalance, + account info, config admin, heal, object zip download, router, and admin + service config call paths. + - Verification: focused RustFS compile, formatting, migration/layer guards, + admin contract root consumer scan, diff hygiene, and Rust risk scan passed; + full PR gate passed before PR. ## Next PRs -1. `consumer-migration`: continue larger owner boundary batches after API-245. +1. `consumer-migration`: continue larger owner boundary batches after API-246. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-246 segments the admin local contract facade into domain modules instead of flat root contract exposure. | +| Migration preservation | pass | Bucket metadata, replication, site replication, rebalance, account info, config admin, heal, object zip download, router, and admin service config paths keep the same underlying contracts. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, admin contract root consumer scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. | | Quality/architecture | pass | API-245 segments the storage-owner local contract facade into domain modules instead of flat root contract exposure. | | Migration preservation | pass | ECFS bucket/object calls, storage RPC bucket/admin calls, S3 list/multipart DTO projections, request option parsing, access checks, and topology snapshot construction keep the same underlying contracts. | | Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, storage-owner raw contract/root consumer scans, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | @@ -5905,6 +5922,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: +- Issue #660 API-246 current slice: + - Branch freshness check: rebased onto current `origin/main` after PR #3911 + merged. + - `cargo check -p rustfs --lib`: passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Admin contract root consumer scan: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `make pre-pr`: passed. + - Issue #660 API-245 current slice: - Branch freshness check: rebased onto current `origin/main` after PR #3910 merged. diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index 5ab3b3317..9adf1c49a 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -16,7 +16,8 @@ use crate::admin::auth::authenticate_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{resolve_action_credentials, resolve_object_store_handle}; use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys; -use crate::admin::storage_api::contract::{BucketOperations, BucketOptions, StorageAdminApi}; +use crate::admin::storage_api::contract::admin::StorageAdminApi; +use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::auth::get_condition_values; use crate::server::{ADMIN_PREFIX, RemoteAddr}; use http::{HeaderMap, HeaderValue}; diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index a4ed995a0..18b393c12 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -23,7 +23,7 @@ use crate::admin::storage_api::bucket::{ quota::BucketQuota, target::BucketTargets, }; -use crate::admin::storage_api::contract::{BucketOperations, BucketOptions, MakeBucketOptions}; +use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions, MakeBucketOptions}; use crate::admin::storage_api::error::StorageError; use crate::{ admin::runtime_sources::resolve_object_store_handle, diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index d2cda02f9..18a7c0352 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -24,7 +24,7 @@ use crate::admin::storage_api::config::{ RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, save_admin_config, save_admin_server_config, }; -use crate::admin::storage_api::contract::ListOperations as _; +use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; use crate::auth::{check_key_valid, get_session_token}; use crate::error::ApiError; diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 9ea118e4f..11b69e560 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -18,7 +18,7 @@ use crate::admin::runtime_sources::resolve_object_store_handle; use crate::admin::storage_api::access::spawn_traced; use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; use crate::admin::storage_api::bucket::utils::is_valid_object_prefix; -use crate::admin::storage_api::contract::HealOperations as _; +use crate::admin::storage_api::contract::heal::HealOperations as _; use crate::server::ADMIN_PREFIX; use crate::server::RemoteAddr; use bytes::Bytes; diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index dee116c9a..5ab14dd82 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -15,9 +15,9 @@ use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{resolve_action_credentials, resolve_object_store_handle, resolve_region}; use crate::admin::storage_api::access::{ReqInfo, authorize_request}; -use crate::admin::storage_api::contract::{ - BucketOperations, BucketOptions, ListOperations as _, ObjectIO as _, ObjectOperations as _, -}; +use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; +use crate::admin::storage_api::contract::list::ListOperations as _; +use crate::admin::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _}; use crate::admin::storage_api::object::StorageObjectOptions as ObjectOptions; use crate::auth::{check_key_valid, get_session_token}; use crate::error::ApiError; diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 1b12dfef7..7a4a4b406 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::storage_api::contract::{BucketOperations, BucketOptions, StorageAdminApi}; +use crate::admin::storage_api::contract::admin::StorageAdminApi; +use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::rebalance::{ DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStopPropagationRecord, diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 426282b74..354dbe2b9 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -22,7 +22,7 @@ use crate::admin::storage_api::bucket::metadata_sys::get_replication_config; use crate::admin::storage_api::bucket::replication::BucketStats; use crate::admin::storage_api::bucket::target::BucketTarget; use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys}; -use crate::admin::storage_api::contract::{BucketOperations, BucketOptions}; +use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::admin::storage_api::error::StorageError; use crate::admin::utils::read_compatible_admin_body; use crate::auth::{check_key_valid, get_session_token}; diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 1b9e038d2..f126fddc9 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -34,7 +34,7 @@ use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; use crate::admin::storage_api::bucket::utils::{deserialize, serialize}; use crate::admin::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; use crate::admin::storage_api::config::{delete_admin_config, read_admin_config, save_admin_config}; -use crate::admin::storage_api::contract::{ +use crate::admin::storage_api::contract::bucket::{ BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, }; use crate::admin::storage_api::error::Error as StorageError; diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 93f9c70bd..1bfa4266a 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -33,7 +33,7 @@ use crate::admin::runtime_sources::{ resolve_server_config, }; use crate::admin::storage_api::access::{ReqInfo, authorize_request, spawn_traced}; -use crate::admin::storage_api::contract::{BucketOperations, BucketOptions}; +use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::auth::{check_key_valid, get_session_token}; use crate::error::ApiError; use crate::license::license_check; diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index 52667540e..1e4735e6a 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -17,7 +17,7 @@ use crate::admin::runtime_sources::{ resolve_object_store_handle, resolve_object_store_handle_for_context, }; use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass}; -use crate::admin::storage_api::contract::StorageAdminApi; +use crate::admin::storage_api::contract::admin::StorageAdminApi; use crate::admin::storage_api::runtime::ECStore; use rustfs_audit::reload_audit_config; use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS}; diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 53f3f16bf..bd825f7bc 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -469,10 +469,27 @@ pub(crate) mod config { } pub(crate) mod contract { - pub(crate) use super::storage_contracts::{ - BucketOperations, BucketOptions, DeleteBucketOptions, HealOperations, ListOperations, MakeBucketOptions, ObjectIO, - ObjectOperations, SRBucketDeleteOp, StorageAdminApi, - }; + pub(crate) mod admin { + pub(crate) use super::super::storage_contracts::StorageAdminApi; + } + + pub(crate) mod bucket { + pub(crate) use super::super::storage_contracts::{ + BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, + }; + } + + pub(crate) mod heal { + pub(crate) use super::super::storage_contracts::HealOperations; + } + + pub(crate) mod list { + pub(crate) use super::super::storage_contracts::ListOperations; + } + + pub(crate) mod object { + pub(crate) use super::super::storage_contracts::{ObjectIO, ObjectOperations}; + } } pub(crate) mod error { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index d715eab49..98c94dcd9 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -409,6 +409,8 @@ pin_project! { #[pin] inner: ReaderStream, remaining: usize, + emitted: usize, + expected: usize, } } @@ -430,6 +432,8 @@ where Self { inner: ReaderStream::with_capacity(reader, capacity), remaining, + emitted: 0, + expected: remaining, } } } @@ -466,13 +470,32 @@ where bytes.truncate(*this.remaining); } *this.remaining -= bytes.len(); + #[cfg(feature = "tracing-chunk-debug")] + { + *this.emitted += bytes.len(); + tracing::debug!( + emitted = *this.emitted, + expected = *this.expected, + chunk_len = bytes.len(), + "GetObject ReaderStream emitted bytes" + ); + } if bytes.is_empty() { Poll::Ready(None) } else { Poll::Ready(Some(Ok(bytes))) } } - Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(Box::new(err)))), + Poll::Ready(Some(Err(err))) => { + #[cfg(feature = "tracing-chunk-debug")] + tracing::error!( + emitted = *this.emitted, + expected = *this.expected, + error = %err, + "GetObject ReaderStream returned error" + ); + Poll::Ready(Some(Err(Box::new(err)))) + } Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, } @@ -1734,25 +1757,6 @@ impl DefaultObjectUsecase { ); } let handoff_start = get_stage_metrics_enabled.then(std::time::Instant::now); - #[cfg(feature = "tracing-chunk-debug")] - let stream = { - let mut emitted = 0usize; - GetObjectReaderStream::new(reader, stream_buffer_size, expected).inspect(move |item| match item { - Ok(bytes) => { - emitted += bytes.len(); - tracing::debug!(emitted, expected, chunk_len = bytes.len(), "GetObject ReaderStream emitted bytes"); - } - Err(err) => { - tracing::error!( - emitted, - expected, - error = %err, - "GetObject ReaderStream returned error" - ); - } - }) - }; - #[cfg(not(feature = "tracing-chunk-debug"))] let stream = GetObjectReaderStream::new(reader, stream_buffer_size, expected); let blob = StreamingBlob::new(stream); if let Some(handoff_start) = handoff_start { diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 8b01d96d5..9bae8b4e9 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -95,6 +95,7 @@ RUSTFS_STORAGE_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_storage RUSTFS_STORAGE_OWNER_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_compat_reexport_hits.txt" RUSTFS_STORAGE_OWNER_DIRECT_STORAGE_SOURCE_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_direct_storage_source_hits.txt" RUSTFS_STORAGE_OWNER_CONTRACT_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_contract_root_consumer_hits.txt" +RUSTFS_ADMIN_CONTRACT_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_admin_contract_root_consumer_hits.txt" RUSTFS_ADMIN_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_admin_bucket_storage_compat_module_hits.txt" RUSTFS_APP_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/rustfs_app_bucket_storage_compat_module_hits.txt" RUSTFS_OUTER_COMPAT_FACADE_ALIAS_HITS_FILE="${TMP_DIR}/rustfs_outer_compat_facade_alias_hits.txt" @@ -1901,6 +1902,32 @@ if [[ -s "$RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" ]]; then report_failure "RustFS admin storage contracts must stay behind rustfs/src/admin/storage_api.rs: $(paste -sd '; ' "$RUSTFS_ADMIN_STORAGE_API_CONTRACT_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + { + rg -n -U --with-filename 'crate::admin::storage_api::contract::\{\s*[A-Z]' \ + rustfs/src/admin \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + rg -n --with-filename 'crate::admin::storage_api::contract::[A-Z]' \ + rustfs/src/admin \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + rg -n -U --with-filename 'super::storage_api::contract::\{\s*[A-Z]' \ + rustfs/src/admin \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + rg -n --with-filename 'super::storage_api::contract::[A-Z]' \ + rustfs/src/admin \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + } +) >"$RUSTFS_ADMIN_CONTRACT_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_ADMIN_CONTRACT_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS admin modules must import storage contracts from domain modules, not the root contract facade: $(paste -sd '; ' "$RUSTFS_ADMIN_CONTRACT_ROOT_CONSUMER_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename 'crate::app::context::|use crate::app::context|app::context::' rustfs/src/storage --glob '*.rs' |