diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index a9fba588f..6c0ae3c15 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,17 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-owner-compat-consumer-batch` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101`. -- Based on: `origin/main` after API-098/API-099/API-100 merge (#3708). +- Branch: `overtrue/arch-storage-core-compat-consumer-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102`. +- Based on: latest `main` after the API-101 merge. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: move admin handler/service/router, app usecase/context, - and storage RPC/S3 API ECStore compatibility consumers behind owner-local - compatibility boundaries. -- CI/script changes: guard against restoring direct owner compatibility - consumers outside local compatibility boundary modules. -- Docs changes: record the API-101 owner compatibility consumer cleanup. +- Rust code changes: move storage core ECStore compatibility consumers behind + the `core_storage_compat` boundary. +- CI/script changes: extend owner compatibility consumer guard coverage across + `rustfs/src/storage`. +- Docs changes: record the API-102 storage core compatibility consumer cleanup. ## Phase 0 Tasks @@ -449,6 +448,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block - Verification: RustFS test-target compile coverage, owner compatibility consumer residual scan, migration and layer guards, formatting, diff hygiene, Rust risk scan, pre-commit quality gate, and three-expert review. +- [x] `API-102` Localize storage core compatibility consumers. + - Completed slice: route storage access, ECFS, ECFS extension, head-prefix, + options, SSE, storage module aliases, and storage tests through + `core_storage_compat` instead of the storage owner `storage_compat.rs` + facade. + - Acceptance: no non-compat RustFS storage source imports + `crate::storage::storage_compat` directly; migration rules reject + regressions across `rustfs/src/storage`. + - Must preserve: bucket access validation, ECFS object operations, SSE + encryption/decryption setup, storage option mapping, storage object aliases, + and storage compatibility tests. + - Verification: RustFS test-target compile coverage, storage compatibility + consumer residual scan, migration and layer guards, formatting, diff + hygiene, Rust risk scan, pre-commit quality gate, and three-expert review. - [x] `G-012` Inventory placement and repair invariants. - Acceptance: [`placement-repair-invariants.md`](placement-repair-invariants.md) records @@ -3490,6 +3503,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: +- Issue #660 API-102 current slice: + - `cargo check -p rustfs --tests`: passed. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Storage compatibility consumer residual scan: passed. + - Rust risk scan on changed Rust files and guard script: passed. + - `make pre-commit`: passed. + - Issue #660 API-101 current slice: - `cargo check -p rustfs --tests`: passed. - `cargo fmt --all`: passed. diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index a4e4a2c9e..e6c5f247b 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -17,12 +17,12 @@ use crate::auth::{check_key_valid, get_condition_values_with_query_and_client_in use crate::error::ApiError; use crate::license::license_check; use crate::server::RemoteAddr; -use crate::storage::request_context::RequestContext; -use crate::storage::storage_compat::ECStore; -use crate::storage::storage_compat::resolve_object_store_handle; -use crate::storage::storage_compat::{ +use crate::storage::core_storage_compat::ECStore; +use crate::storage::core_storage_compat::resolve_object_store_handle; +use crate::storage::core_storage_compat::{ PolicySys, StorageError, get_bucket_metadata, get_bucket_policy_raw, get_public_access_block_config, is_err_bucket_not_found, }; +use crate::storage::request_context::RequestContext; use metrics::counter; use rustfs_iam::error::Error as IamError; use rustfs_policy::policy::action::{Action, AdminAction, S3Action}; @@ -930,7 +930,7 @@ impl S3Access for FS { let req_info = ReqInfo { cred, is_owner, - region: crate::storage::storage_compat::get_global_region(), + region: crate::storage::core_storage_compat::get_global_region(), request_context, ..Default::default() }; diff --git a/rustfs/src/storage/core_storage_compat.rs b/rustfs/src/storage/core_storage_compat.rs new file mode 100644 index 000000000..f3d75945f --- /dev/null +++ b/rustfs/src/storage/core_storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use crate::storage::storage_compat::*; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index c92fe19ee..b68432b87 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -18,10 +18,7 @@ use crate::app::multipart_usecase::DefaultMultipartUsecase; use crate::app::object_usecase::DefaultObjectUsecase; use crate::error::ApiError; use crate::storage::access::has_bypass_governance_header; -use crate::storage::helper::OperationHelper; -use crate::storage::options::get_opts; -use crate::storage::s3_api::acl; -use crate::storage::storage_compat::{ +use crate::storage::core_storage_compat::{ BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, check_retention_for_modification, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, encode_tags, get_bucket_accelerate_config, get_bucket_logging_config, @@ -29,7 +26,10 @@ use crate::storage::storage_compat::{ is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, record_replication_proxy, serialize, update_bucket_metadata_config, }; -use crate::storage::storage_compat::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; +use crate::storage::core_storage_compat::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; +use crate::storage::helper::OperationHelper; +use crate::storage::options::get_opts; +use crate::storage::s3_api::acl; use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled}; use crate::table_catalog; use http::StatusCode; diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 2dff27e63..f570850bf 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -15,12 +15,12 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled}; use crate::error::ApiError; use crate::server::cors; -use crate::storage::ecfs::ListObjectUnorderedQuery; -use crate::storage::storage_compat::StorageReplicationConfigExt as _; -use crate::storage::storage_compat::{ +use crate::storage::core_storage_compat::StorageReplicationConfigExt as _; +use crate::storage::core_storage_compat::{ StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, get_bucket_replication_config, resolve_object_store_handle, }; +use crate::storage::ecfs::ListObjectUnorderedQuery; use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; use http::{HeaderMap, HeaderValue, StatusCode}; use metrics::counter; @@ -738,7 +738,7 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet } /// Helper function to get store and validate bucket exists -pub(crate) async fn get_validated_store(bucket: &str) -> S3Result> { +pub(crate) async fn get_validated_store(bucket: &str) -> S3Result> { let Some(store) = resolve_object_store_handle() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 79c7f8f7d..e33d19d5c 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -16,11 +16,11 @@ mod tests { use crate::config::WorkloadProfile; use crate::server::cors; - use crate::storage::ecfs::{FS, validate_object_lock_configuration_input}; - use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; - use crate::storage::storage_compat::{ + use crate::storage::core_storage_compat::{ BucketMetadata, DEFAULT_READ_BUFFER_SIZE, get_global_bucket_metadata_sys, set_bucket_metadata, }; + use crate::storage::ecfs::{FS, validate_object_lock_configuration_input}; + use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use crate::storage::{ StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag, @@ -941,8 +941,8 @@ mod tests { #[tokio::test] async fn test_validate_bucket_object_lock_enabled() { - use crate::storage::storage_compat::bucket_metadata_sys_initialized; - use crate::storage::storage_compat::set_bucket_metadata; + use crate::storage::core_storage_compat::bucket_metadata_sys_initialized; + use crate::storage::core_storage_compat::set_bucket_metadata; use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; use time::OffsetDateTime; @@ -1781,7 +1781,7 @@ mod tests { /// with a single-element vec value, matching the format expected by policy evaluation. #[test] fn test_object_tag_condition_key_format() { - use crate::storage::storage_compat::decode_tags_to_map; + use crate::storage::core_storage_compat::decode_tags_to_map; use std::collections::HashMap; let tags_str = "security=public&project=webapp&env=prod"; diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 1e93de115..ffd717f5c 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::storage_compat::ECStore; +use crate::storage::core_storage_compat::ECStore; use rustfs_storage_api::ListOperations as _; use std::sync::Arc; diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 62de03fae..bff384c3f 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -15,6 +15,7 @@ pub mod access; pub mod backpressure; pub mod concurrency; +pub(crate) mod core_storage_compat; pub mod deadlock_detector; pub mod ecfs; pub(crate) mod helper; @@ -29,11 +30,11 @@ pub mod timeout_wrapper; pub mod tonic_service; pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject; -pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::GetObjectReader; -pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ObjectInfo; -pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ObjectOptions; +pub(crate) type StorageGetObjectReader = crate::storage::core_storage_compat::GetObjectReader; +pub(crate) type StorageObjectInfo = crate::storage::core_storage_compat::ObjectInfo; +pub(crate) type StorageObjectOptions = crate::storage::core_storage_compat::ObjectOptions; pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete; -pub(crate) type StoragePutObjReader = crate::storage::storage_compat::PutObjReader; +pub(crate) type StoragePutObjReader = crate::storage::core_storage_compat::PutObjReader; #[cfg(test)] mod concurrent_fix_test; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index bf5b98393..a96324d90 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::storage_compat::BucketVersioningSys; -use crate::storage::storage_compat::Result; -use crate::storage::storage_compat::StorageError; +use crate::storage::core_storage_compat::BucketVersioningSys; +use crate::storage::core_storage_compat::Result; +use crate::storage::core_storage_compat::StorageError; use http::header::{IF_MATCH, IF_NONE_MATCH}; use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 6c54b9d75..9774a0f92 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -69,7 +69,7 @@ //! } //! ``` -use crate::storage::storage_compat::StorageError; +use crate::storage::core_storage_compat::StorageError; #[cfg(feature = "rio-v2")] use aes_gcm::aead::Payload; use aes_gcm::{ @@ -134,8 +134,8 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE; const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation"; use crate::error::ApiError; -use crate::storage::storage_compat::Error; -use crate::storage::storage_compat::get_bucket_sse_config; +use crate::storage::core_storage_compat::Error; +use crate::storage::core_storage_compat::get_bucket_sse_config; use rustfs_utils::http::headers::{ AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, @@ -751,19 +751,19 @@ pub struct ManagedSealedKey { } impl EncryptionMaterial { - pub fn write_encryption(&self, multipart_part_number: Option) -> crate::storage::storage_compat::WriteEncryption { + pub fn write_encryption(&self, multipart_part_number: Option) -> crate::storage::core_storage_compat::WriteEncryption { match (self.key_kind, multipart_part_number) { (EncryptionKeyKind::Object, Some(part_number)) => { - crate::storage::storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32) + crate::storage::core_storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32) } (EncryptionKeyKind::Object, None) => { - crate::storage::storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes) + crate::storage::core_storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes) } (EncryptionKeyKind::Direct, Some(part_number)) => { - crate::storage::storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number) + crate::storage::core_storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number) } (EncryptionKeyKind::Direct, None) => { - crate::storage::storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce) + crate::storage::core_storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce) } } } diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 5cf237865..9f1e048fb 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -838,7 +838,7 @@ fi rustfs/src/app \ --glob '!**/*storage_compat.rs' || true rg -n --no-heading 'crate::storage::storage_compat' \ - rustfs/src/storage/rpc rustfs/src/storage/s3_api \ + rustfs/src/storage \ --glob '!**/*storage_compat.rs' || true } ) >"$RUSTFS_OWNER_COMPAT_CONSUMER_HITS_FILE"